From d1f127d8eff872cab49db922056df1d21c0514c7 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Tue, 21 Jul 2015 10:15:17 -0500 Subject: [PATCH 001/910] Add changes for krell related products to binutils. Add a patch that installs libiberty_pic.a which krell products will favor/use. --- .../binutils/binutilskrell-2.24.patch | 52 +++++++++++++++++++ var/spack/packages/binutils/package.py | 26 +++++++--- 2 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 var/spack/packages/binutils/binutilskrell-2.24.patch diff --git a/var/spack/packages/binutils/binutilskrell-2.24.patch b/var/spack/packages/binutils/binutilskrell-2.24.patch new file mode 100644 index 00000000000..f48291a6c91 --- /dev/null +++ b/var/spack/packages/binutils/binutilskrell-2.24.patch @@ -0,0 +1,52 @@ +--- binutils-2.24/libiberty/Makefile.in 2013-11-04 10:33:40.000000000 -0500 ++++ binutils-2.24-fixes/libiberty/Makefile.in 2014-10-17 16:22:31.413655000 -0400 +@@ -66,6 +66,7 @@ + MAKEOVERRIDES = + + TARGETLIB = ./libiberty.a ++TARGETLIBPIC = ./libiberty_pic.a + TESTLIB = ./testlib.a + + LIBOBJS = @LIBOBJS@ +@@ -355,27 +356,27 @@ + # since it will be passed the multilib flags. + MULTIOSDIR = `$(CC) $(CFLAGS) -print-multi-os-directory` + install_to_libdir: all +- if test -n "${target_header_dir}"; then \ +- ${mkinstalldirs} $(DESTDIR)$(libdir)/$(MULTIOSDIR); \ +- $(INSTALL_DATA) $(TARGETLIB) $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIB)n; \ +- ( cd $(DESTDIR)$(libdir)/$(MULTIOSDIR) ; chmod 644 $(TARGETLIB)n ;$(RANLIB) $(TARGETLIB)n ); \ +- mv -f $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIB)n $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIB); \ +- case "${target_header_dir}" in \ +- /*) thd=${target_header_dir};; \ +- *) thd=${includedir}/${target_header_dir};; \ +- esac; \ +- ${mkinstalldirs} $(DESTDIR)$${thd}; \ +- for h in ${INSTALLED_HEADERS}; do \ +- ${INSTALL_DATA} $$h $(DESTDIR)$${thd}; \ +- done; \ +- fi ++ ${mkinstalldirs} $(DESTDIR)$(libdir)/$(MULTIOSDIR); \ ++ $(INSTALL_DATA) $(TARGETLIB) $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIB)n; \ ++ $(INSTALL_DATA) pic/$(TARGETLIB) $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIBPIC)n; \ ++ ( cd $(DESTDIR)$(libdir)/$(MULTIOSDIR) ; chmod 644 $(TARGETLIB)n ;$(RANLIB) $(TARGETLIB)n ); \ ++ ( cd $(DESTDIR)$(libdir)/$(MULTIOSDIR) ; chmod 644 $(TARGETLIBPIC)n ;$(RANLIB) $(TARGETLIBPIC)n ); \ ++ mv -f $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIB)n $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIB); \ ++ mv -f $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIBPIC)n $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIBPIC); \ ++ ${mkinstalldirs} $(DESTDIR)$${includedir}; \ ++ for h in ${INSTALLED_HEADERS}; do \ ++ ${INSTALL_DATA} $$h $(DESTDIR)$${includedir}; \ ++ done; + @$(MULTIDO) $(FLAGS_TO_PASS) multi-do DO=install + + install_to_tooldir: all + ${mkinstalldirs} $(DESTDIR)$(tooldir)/lib/$(MULTIOSDIR) + $(INSTALL_DATA) $(TARGETLIB) $(DESTDIR)$(tooldir)/lib/$(MULTIOSDIR)/$(TARGETLIB)n ++ $(INSTALL_DATA) pic/$(TARGETLIB) $(DESTDIR)$(tooldir)/lib/$(MULTIOSDIR)/$(TARGETLIBPIC)n + ( cd $(DESTDIR)$(tooldir)/lib/$(MULTIOSDIR) ; chmod 644 $(TARGETLIB)n; $(RANLIB) $(TARGETLIB)n ) ++ ( cd $(DESTDIR)$(tooldir)/lib/$(MULTIOSDIR) ; chmod 644 $(TARGETLIBPIC)n; $(RANLIB) $(TARGETLIBPIC)n ) + mv -f $(DESTDIR)$(tooldir)/lib/$(MULTIOSDIR)/$(TARGETLIB)n $(DESTDIR)$(tooldir)/lib/$(MULTIOSDIR)/$(TARGETLIB) ++ mv -f $(DESTDIR)$(tooldir)/lib/$(MULTIOSDIR)/$(TARGETLIBPIC)n $(DESTDIR)$(tooldir)/lib/$(MULTIOSDIR)/$(TARGETLIBPIC) + @$(MULTIDO) $(FLAGS_TO_PASS) multi-do DO=install + + # required-list was used when building a shared bfd/opcodes/libiberty diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py index 5a3059bbcf5..43a670585e6 100644 --- a/var/spack/packages/binutils/package.py +++ b/var/spack/packages/binutils/package.py @@ -3,15 +3,29 @@ class Binutils(Package): """GNU binutils, which contain the linker, assembler, objdump and others""" homepage = "http://www.gnu.org/software/binutils/" - url = "ftp://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2" - version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66') - version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b') - version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e') - version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764') + version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2") + version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.24.tar.bz2") + version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.23.2.tar.bz2") + version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.20.1.tar.bz2") + + # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell + variant('krellpatch', default=False, description="build with openspeedshop based patch.") + patch('binutilskrell-2.24.patch', when='@2.24+krellpatch') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + + # Add additional configuration options for use in the OpenSpeedShop project + if '+krellpatch' in spec: + configure('--prefix', self.prefix, + '--libdir', self.prefix.lib, + '--enable-shared', + '--enable-install-libiberty', + '--disable-multilib' + ) + else: + configure("--prefix=%s" % prefix) + make() make("install") From 0591980cbe4c91f60b5fe2ca811bc7ee306a1171 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Tue, 21 Jul 2015 10:17:37 -0500 Subject: [PATCH 002/910] Add configuration change for krell related products, they expect the dyninst includes in include/dyninst. --- var/spack/packages/dyninst/package.py | 34 ++++++++++++++++++++------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/var/spack/packages/dyninst/package.py b/var/spack/packages/dyninst/package.py index 41ec57dd2f0..81df9aed808 100644 --- a/var/spack/packages/dyninst/package.py +++ b/var/spack/packages/dyninst/package.py @@ -38,6 +38,8 @@ class Dyninst(Package): version('8.1.1', 'd1a04e995b7aa70960cd1d1fac8bd6ac', url="http://www.paradyn.org/release8.1/DyninstAPI-8.1.1.tgz") + variant('krelloptions', default=False, description="build dyninst includes into include/dyninst.") + depends_on("libelf") depends_on("libdwarf") depends_on("boost@1.42:") @@ -48,15 +50,29 @@ def install(self, spec, prefix): libdwarf = spec['libdwarf'].prefix with working_dir('spack-build', create=True): - cmake('..', - '-DBoost_INCLUDE_DIR=%s' % spec['boost'].prefix.include, - '-DBoost_LIBRARY_DIR=%s' % spec['boost'].prefix.lib, - '-DBoost_NO_SYSTEM_PATHS=TRUE', - '-DLIBELF_INCLUDE_DIR=%s' % join_path(libelf.include, 'libelf'), - '-DLIBELF_LIBRARIES=%s' % join_path(libelf.lib, 'libelf.so'), - '-DLIBDWARF_INCLUDE_DIR=%s' % libdwarf.include, - '-DLIBDWARF_LIBRARIES=%s' % join_path(libdwarf.lib, 'libdwarf.so'), - *std_cmake_args) + # cbtf-krell and openspeedshop expect to reference dyninst/include, so adding include specification to that end + if '+krelloptions' in spec: + cmake('..', + '-DINSTALL_INCLUDE_DIR=%s' % join_path(self.prefix.include, 'dyninst'), + '-DBoost_INCLUDE_DIR=%s' % spec['boost'].prefix.include, + '-DBoost_LIBRARY_DIR=%s' % spec['boost'].prefix.lib, + '-DBoost_NO_SYSTEM_PATHS=TRUE', + '-DLIBELF_INCLUDE_DIR=%s' % join_path(libelf.include, 'libelf'), + '-DLIBELF_LIBRARIES=%s' % join_path(libelf.lib, 'libelf.so'), + '-DLIBDWARF_INCLUDE_DIR=%s' % libdwarf.include, + '-DLIBDWARF_LIBRARIES=%s' % join_path(libdwarf.lib, 'libdwarf.so'), + *std_cmake_args) + else: + cmake('..', + '-DBoost_INCLUDE_DIR=%s' % spec['boost'].prefix.include, + '-DBoost_LIBRARY_DIR=%s' % spec['boost'].prefix.lib, + '-DBoost_NO_SYSTEM_PATHS=TRUE', + '-DLIBELF_INCLUDE_DIR=%s' % join_path(libelf.include, 'libelf'), + '-DLIBELF_LIBRARIES=%s' % join_path(libelf.lib, 'libelf.so'), + '-DLIBDWARF_INCLUDE_DIR=%s' % libdwarf.include, + '-DLIBDWARF_LIBRARIES=%s' % join_path(libdwarf.lib, 'libdwarf.so'), + *std_cmake_args) + make() make("install") From 13421dc4aa8dd0705728660e5bf6dab657cc170e Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Tue, 21 Jul 2015 10:20:05 -0500 Subject: [PATCH 003/910] Add patches that enable features added for krell related products. Update the package file to add a krellpatch variant to control adding the patches. --- .../libmonitor/libmonitorkrell-0000.patch | 18 + .../libmonitor/libmonitorkrell-0001.patch | 395 ++++++++++++++++++ .../libmonitor/libmonitorkrell-0002.patch | 106 +++++ var/spack/packages/libmonitor/package.py | 7 + 4 files changed, 526 insertions(+) create mode 100644 var/spack/packages/libmonitor/libmonitorkrell-0000.patch create mode 100644 var/spack/packages/libmonitor/libmonitorkrell-0001.patch create mode 100644 var/spack/packages/libmonitor/libmonitorkrell-0002.patch diff --git a/var/spack/packages/libmonitor/libmonitorkrell-0000.patch b/var/spack/packages/libmonitor/libmonitorkrell-0000.patch new file mode 100644 index 00000000000..3a901068506 --- /dev/null +++ b/var/spack/packages/libmonitor/libmonitorkrell-0000.patch @@ -0,0 +1,18 @@ +--- libmonitor-20130218/configure 2013-02-17 23:08:32.000000000 -0800 ++++ libmonitor-20130218-fixes/configure 2013-02-18 10:34:05.237918411 -0800 +@@ -3600,13 +3600,13 @@ + CFLAGS=$ac_save_CFLAGS + elif test $ac_cv_prog_cc_g = yes; then + if test "$GCC" = yes; then +- CFLAGS="-g -O2" ++ CFLAGS="-g -O0" + else + CFLAGS="-g" + fi + else + if test "$GCC" = yes; then +- CFLAGS="-O2" ++ CFLAGS="-O0" + else + CFLAGS= + fi diff --git a/var/spack/packages/libmonitor/libmonitorkrell-0001.patch b/var/spack/packages/libmonitor/libmonitorkrell-0001.patch new file mode 100644 index 00000000000..85d69595c0a --- /dev/null +++ b/var/spack/packages/libmonitor/libmonitorkrell-0001.patch @@ -0,0 +1,395 @@ +--- libmonitor-20130218/src/callback.c 2013-02-17 23:08:32.000000000 -0800 ++++ libmonitor-20130218-fixes/src/callback.c 2013-02-18 10:34:17.839842826 -0800 +@@ -173,3 +173,18 @@ + MONITOR_DEBUG1("(default callback)\n"); + return 0; + } ++ ++ ++void __attribute__ ((weak)) ++monitor_mpi_post_comm_rank(void) ++{ ++ MONITOR_DEBUG1("(default callback)\n"); ++} ++ ++void __attribute__ ((weak)) ++monitor_mpi_pcontrol(int level) ++{ ++ MONITOR_DEBUG("(default callback) level = %d\n", level); ++} ++ ++ +--- libmonitor-20130218/src/main.c 2013-02-17 23:08:32.000000000 -0800 ++++ libmonitor-20130218-fixes/src/main.c 2013-02-18 10:34:17.839842826 -0800 +@@ -230,6 +230,44 @@ + monitor_fini_library_called = 1; + } + ++/* ++ * Internal monitor functions. ++ */ ++ ++struct monitor_thread_node * __attribute__ ((weak)) ++monitor_get_tn(void) ++{ ++ return &monitor_main_tn; ++} ++ ++int __attribute__ ((weak)) ++monitor_get_thread_num(void) ++{ ++ return (0); ++} ++ ++void __attribute__ ((weak)) ++monitor_reset_thread_list(struct monitor_thread_node *main_tn) ++{ ++ MONITOR_DEBUG1("(weak)\n"); ++ return; ++} ++ ++void __attribute__ ((weak)) ++monitor_thread_release(void) ++{ ++ MONITOR_DEBUG1("(weak)\n"); ++ return; ++} ++ ++void __attribute__ ((weak)) ++monitor_thread_shootdown(void) ++{ ++ MONITOR_DEBUG1("(weak)\n"); ++ return; ++} ++ ++ + void + monitor_begin_process_fcn(void *user_data, int is_fork) + { +@@ -625,12 +663,6 @@ + return (monitor_main_tn.tn_user_data); + } + +-int __attribute__ ((weak)) +-monitor_get_thread_num(void) +-{ +- return (0); +-} +- + void * __attribute__ ((weak)) + monitor_get_addr_thread_start(void) + { +@@ -691,33 +723,3 @@ + MONITOR_DEBUG1("(weak)\n"); + return (FALSE); + } +- +-/* +- * Internal monitor functions. +- */ +-struct monitor_thread_node * __attribute__ ((weak)) +-monitor_get_tn(void) +-{ +- return &monitor_main_tn; +-} +- +-void __attribute__ ((weak)) +-monitor_reset_thread_list(struct monitor_thread_node *main_tn) +-{ +- MONITOR_DEBUG1("(weak)\n"); +- return; +-} +- +-void __attribute__ ((weak)) +-monitor_thread_release(void) +-{ +- MONITOR_DEBUG1("(weak)\n"); +- return; +-} +- +-void __attribute__ ((weak)) +-monitor_thread_shootdown(void) +-{ +- MONITOR_DEBUG1("(weak)\n"); +- return; +-} +--- libmonitor-20130218/src/monitor.h 2013-02-17 23:08:32.000000000 -0800 ++++ libmonitor-20130218-fixes/src/monitor.h 2013-02-18 10:34:17.840842821 -0800 +@@ -73,6 +73,11 @@ + extern void monitor_fini_mpi(void); + extern void monitor_mpi_post_fini(void); + ++extern void monitor_mpi_post_comm_rank(void); ++extern void monitor_mpi_pcontrol(int level); ++ ++ ++ + /* + * Monitor support functions. + */ +--- libmonitor-20130218/src/mpi_comm_c.c 2013-02-17 23:08:32.000000000 -0800 ++++ libmonitor-20130218-fixes/src/mpi_comm_c.c 2013-02-18 10:34:17.878842622 -0800 +@@ -36,5 +36,8 @@ + ret = (*real_mpi_comm_rank)(comm, rank); + monitor_set_mpi_size_rank(size, *rank); + ++ monitor_mpi_post_comm_rank(); ++ ++ + return (ret); + } +--- libmonitor-20130218/src/mpi_comm_f0.c 2013-02-17 23:08:32.000000000 -0800 ++++ libmonitor-20130218-fixes/src/mpi_comm_f0.c 2013-02-18 10:34:17.879842617 -0800 +@@ -34,4 +34,5 @@ + (*real_mpi_comm_size)(comm, &size, ierror); + (*real_mpi_comm_rank)(comm, rank, ierror); + monitor_set_mpi_size_rank(size, *rank); ++ monitor_mpi_post_comm_rank(); + } +--- libmonitor-20130218/src/mpi_comm_f1.c 2013-02-17 23:08:32.000000000 -0800 ++++ libmonitor-20130218-fixes/src/mpi_comm_f1.c 2013-02-18 10:34:17.880842612 -0800 +@@ -34,4 +34,5 @@ + (*real_mpi_comm_size)(comm, &size, ierror); + (*real_mpi_comm_rank)(comm, rank, ierror); + monitor_set_mpi_size_rank(size, *rank); ++ monitor_mpi_post_comm_rank(); + } +--- libmonitor-20130218/src/mpi_comm_f2.c 2013-02-17 23:08:32.000000000 -0800 ++++ libmonitor-20130218-fixes/src/mpi_comm_f2.c 2013-02-18 10:34:17.880842612 -0800 +@@ -34,4 +34,5 @@ + (*real_mpi_comm_size)(comm, &size, ierror); + (*real_mpi_comm_rank)(comm, rank, ierror); + monitor_set_mpi_size_rank(size, *rank); ++ monitor_mpi_post_comm_rank(); + } +--- libmonitor-20130218/src/pmpi.c 2013-02-17 23:08:32.000000000 -0800 ++++ libmonitor-20130218-fixes/src/pmpi.c 2013-02-18 10:34:17.881842607 -0800 +@@ -52,11 +52,15 @@ + typedef int mpi_init_thread_fcn_t(int *, char ***, int, int *); + typedef int mpi_finalize_fcn_t(void); + typedef int mpi_comm_fcn_t(void *, int *); ++typedef int mpi_pcontrol_fcn_t(int ); ++ + + typedef void f_mpi_init_fcn_t(int *); + typedef void f_mpi_init_thread_fcn_t(int *, int *, int *); + typedef void f_mpi_finalize_fcn_t(int *); + typedef void f_mpi_comm_fcn_t(int *, int *, int *); ++typedef int f_mpi_pcontrol_fcn_t(int ); ++ + + static mpi_init_fcn_t *real_pmpi_init = NULL; + static f_mpi_init_fcn_t *real_pmpi_init_f0 = NULL; +@@ -83,6 +87,12 @@ + static f_mpi_comm_fcn_t *real_pmpi_comm_rank_f1 = NULL; + static f_mpi_comm_fcn_t *real_pmpi_comm_rank_f2 = NULL; + ++static mpi_pcontrol_fcn_t *real_pmpi_pcontrol = NULL; ++static f_mpi_pcontrol_fcn_t *real_pmpi_pcontrol_f0 = NULL; ++static f_mpi_pcontrol_fcn_t *real_pmpi_pcontrol_f1 = NULL; ++static f_mpi_pcontrol_fcn_t *real_pmpi_pcontrol_f2 = NULL; ++ ++ + /* + *---------------------------------------------------------------------- + * PMPI_INIT OVERRIDE FUNCTIONS +@@ -297,6 +307,7 @@ + ret = (*real_pmpi_comm_size)(comm, &size); + ret = (*real_pmpi_comm_rank)(comm, rank); + monitor_set_mpi_size_rank(size, *rank); ++ monitor_mpi_post_comm_rank(); + + return (ret); + } +@@ -308,7 +319,9 @@ + MONITOR_GET_REAL_NAME_WRAP(rank_var, rank_fcn); \ + (*size_var)(comm, &size, ierror); \ + (*rank_var)(comm, rank, ierror); \ +- monitor_set_mpi_size_rank(size, *rank); ++ monitor_set_mpi_size_rank(size, *rank); \ ++ monitor_mpi_post_comm_rank(); ++ + + /* + * In Fortran, MPI_Comm is always int. +@@ -333,3 +346,48 @@ + FORTRAN_COMM_RANK_BODY(real_pmpi_comm_size_f2, pmpi_comm_size__, + real_pmpi_comm_rank_f2, pmpi_comm_rank__); + } ++ ++ ++/* ++ *---------------------------------------------------------------------- ++ * PMPI_PCONTROL OVERRIDE FUNCTIONS ++ *---------------------------------------------------------------------- ++ */ ++ ++int ++MONITOR_WRAP_NAME(PMPI_Pcontrol)(int level) ++{ ++ int ret; ++ ++ MONITOR_DEBUG("level = %d\n", level); \ ++ MONITOR_GET_REAL_NAME_WRAP(real_pmpi_pcontrol, PMPI_Pcontrol); ++ ret = (*real_pmpi_pcontrol) (level) ; ++ monitor_mpi_pcontrol(level); ++ ++ return (ret); ++} ++ ++#define FORTRAN_PCONTROL_BODY(var_name, fcn_name)\ ++ int ret; \ ++ MONITOR_DEBUG("level = %d\n", level); \ ++ MONITOR_GET_REAL_NAME_WRAP(var_name, fcn_name); \ ++ ret = (*var_name) (level) ; \ ++ monitor_mpi_pcontrol(level); ++ ++int ++MONITOR_WRAP_NAME(pmpi_pcontrol)(int level) ++{ ++ FORTRAN_PCONTROL_BODY(real_pmpi_pcontrol_f0, pmpi_pcontrol); ++} ++ ++int ++MONITOR_WRAP_NAME(pmpi_pcontrol_)(int level) ++{ ++ FORTRAN_PCONTROL_BODY(real_pmpi_pcontrol_f1, pmpi_pcontrol_); ++} ++ ++int ++MONITOR_WRAP_NAME(pmpi_pcontrol__)(int level) ++{ ++ FORTRAN_PCONTROL_BODY(real_pmpi_pcontrol_f2, pmpi_pcontrol__); ++} +--- libmonitor-20130218/src/mpi_pcontrol_c.c 1969-12-31 16:00:00.000000000 -0800 ++++ libmonitor-20130218-fixes/src/mpi_pcontrol_c.c 2013-02-18 10:34:17.882842602 -0800 +@@ -0,0 +1,30 @@ ++/* ++ * Override MPI_Pcontrol in C/C++. ++ * ++ * ++ */ ++ ++#include "config.h" ++#include "common.h" ++#include "monitor.h" ++ ++typedef int mpi_pcontrol_fcn_t(int level); ++#ifdef MONITOR_STATIC ++extern mpi_pcontrol_fcn_t __real_MPI_Pcontrol; ++#endif ++static mpi_pcontrol_fcn_t *real_mpi_pcontrol = NULL; ++ ++int ++MONITOR_WRAP_NAME(MPI_Pcontrol)(int level) ++{ ++ int ret, count; ++ ++ MONITOR_DEBUG("level = %d\n", level); \ ++ ++ MONITOR_GET_REAL_NAME_WRAP(real_mpi_pcontrol, MPI_Pcontrol); ++ ret = (*real_mpi_pcontrol)(level); ++ monitor_mpi_pcontrol(level); ++ ++ ++ return (ret); ++} +--- libmonitor-20130218/src/mpi_pcontrol_f0.c 1969-12-31 16:00:00.000000000 -0800 ++++ libmonitor-20130218-fixes/src/mpi_pcontrol_f0.c 2013-02-18 10:34:17.882842602 -0800 +@@ -0,0 +1,24 @@ ++/* ++ * Override mpi_pcontrol in Fortran. ++ * ++ */ ++ ++#include "config.h" ++#include "common.h" ++#include "monitor.h" ++ ++typedef void mpi_pcontrol_fcn_t(int level); ++#ifdef MONITOR_STATIC ++extern mpi_pcontrol_fcn_t __real_mpi_pcontrol; ++#endif ++static mpi_pcontrol_fcn_t *real_mpi_pcontrol = NULL; ++ ++int ++MONITOR_WRAP_NAME(mpi_pcontrol)(int level) ++{ ++ int count; ++ ++ MONITOR_DEBUG1("\n"); ++ MONITOR_GET_REAL_NAME_WRAP(real_mpi_pcontrol, mpi_pcontrol); ++ (*real_mpi_pcontrol)(level); ++} +--- libmonitor-20130218/src/mpi_pcontrol_f1.c 1969-12-31 16:00:00.000000000 -0800 ++++ libmonitor-20130218-fixes/src/mpi_pcontrol_f1.c 2013-02-18 10:34:17.883842597 -0800 +@@ -0,0 +1,24 @@ ++/* ++ * Override mpi_pcontrol_ in Fortran. ++ * ++ */ ++ ++#include "config.h" ++#include "common.h" ++#include "monitor.h" ++ ++typedef void mpi_pcontrol_fcn_t(int level); ++#ifdef MONITOR_STATIC ++extern mpi_pcontrol_fcn_t __real_mpi_pcontrol_; ++#endif ++static mpi_pcontrol_fcn_t *real_mpi_pcontrol = NULL; ++ ++int ++MONITOR_WRAP_NAME(mpi_pcontrol_)(int level) ++{ ++ int count; ++ ++ MONITOR_DEBUG1("\n"); ++ MONITOR_GET_REAL_NAME_WRAP(real_mpi_pcontrol, mpi_pcontrol_); ++ (*real_mpi_pcontrol)(level); ++} +--- libmonitor-20130218/src/mpi_pcontrol_f2.c 1969-12-31 16:00:00.000000000 -0800 ++++ libmonitor-20130218-fixes/src/mpi_pcontrol_f2.c 2013-02-18 10:34:17.883842597 -0800 +@@ -0,0 +1,24 @@ ++/* ++ * Override mpi_pcontrol__ in Fortran. ++ * ++ */ ++ ++#include "config.h" ++#include "common.h" ++#include "monitor.h" ++ ++typedef void mpi_pcontrol_fcn_t(int level); ++#ifdef MONITOR_STATIC ++extern mpi_pcontrol_fcn_t __real_mpi_pcontrol__; ++#endif ++static mpi_pcontrol_fcn_t *real_mpi_pcontrol = NULL; ++ ++int ++MONITOR_WRAP_NAME(mpi_pcontrol__)(int level) ++{ ++ int count; ++ ++ MONITOR_DEBUG1("\n"); ++ MONITOR_GET_REAL_NAME_WRAP(real_mpi_pcontrol, mpi_pcontrol__); ++ (*real_mpi_pcontrol)(level); ++} +--- libmonitor-20130218/src/Makefile.am 2013-02-17 23:08:32.000000000 -0800 ++++ libmonitor-20130218-fixes/src/Makefile.am 2013-02-18 10:34:17.931842343 -0800 +@@ -38,10 +38,11 @@ + MONITOR_THREAD_FILES = pthread.c + MONITOR_SCRIPT_FILES = monitor-link monitor-run + MONITOR_MPI_FILES = \ +- mpi_init_c.c mpi_init_thread_c.c mpi_final_c.c mpi_comm_c.c \ +- mpi_init_f0.c mpi_init_thread_f0.c mpi_final_f0.c mpi_comm_f0.c \ +- mpi_init_f1.c mpi_init_thread_f1.c mpi_final_f1.c mpi_comm_f1.c \ +- mpi_init_f2.c mpi_init_thread_f2.c mpi_final_f2.c mpi_comm_f2.c ++ mpi_init_c.c mpi_init_thread_c.c mpi_final_c.c mpi_comm_c.c mpi_pcontrol_c.c \ ++ mpi_init_f0.c mpi_init_thread_f0.c mpi_final_f0.c mpi_comm_f0.c mpi_pcontrol_f0.c \ ++ mpi_init_f1.c mpi_init_thread_f1.c mpi_final_f1.c mpi_comm_f1.c mpi_pcontrol_f1.c \ ++ mpi_init_f2.c mpi_init_thread_f2.c mpi_final_f2.c mpi_comm_f2.c mpi_pcontrol_f2.c ++ + + include_HEADERS = monitor.h + bin_SCRIPTS = diff --git a/var/spack/packages/libmonitor/libmonitorkrell-0002.patch b/var/spack/packages/libmonitor/libmonitorkrell-0002.patch new file mode 100644 index 00000000000..514dfdf13e1 --- /dev/null +++ b/var/spack/packages/libmonitor/libmonitorkrell-0002.patch @@ -0,0 +1,106 @@ +--- libmonitor-20130218/src/pmpi.c 2013-02-18 11:34:17.000000000 -0700 ++++ libmonitor-20130218-fixed/src/pmpi.c 2013-04-11 10:03:59.300550393 -0600 +@@ -355,7 +355,7 @@ + */ + + int +-MONITOR_WRAP_NAME(PMPI_Pcontrol)(int level) ++MONITOR_WRAP_NAME(PMPI_Pcontrol)(int level, int *ierror ) + { + int ret; + +@@ -375,19 +375,19 @@ + monitor_mpi_pcontrol(level); + + int +-MONITOR_WRAP_NAME(pmpi_pcontrol)(int level) ++MONITOR_WRAP_NAME(pmpi_pcontrol)(int level, int *ierror ) + { + FORTRAN_PCONTROL_BODY(real_pmpi_pcontrol_f0, pmpi_pcontrol); + } + + int +-MONITOR_WRAP_NAME(pmpi_pcontrol_)(int level) ++MONITOR_WRAP_NAME(pmpi_pcontrol_)(int level, int *ierror ) + { + FORTRAN_PCONTROL_BODY(real_pmpi_pcontrol_f1, pmpi_pcontrol_); + } + + int +-MONITOR_WRAP_NAME(pmpi_pcontrol__)(int level) ++MONITOR_WRAP_NAME(pmpi_pcontrol__)(int level, int *ierror ) + { + FORTRAN_PCONTROL_BODY(real_pmpi_pcontrol_f2, pmpi_pcontrol__); + } +--- libmonitor-20130218/src/mpi_pcontrol_f0.c 2013-02-18 11:34:17.000000000 -0700 ++++ libmonitor-20130218-fixed/src/mpi_pcontrol_f0.c 2013-04-11 10:13:47.783002000 -0600 +@@ -7,18 +7,18 @@ + #include "common.h" + #include "monitor.h" + +-typedef void mpi_pcontrol_fcn_t(int level); ++typedef void mpi_pcontrol_fcn_t(int level, int *ierror); + #ifdef MONITOR_STATIC + extern mpi_pcontrol_fcn_t __real_mpi_pcontrol; + #endif + static mpi_pcontrol_fcn_t *real_mpi_pcontrol = NULL; + + int +-MONITOR_WRAP_NAME(mpi_pcontrol)(int level) ++MONITOR_WRAP_NAME(mpi_pcontrol)(int level, int *ierror) + { + int count; + + MONITOR_DEBUG1("\n"); + MONITOR_GET_REAL_NAME_WRAP(real_mpi_pcontrol, mpi_pcontrol); +- (*real_mpi_pcontrol)(level); ++ (*real_mpi_pcontrol)(level, ierror); + } +--- libmonitor-20130218/src/mpi_pcontrol_f1.c 2013-02-18 11:34:17.000000000 -0700 ++++ libmonitor-20130218-fixed/src/mpi_pcontrol_f1.c 2013-04-11 10:14:08.039214000 -0600 +@@ -7,18 +7,18 @@ + #include "common.h" + #include "monitor.h" + +-typedef void mpi_pcontrol_fcn_t(int level); ++typedef void mpi_pcontrol_fcn_t(int level, int *ierror); + #ifdef MONITOR_STATIC + extern mpi_pcontrol_fcn_t __real_mpi_pcontrol_; + #endif + static mpi_pcontrol_fcn_t *real_mpi_pcontrol = NULL; + + int +-MONITOR_WRAP_NAME(mpi_pcontrol_)(int level) ++MONITOR_WRAP_NAME(mpi_pcontrol_)(int level, int *ierror) + { + int count; + + MONITOR_DEBUG1("\n"); + MONITOR_GET_REAL_NAME_WRAP(real_mpi_pcontrol, mpi_pcontrol_); +- (*real_mpi_pcontrol)(level); ++ (*real_mpi_pcontrol)(level, ierror); + } +--- libmonitor-20130218/src/mpi_pcontrol_f2.c 2013-02-18 11:34:17.000000000 -0700 ++++ libmonitor-20130218-fixed/src/mpi_pcontrol_f2.c 2013-04-11 10:14:19.000960000 -0600 +@@ -7,18 +7,18 @@ + #include "common.h" + #include "monitor.h" + +-typedef void mpi_pcontrol_fcn_t(int level); ++typedef void mpi_pcontrol_fcn_t(int level, int *ierror); + #ifdef MONITOR_STATIC + extern mpi_pcontrol_fcn_t __real_mpi_pcontrol__; + #endif + static mpi_pcontrol_fcn_t *real_mpi_pcontrol = NULL; + + int +-MONITOR_WRAP_NAME(mpi_pcontrol__)(int level) ++MONITOR_WRAP_NAME(mpi_pcontrol__)(int level, int *ierror) + { + int count; + + MONITOR_DEBUG1("\n"); + MONITOR_GET_REAL_NAME_WRAP(real_mpi_pcontrol, mpi_pcontrol__); +- (*real_mpi_pcontrol)(level); ++ (*real_mpi_pcontrol)(level, ierror); + } diff --git a/var/spack/packages/libmonitor/package.py b/var/spack/packages/libmonitor/package.py index 3b95b86ddf5..b31d7576c00 100644 --- a/var/spack/packages/libmonitor/package.py +++ b/var/spack/packages/libmonitor/package.py @@ -29,6 +29,13 @@ class Libmonitor(Package): homepage = "http://hpctoolkit.org" version('20130218', svn='http://libmonitor.googlecode.com/svn/trunk/', revision=146) + variant('krellpatch', default=False, description="build with openspeedshop based patch.") + + + patch('libmonitorkrell-0000.patch', when='@20130218+krellpatch') + patch('libmonitorkrell-0001.patch', when='@20130218+krellpatch') + patch('libmonitorkrell-0002.patch', when='@20130218+krellpatch') + def install(self, spec, prefix): configure("--prefix=" + prefix) From 64c8fd3fa18dd6644a67cbd9e9aa5f20eb5e85a7 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Tue, 21 Jul 2015 10:49:15 -0500 Subject: [PATCH 004/910] Add krelloptions variant that is used to turn on a configuration option to build the thread safe lightweight libraries. --- var/spack/packages/mrnet/package.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/mrnet/package.py b/var/spack/packages/mrnet/package.py index 6e9766f275c..08c57d74369 100644 --- a/var/spack/packages/mrnet/package.py +++ b/var/spack/packages/mrnet/package.py @@ -8,12 +8,17 @@ class Mrnet(Package): version('4.0.0', 'd00301c078cba57ef68613be32ceea2f') version('4.1.0', '5a248298b395b329e2371bf25366115c') + variant('krelloptions', default=False, description="Also build the MRNet LW threadsafe libraries") parallel = False depends_on("boost") def install(self, spec, prefix): - configure("--prefix=%s" %prefix, "--enable-shared") + # Build the MRNet LW thread safe libraries when the krelloptions variant is present + if '+krelloptions' in spec: + configure("--prefix=%s" %prefix, "--enable-shared", "--enable-ltwt-threadsafe") + else: + configure("--prefix=%s" %prefix, "--enable-shared") make() make("install") From b4e0804b374f1a725d42bdf3a11c3602ff009137 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Tue, 21 Jul 2015 10:50:11 -0500 Subject: [PATCH 005/910] Add the latest version to the papi package. --- var/spack/packages/papi/package.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/papi/package.py b/var/spack/packages/papi/package.py index 596f7114d60..947311ce3b7 100644 --- a/var/spack/packages/papi/package.py +++ b/var/spack/packages/papi/package.py @@ -11,8 +11,11 @@ class Papi(Package): components that expose performance measurement opportunites across the hardware and software stack.""" homepage = "http://icl.cs.utk.edu/papi/index.html" - url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.3.0.tar.gz" + url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.4.1.tar.gz" + version('5.4.1', '9134a99219c79767a11463a76b0b01a2') + + url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.3.0.tar.gz" version('5.3.0', '367961dd0ab426e5ae367c2713924ffb') def install(self, spec, prefix): From 15ec3cb3cd713e2ae1810440d01535f6990c4909 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Tue, 21 Jul 2015 10:51:53 -0500 Subject: [PATCH 006/910] Add qt3 version support and a variant that controls patching the qt3 version for compile issues and answering yes to the license questions during build. --- var/spack/packages/qt/package.py | 22 ++++++++- var/spack/packages/qt/qt3krell.patch | 68 ++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 var/spack/packages/qt/qt3krell.patch diff --git a/var/spack/packages/qt/package.py b/var/spack/packages/qt/package.py index 0e4abe3b1d7..e8d843519de 100644 --- a/var/spack/packages/qt/package.py +++ b/var/spack/packages/qt/package.py @@ -10,14 +10,23 @@ class Qt(Package): version('5.4.0', 'e8654e4b37dd98039ba20da7a53877e6', url='http://download.qt-project.org/official_releases/qt/5.4/5.4.0/single/qt-everywhere-opensource-src-5.4.0.tar.gz') + version('5.3.2', 'febb001129927a70174467ecb508a682', url='http://download.qt.io/archive/qt/5.3/5.3.2/single/qt-everywhere-opensource-src-5.3.2.tar.gz') version('5.2.1', 'a78408c887c04c34ce615da690e0b4c8', url='http://download.qt.io/archive/qt/5.2/5.2.1/single/qt-everywhere-opensource-src-5.2.1.tar.gz') + version('4.8.6', '2edbe4d6c2eff33ef91732602f3518eb', url="http://download.qt-project.org/official_releases/qt/4.8/4.8.6/qt-everywhere-opensource-src-4.8.6.tar.gz") + version('3.3.8b', '9f05b4125cfe477cc52c9742c3c09009', + url="http://download.qt.io/archive/qt/3/qt-x11-free-3.3.8b.tar.gz") + + # Add patch for compile issues with qt3 found with use in the OpenSpeedShop project + variant('krellpatch', default=False, description="build with openspeedshop based patch.") + patch('qt3krell.patch', when='@3.3.8b+krellpatch') + # Use system openssl for security. #depends_on("openssl") @@ -25,7 +34,7 @@ class Qt(Package): depends_on("gtkplus") depends_on("libxml2") depends_on("zlib") - depends_on("dbus") + depends_on("dbus", when='@4:') depends_on("libtiff") depends_on("libpng") depends_on("libmng") @@ -39,7 +48,7 @@ class Qt(Package): # depends_on("icu4c") # OpenGL hardware acceleration - depends_on("mesa") + depends_on("mesa", when='@4:') depends_on("libxcb") @@ -85,6 +94,15 @@ def common_config_args(self): # Don't disable all the database drivers, but should # really get them into spack at some point. + @when('@3') + def configure(self): + configure('-prefix', self.prefix, + '-v', + '-thread', + '-shared', + '-release', + '-fast' + ) @when('@4') def configure(self): diff --git a/var/spack/packages/qt/qt3krell.patch b/var/spack/packages/qt/qt3krell.patch new file mode 100644 index 00000000000..3333eeacd4c --- /dev/null +++ b/var/spack/packages/qt/qt3krell.patch @@ -0,0 +1,68 @@ +--- qt-x11-free-3.3.8b/src/tools/qmap.h 2008-01-15 13:09:13.000000000 -0600 ++++ qt-x11-free-3.3.8b-fixes/src/tools/qmap.h 2015-07-08 15:47:34.757565247 -0500 +@@ -52,6 +52,7 @@ + #ifndef QT_NO_STL + #include + #include ++#include + #endif + + //#define QT_CHECK_MAP_RANGE +--- qt-x11-free-3.3.8b/src/tools/qvaluelist.h 2008-01-15 13:09:13.000000000 -0600 ++++ qt-x11-free-3.3.8b-fixes/src/tools/qvaluelist.h 2015-07-08 15:47:34.758565247 -0500 +@@ -50,6 +50,7 @@ + #ifndef QT_NO_STL + #include + #include ++#include + #endif + + //#define QT_CHECK_VALUELIST_RANGE +--- qt-x11-free-3.3.8b/src/tools/qvaluevector.h 2008-01-15 13:09:13.000000000 -0600 ++++ qt-x11-free-3.3.8b-fixes/src/tools/qvaluevector.h 2015-07-08 15:47:34.758565247 -0500 +@@ -47,6 +47,7 @@ + + #ifndef QT_NO_STL + #include ++#include + #endif + + template +--- qt-x11-free-3.3.8b/configure 2008-01-15 13:09:15.000000000 -0600 ++++ qt-x11-free-3.3.8b-fixes/configure 2015-07-08 15:49:03.379560633 -0500 +@@ -2339,7 +2339,7 @@ + else + echo "Do you accept the terms of the $TheLicense? \c" + fi +- read acceptance ++ acceptance=yes + echo + if [ "$acceptance" = yes ]; then + break +@@ -2397,7 +2397,7 @@ + else + echo "Do you accept the terms of $affix license? \c" + fi +- read acceptance ++ acceptance=yes + echo + if [ "$acceptance" = "yes" ]; then + break +@@ -2443,7 +2443,7 @@ + else + echo "Do you accept the terms of the license? \c" + fi +- read acceptance ++ acceptance=yes + echo + if [ "$acceptance" = "yes" ]; then + break +@@ -2524,7 +2524,7 @@ + else + echo "Do you accept the terms of the $Platform License? \c" + fi +- read acceptance ++ acceptance=yes + echo + if [ "$acceptance" = "yes" ]; then + break From f5093094eb715e8e3072ecfc3c526951e47d22fe Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Tue, 21 Jul 2015 10:58:16 -0500 Subject: [PATCH 007/910] Add the cbtf-argonavis project package file. cbtf-argonavis is a project based on CBTF that supports performance analysis of GPU applications. --- var/spack/packages/cbtf-argonavis/package.py | 65 ++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 var/spack/packages/cbtf-argonavis/package.py diff --git a/var/spack/packages/cbtf-argonavis/package.py b/var/spack/packages/cbtf-argonavis/package.py new file mode 100644 index 00000000000..5f76a18dae4 --- /dev/null +++ b/var/spack/packages/cbtf-argonavis/package.py @@ -0,0 +1,65 @@ +################################################################################ +# Copyright (c) 2015 Krell Institute. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA +################################################################################ + +from spack import * + +class CbtfArgonavis(Package): + """CBTF Argo Navis project contains the CUDA collector and supporting + libraries that was done as a result of a DOE SBIR grant.""" + homepage = "http://sourceforge.net/p/cbtf/wiki/Home/" + + url = "http://sourceforge.net/projects/cbtf/files/cbtf-1.5/cbtf-argonavis-1.5.tar.gz/download" + version('1.5', '8708197dbc08980fcd7b1a3a3a40a2c9') + + # Mirror access template example + #url = "file:/opt/spack-mirror-2015-02-27/cbtf-argonavis/cbtf-argonavis-1.5.tar.gz" + #version('1.5', '1f7f6512f55409ed2135cfceabe26b82') + + depends_on("cmake") + depends_on("papi") + depends_on("cbtf") + depends_on("cbtf-krell") + #depends_on("cuda") + + parallel = False + + def install(self, spec, prefix): + + # Look for package installation information in the cbtf and cbtf-krell prefixes + cmake_prefix_path = join_path(spec['cbtf'].prefix) + ':' + join_path(spec['cbtf-krell'].prefix) + + # FIXME, hard coded for testing purposes, we will alter when the external package feature is available + cuda_prefix_path = "/usr/local/cuda-6.0" + cupti_prefix_path = "/usr/local/cuda-6.0/extras/CUPTI" + + with working_dir('CUDA'): + with working_dir('build', create=True): + cmake('..', + '-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, + '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, + '-DCUDA_INSTALL_PATH=%s' % cuda_prefix_path, + '-DCUDA_ROOT=%s' % cuda_prefix_path, + '-DCUPTI_ROOT=%s' % cupti_prefix_path, + '-DPAPI_ROOT=%s' % spec['papi'].prefix, + '-DCBTF_PREFIX=%s' % spec['cbtf'].prefix, + *std_cmake_args) + make("clean") + make() + make("install") + From a9c737f9113baa4e7ae16dfe4af3d6669f521b42 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Tue, 21 Jul 2015 10:59:52 -0500 Subject: [PATCH 008/910] Add spack build package for Xerces-C which is a supporting package for the krell related projects. --- var/spack/packages/xerces-c/package.py | 36 ++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 var/spack/packages/xerces-c/package.py diff --git a/var/spack/packages/xerces-c/package.py b/var/spack/packages/xerces-c/package.py new file mode 100644 index 00000000000..b59ab178aed --- /dev/null +++ b/var/spack/packages/xerces-c/package.py @@ -0,0 +1,36 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install xerces-c +# +# You can always get back here to change things with: +# +# spack edit xerces-c +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class XercesC(Package): + """ Xerces-C++ is a validating XML parser written in a portable subset of C++. + Xerces-C++ makes it easy to give your application the ability to read and + write XML data. A shared library is provided for parsing, generating, + manipulating, and validating XML documents using the DOM, SAX, and SAX2 APIs. + """ + + homepage = "https://xerces.apache.org/xerces-c" + url = "https://www.apache.org/dist/xerces/c/3/sources/xerces-c-3.1.2.tar.gz" + version('3.1.2', '9eb1048939e88d6a7232c67569b23985') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--disable-network") + make("clean") + make() + make("install") + From eff5016653980f24c5c55dfb866dbe108f50dedf Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Tue, 21 Jul 2015 11:15:53 -0500 Subject: [PATCH 009/910] Add the cbtf spack build package. cbtf is the base package for the component based tool framework and is used for building and connecting cbtf components, including distributed components via the MRNet transfer mechanism. --- var/spack/packages/cbtf/package.py | 65 ++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 var/spack/packages/cbtf/package.py diff --git a/var/spack/packages/cbtf/package.py b/var/spack/packages/cbtf/package.py new file mode 100644 index 00000000000..d2023ada858 --- /dev/null +++ b/var/spack/packages/cbtf/package.py @@ -0,0 +1,65 @@ +################################################################################ +# Copyright (c) 2015 Krell Institute. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA +################################################################################ + +from spack import * + +class Cbtf(Package): + """CBTF project contains the base code for CBTF that supports creating components, + component networks and the support to connect these components and component + networks into sequential and distributed network tools.""" + homepage = "http://sourceforge.net/p/cbtf/wiki/Home" + + url = "http://sourceforge.net/projects/cbtf/files/cbtf-1.5/cbtf-1.5.tar.gz/download" + version('1.5', '75a97e0161d19b3a12305ed1ffb3d3e2') + + # Mirror access template example + #url = "file:/opt/spack-mirror-2015-02-27/cbtf/cbtf-1.5.tar.gz" + #version('1.5', '1ca88a8834759c4c74452cb97fe7b70a') + + # Use when the git repository is available + #version('1.5', branch='master', git='http://git.code.sf.net/p/cbtf/cbtf') + + depends_on("cmake") + depends_on("boost@1.41:") + depends_on("mrnet@4.1.0+krelloptions") + depends_on("xerces-c@3.1.1:") + depends_on("libxml2") + + parallel = False + + def install(self, spec, prefix): + with working_dir('build', create=True): + + + # Boost_NO_SYSTEM_PATHS Set to TRUE to suppress searching + # in system paths (or other locations outside of BOOST_ROOT + # or BOOST_INCLUDEDIR). Useful when specifying BOOST_ROOT. + # Defaults to OFF. + + cmake('..', + '--debug-output', + '-DBoost_NO_SYSTEM_PATHS=TRUE', + '-DXERCESC_DIR=%s' % spec['xerces-c'].prefix, + '-DBOOST_ROOT=%s' % spec['boost'].prefix, + '-DMRNET_DIR=%s' % spec['mrnet'].prefix, + '-DCMAKE_MODULE_PATH=%s' % join_path(prefix.share,'KrellInstitute','cmake'), + *std_cmake_args) + + make("clean") + make() + make("install") From 23971bae92f387cbcbed8f4ea5d99ce81bf03ff0 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Tue, 21 Jul 2015 11:20:27 -0500 Subject: [PATCH 010/910] Add the cbtf-krell spack build package. cbtf-krell is the krell contribution to the CBTF project and includes services, messages, and core libraries and components that support performance information gathering, transfer, and reduction. --- var/spack/packages/cbtf-krell/package.py | 114 +++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 var/spack/packages/cbtf-krell/package.py diff --git a/var/spack/packages/cbtf-krell/package.py b/var/spack/packages/cbtf-krell/package.py new file mode 100644 index 00000000000..0661b101a16 --- /dev/null +++ b/var/spack/packages/cbtf-krell/package.py @@ -0,0 +1,114 @@ +################################################################################ +# Copyright (c) 2015 Krell Institute. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA +################################################################################ + +from spack import * + +class CbtfKrell(Package): + """CBTF Krell project contains the Krell Institute contributions to the CBTF project. + These contributions include many performance data collectors and support + libraries as well as some example tools that drive the data collection at + HPC levels of scale.""" + homepage = "http://sourceforge.net/p/cbtf/wiki/Home/" + + url = "http://sourceforge.net/projects/cbtf/files/cbtf-1.5/cbtf-krell-1.5.tar.gz/download" + version('1.5', '86cca5aa48a65206d3ac5502fcbd3ef6') + + # optional mirror access template + #url = "file:/opt/spack-mirror-2015-02-27/cbtf-krell/cbtf-krell-1.5.tar.gz" + #version('1.5', 'b13f6df6a93c44149d977773dd776d2f') + + + # Dependencies for cbtf-krell + + # For binutils service + depends_on("binutils@2.24+krellpatch") + + # collectionTool + depends_on("boost@1.41:") + depends_on("dyninst@8.2.1:+krelloptions") + depends_on("mrnet@4.1.0:+krelloptions") + depends_on("xerces-c@3.1.1:") + depends_on("cbtf") + + # for services and collectors + depends_on("libmonitor+krellpatch") + depends_on("libunwind") + depends_on("papi") + + # MPI Installations + # These have not worked either for build or execution, commenting out for now + #depends_on("openmpi") + #depends_on("mvapich2@2.0") + #depends_on("mpich") + + parallel = False + + def install(self, spec, prefix): + + # Add in paths for finding package config files that tell us where to find these packages + cmake_prefix_path = join_path(spec['cbtf'].prefix) + ':' + join_path(spec['dyninst'].prefix) + + # FIXME - hard code path until external package support is available + # Need to change this path and/or add additional paths for MPI experiment support on different platforms + openmpi_prefix_path = "/opt/openmpi-1.8.2" + # Other possibilities, they will need a -DMVAPICH_DIR=, etc clause in the cmake command to be recognized + # mvapich_prefix_path = "" + # mvapich2_prefix_path = "" + # mpich2_prefix_path = "" + # mpich_prefix_path = "" + # mpt_prefix_path = "" + + # Add in paths for cuda if requested via the cuda variant + # FIXME - hard code path until external package support is available + if '+cuda' in spec: + cuda_prefix_path = "/usr/local/cuda-6.0" + cupti_prefix_path = "/usr/local/cuda-6.0/extras/CUPTI" + else: + cuda_prefix_path = "" + cupti_prefix_path = "" + + #'-DMVAPICH2_DIR=%s' % spec['mvapich2'].prefix, + #'-DOPENMPI_DIR=%s' % spec['openmpi'].prefix, + #'-DMPICH_DIR=%s' % spec['mpich'].prefix, + #'-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, + #'-DLIB_SUFFIX=64', + + # Build cbtf-krell with cmake + with working_dir('build_cbtf_krell', create=True): + cmake('..', + '-DCMAKE_BUILD_TYPE=Debug', + '-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '-DCBTF_DIR=%s' % spec['cbtf'].prefix, + '-DBINUTILS_DIR=%s' % spec['binutils'].prefix, + '-DLIBMONITOR_DIR=%s' % spec['libmonitor'].prefix, + '-DLIBUNWIND_DIR=%s' % spec['libunwind'].prefix, + '-DPAPI_DIR=%s' % spec['papi'].prefix, + '-DBOOST_DIR=%s' % spec['boost'].prefix, + '-DMRNET_DIR=%s' % spec['mrnet'].prefix, + '-DDYNINST_DIR=%s' % spec['dyninst'].prefix, + '-DXERCESC_DIR=%s' % spec['xerces-c'].prefix, + '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, + '-DCUDA_DIR=%s' % cuda_prefix_path, + '-DCUPTI_DIR=%s' % cupti_prefix_path, + '-DOPENMPI_DIR=%s' % openmpi_prefix_path, + *std_cmake_args) + + make("clean") + make() + make("install") + From e8235b10c610aae51213e8f090e3bf692f99adcc Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Tue, 21 Jul 2015 11:23:23 -0500 Subject: [PATCH 011/910] Add the cbtf-lanl spack build package. cbtf-lanl is LANLs contribution to the CBTF project. It contains psTool and memTool which are example tools, showing use case examples for CBTF. --- var/spack/packages/cbtf-lanl/package.py | 59 +++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 var/spack/packages/cbtf-lanl/package.py diff --git a/var/spack/packages/cbtf-lanl/package.py b/var/spack/packages/cbtf-lanl/package.py new file mode 100644 index 00000000000..b833871df78 --- /dev/null +++ b/var/spack/packages/cbtf-lanl/package.py @@ -0,0 +1,59 @@ +################################################################################ +# Copyright (c) 2015 Krell Institute. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA +################################################################################ + +from spack import * + +class CbtfLanl(Package): + """CBTF LANL project contains a memory tool and data center type system command monitoring tool.""" + homepage = "http://sourceforge.net/p/cbtf/wiki/Home/" + + url = "http://sourceforge.net/projects/cbtf/files/cbtf-1.5/cbtf-lanl-1.5.tar.gz/download" + version('1.5', '78d42050f9ec7127e8d0a93d87a66702') + + # Mirror access template example + #url = "file:/opt/spack-mirror-2015-02-27/cbtf-lanl/cbtf-lanl-1.5.tar.gz" + #version('1.5', 'c3f78f967b0a42c6734ce4be0e602426') + + # Dependencies for cbtf-krell + depends_on("boost@1.41:") + depends_on("mrnet@4.1.0+krelloptions") + depends_on("xerces-c@3.1.1:") + depends_on("cbtf") + depends_on("cbtf-krell") + + parallel = False + + def install(self, spec, prefix): + + # Add in paths for finding package config files that tell us where to find these packages + cmake_prefix_path = join_path(spec['cbtf'].prefix) + ':' + join_path(spec['cbtf-krell'].prefix) + + with working_dir('build', create=True): + cmake('..', + '-DCBTF_DIR=%s' % spec['cbtf'].prefix, + '-DCBTF_KRELL_DIR=%s' % spec['cbtf-krell'].prefix, + '-DMRNET_DIR=%s' % spec['mrnet'].prefix, + '-DXERCESC_DIR=%s' % spec['xerces-c'].prefix, + '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, + '-DCMAKE_MODULE_PATH=%s' % join_path(prefix.share,'KrellInstitute','cmake'), + *std_cmake_args) + + make("clean") + make() + make("install") + From 9fee7be695eb92391b8754593877dded097ea8c8 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Tue, 21 Jul 2015 11:26:59 -0500 Subject: [PATCH 012/910] Add the initial version of the OpenSpeedShop spack build file. OpenSpeedShop is an application performance analysis tool that runs on Linux and gathers a number of different types of performance data and displays that data in a command line tool (CLI) and via a graphical user interface (GUI). --- var/spack/packages/openspeedshop/package.py | 193 ++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 var/spack/packages/openspeedshop/package.py diff --git a/var/spack/packages/openspeedshop/package.py b/var/spack/packages/openspeedshop/package.py new file mode 100644 index 00000000000..94fec4a21cb --- /dev/null +++ b/var/spack/packages/openspeedshop/package.py @@ -0,0 +1,193 @@ +################################################################################ +# Copyright (c) 2015 Krell Institute. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA +################################################################################ + +from spack import * + +class Openspeedshop(Package): + """OpenSpeedShop is a community effort by The Krell Institute with current direct funding from DOEs NNSA. + It builds on top of a broad list of community infrastructures, most notably Dyninst and MRNet from UW, + libmonitor from Rice, and PAPI from UTK. OpenSpeedShop is an open source multi platform Linux performance + tool which is targeted to support performance analysis of applications running on both single node and + large scale IA64, IA32, EM64T, AMD64, PPC, ARM, Blue Gene and Cray platforms. OpenSpeedShop development + is hosted by the Krell Institute. The infrastructure and base components of OpenSpeedShop are released + as open source code primarily under LGPL. + """ + + homepage = "http://www.openspeedshop.org" + url = "http://sourceforge.net/projects/openss/files/openss/openspeedshop-2.1/openspeedshop-2.1.tar.gz/download" + version('2.1', '85ab0f883f16ea23815a670b5ec8e5d6') + + # optional mirror template + #url = "file:/home/jeg/OpenSpeedShop_ROOT/SOURCES/openspeedshop-2.1.tar.gz" + #version('2.1', '85ab0f883f16ea23815a670b5ec8e5d6') + + parallel = False + + variant('offline', default=False, description="build with offline instrumentor enabled.") + variant('cbtf', default=False, description="build with cbtf instrumentor enabled.") + variant('runtime', default=False, description="build only the runtime libraries and collectors.") + variant('frontend', default=False, description="build only the front-end tool using the runtime_dir to point to the target build.") + variant('cuda', default=False, description="build with cuda packages included.") + variant('ptgf', default=False, description="build with the PTGF based gui package enabled.") + variant('intelmic', default=False, description="build for the Intel MIC platform.") + variant('cray', default=False, description="build for Cray platforms.") + variant('bluegene', default=False, description="build for Cray platforms.") + variant('rtfe', default=False, description="build for generic cluster platforms that have different processors on the fe and be nodes.") + + # Dependencies for openspeedshop that are common to all the variants of the OpenSpeedShop build + depends_on("bison") + depends_on("flex") + depends_on("binutils@2.24+krellpatch") + depends_on("libelf") + depends_on("libdwarf") + depends_on("sqlite") + depends_on("boost@1.41:") + depends_on("dyninst@8.2.1:+krelloptions") + depends_on("python") + depends_on("qt@3.3.8b+krellpatch") + + # Dependencies only for the openspeedshop offline package. + depends_on("libunwind", when='+offline') + depends_on("papi", when='+offline') + depends_on("libmonitor+krellpatch", when='+offline') + #depends_on("openmpi+krelloptions", when='+offline') + #depends_on("openmpi", when='+offline') + #depends_on("mpich", when='+offline') + + # Dependencies only for the openspeedshop cbtf package. + depends_on("cbtf", when='+cbtf') + depends_on("cbtf-krell", when='+cbtf') + depends_on("cbtf-argonavis", when='+cbtf') + depends_on("mrnet@4.1.0:+krelloptions", when='+cbtf') + + def install(self, spec, prefix): + + openmpi_prefix_path = "/opt/openmpi-1.8.2" + #'-DOPENMPI_DIR=%s' % spec['openmpi'].prefix, + #'-DMPICH_DIR=%s' % spec['mpich'].prefix, + + # FIXME: How do we make this dynamic in spack? That is, can we specify the paths to cuda dynamically? + # WAITING for external package support. + if '+cuda' in spec: + cuda_prefix_path = "/usr/local/cuda-6.0" + cupti_prefix_path = "/usr/local/cuda-6.0/extras/CUPTI" + + if '+offline' in spec: + instrumentor_setting = "offline" + if '+runtime' in spec: + with working_dir('build_runtime', create=True): + cmake('..', + '-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, + '-DINSTRUMENTOR=%s' % instrumentor_setting, + '-DLIBMONITOR_DIR=%s' % spec['libmonitor'].prefix, + '-DLIBUNWIND_DIR=%s' % spec['libunwind'].prefix, + '-DPAPI_DIR=%s' % spec['papi'].prefix, + '-DOPENMPI_DIR=%s' % openmpi_prefix_path, + *std_cmake_args) + make("clean") + make() + make("install") + else: + cmake_prefix_path = join_path(spec['dyninst'].prefix) + with working_dir('build', create=True): + cmake('..', + '-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, + '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, + '-DINSTRUMENTOR=%s' % instrumentor_setting, + '-DBINUTILS_DIR=%s' % spec['binutils'].prefix, + '-DLIBELF_DIR=%s' % spec['libelf'].prefix, + '-DLIBDWARF_DIR=%s' % spec['libdwarf'].prefix, + '-DLIBMONITOR_DIR=%s' % spec['libmonitor'].prefix, + '-DLIBUNWIND_DIR=%s' % spec['libunwind'].prefix, + '-DPAPI_DIR=%s' % spec['papi'].prefix, + '-DSQLITE3_DIR=%s' % spec['sqlite'].prefix, + '-DQTLIB_DIR=%s' % spec['qt'].prefix, + '-DPYTHON_DIR=%s' % spec['python'].prefix, + '-DBOOST_DIR=%s' % spec['boost'].prefix, + '-DDYNINST_DIR=%s' % spec['dyninst'].prefix, + '-DOPENMPI_DIR=%s' % openmpi_prefix_path, + *std_cmake_args) + make("clean") + make() + make("install") + + elif '+cbtf' in spec: + instrumentor_setting = "cbtf" + cmake_prefix_path = join_path(spec['cbtf'].prefix) + ':' + join_path(spec['cbtf-krell'].prefix) + ':' + join_path(spec['dyninst'].prefix) + if '+runtime' in spec: + with working_dir('build_cbtf_runtime', create=True): + cmake('..', + '-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, + '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, + '-DINSTRUMENTOR=%s' % instrumentor_setting, + '-DBINUTILS_DIR=%s' % spec['binutils'].prefix, + '-DLIBELF_DIR=%s' % spec['libelf'].prefix, + '-DLIBDWARF_DIR=%s' % spec['libdwarf'].prefix, + '-DCBTF_DIR=%s' % spec['cbtf'].prefix, + '-DCBTF_KRELL_DIR=%s' % spec['cbtf-krell'].prefix, + '-DPYTHON_DIR=%s' % spec['python'].prefix, + '-DBOOST_DIR=%s' % spec['boost'].prefix, + '-DDYNINST_DIR=%s' % spec['dyninst'].prefix, + '-DMRNET_DIR=%s' % spec['mrnet'].prefix, + *std_cmake_args) + make("clean") + make() + make("install") + + else: + with working_dir('build_cbtf', create=True): + cmake('..', + '-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, + '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, + '-DINSTRUMENTOR=%s' % instrumentor_setting, + '-DBINUTILS_DIR=%s' % spec['binutils'].prefix, + '-DLIBELF_DIR=%s' % spec['libelf'].prefix, + '-DLIBDWARF_DIR=%s' % spec['libdwarf'].prefix, + '-DSQLITE3_DIR=%s' % spec['sqlite'].prefix, + '-DCBTF_DIR=%s' % spec['cbtf'].prefix, + '-DCBTF_KRELL_DIR=%s' % spec['cbtf-krell'].prefix, + '-DQTLIB_DIR=%s' % spec['qt'].prefix, + '-DPYTHON_DIR=%s' % spec['python'].prefix, + '-DBOOST_DIR=%s' % spec['boost'].prefix, + '-DDYNINST_DIR=%s' % spec['dyninst'].prefix, + '-DMRNET_DIR=%s' % spec['mrnet'].prefix, + *std_cmake_args) + make("clean") + make() + make("install") + + #if '+frontend' in spec: + # with working_dir('build_frontend', create=True): + # tbd + + + #if '+intelmic' in spec: + # with working_dir('build_intelmic_compute', create=True): + # tbd + # with working_dir('build_intelmic_frontend', create=True): + # tbd + + #if '+cray' in spec: + # with working_dir('build_cray_compute', create=True): + # tbd + # with working_dir('build_cray_frontend', create=True): + # tbd From c7b8d09c7f180da5922801450fe0ae6a0f802377 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 20 Apr 2015 10:38:18 -0700 Subject: [PATCH 013/910] Add packagerepos to spack, allowing for creating multiple package repositories. --- lib/spack/spack/__init__.py | 6 ++ lib/spack/spack/build_environment.py | 25 ++++- lib/spack/spack/cmd/create.py | 12 +++ lib/spack/spack/cmd/packagerepo.py | 85 ++++++++++++++++ lib/spack/spack/packages.py | 147 ++++++++++++++++++--------- lib/spack/spack/repo_loader.py | 115 +++++++++++++++++++++ var/spack/packages/reponame | 1 + 7 files changed, 343 insertions(+), 48 deletions(-) create mode 100644 lib/spack/spack/cmd/packagerepo.py create mode 100644 lib/spack/spack/repo_loader.py create mode 100644 var/spack/packages/reponame diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index caa09eb6e00..1d67b453417 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -62,6 +62,12 @@ mock_site_config = join_path(mock_config_path, "site_spackconfig") mock_user_config = join_path(mock_config_path, "user_spackconfig") +# +# Setup the spack.repos namespace +# +from spack.repo_loader import RepoNamespace +repos = RepoNamespace() + # # This controls how spack lays out install prefixes and # stage directories. diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index a133faa6295..03a49302596 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -157,7 +157,7 @@ def set_build_environment_variables(pkg): path_set("PKG_CONFIG_PATH", pkg_config_dirs) -def set_module_variables_for_package(pkg): +def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. """ @@ -228,11 +228,32 @@ def get_rpaths(pkg): return rpaths +def parent_class_modules(cls): + """Get list of super class modules that are all descend from spack.Package""" + if not issubclass(cls, spack.Package) or issubclass(spack.Package, cls): + return [] + result = [] + module = sys.modules.get(cls.__module__) + if module: + result = [ module ] + for c in cls.__bases__: + result.extend(parent_class_modules(c)) + return result + + def setup_package(pkg): """Execute all environment setup routines.""" set_compiler_environment_variables(pkg) set_build_environment_variables(pkg) - set_module_variables_for_package(pkg) + + # If a user makes their own package repo, e.g. + # spack.repos.mystuff.libelf.Libelf, and they inherit from + # an existing class like spack.repos.original.libelf.Libelf, + # then set the module variables for both classes so the + # parent class can still use them if it gets called. + modules = parent_class_modules(pkg.__class__) + for mod in modules: + set_module_variables_for_package(pkg, mod) # Allow dependencies to set up environment as well. for dep_spec in pkg.spec.traverse(root=False): diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 46e6bcec14a..1502942f2c9 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -93,6 +93,9 @@ def setup_parser(subparser): subparser.add_argument( '-n', '--name', dest='alternate_name', default=None, help="Override the autodetected name for the created package.") + subparser.add_argument( + '-p', '--package-repo', dest='package_repo', default=None, + help="Create the package in the specified packagerepo.") subparser.add_argument( '-f', '--force', action='store_true', dest='force', help="Overwrite any existing package file with the same name.") @@ -160,12 +163,21 @@ def create(parser, args): tty.die("Couldn't guess a name for this package. Try running:", "", "spack create --name ") + package_repo = args.package_repo + if not valid_module_name(name): tty.die("Package name can only contain A-Z, a-z, 0-9, '_' and '-'") tty.msg("This looks like a URL for %s version %s." % (name, version)) tty.msg("Creating template for package %s" % name) + # Create a directory for the new package. + pkg_path = spack.db.filename_for_package_name(name, package_repo) + if os.path.exists(pkg_path) and not args.force: + tty.die("%s already exists." % pkg_path) + else: + mkdirp(os.path.dirname(pkg_path)) + versions = spack.package.find_versions_of_archive(url) rkeys = sorted(versions.keys(), reverse=True) versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys))) diff --git a/lib/spack/spack/cmd/packagerepo.py b/lib/spack/spack/cmd/packagerepo.py new file mode 100644 index 00000000000..66bad0ecbf3 --- /dev/null +++ b/lib/spack/spack/cmd/packagerepo.py @@ -0,0 +1,85 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse + +import llnl.util.tty as tty +from llnl.util.tty.color import colorize +from llnl.util.tty.colify import colify +from llnl.util.lang import index_by + +import spack.spec +import spack.config +from spack.util.environment import get_path + +description = "Manage package sources" + +def setup_parser(subparser): + sp = subparser.add_subparsers( + metavar='SUBCOMMAND', dest='packagerepo_command') + + add_parser = sp.add_parser('add', help=packagerepo_add.__doc__) + add_parser.add_argument('directory', help="Directory containing the packages.") + + remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__) + remove_parser.add_argument('name') + + list_parser = sp.add_parser('list', help=packagerepo_list.__doc__) + + +def packagerepo_add(args): + """Add package sources to the Spack configuration.""" + config = spack.config.get_config() + user_config = spack.config.get_config('user') + orig = None + if config.has_value('packagerepo', '', 'directories'): + orig = config.get_value('packagerepo', '', 'directories') + if orig and args.directory in orig.split(':'): + tty.die('Repo directory %s already exists in the repo list' % args.directory) + + newsetting = orig + ':' + args.directory if orig else args.directory + user_config.set_value('packagerepo', '', 'directories', newsetting) + user_config.write() + + +def packagerepo_remove(args): + """Remove a package source from the Spack configuration""" + pass + + +def packagerepo_list(args): + """List package sources and their mnemoics""" + root_names = spack.db.repos + max_len = max(len(s[0]) for s in root_names) + fmt = "%%-%ds%%s" % (max_len + 4) + for root in root_names: + print fmt % (root[0], root[1]) + + + +def packagerepo(parser, args): + action = { 'add' : packagerepo_add, + 'remove' : packagerepo_remove, + 'list' : packagerepo_list } + action[args.packagerepo_command](args) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index adfbc26c1d4..79dbd607037 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -23,10 +23,14 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import exceptions import sys import inspect import glob import imp +import spack.config +import re +from contextlib import closing import llnl.util.tty as tty from llnl.util.filesystem import join_path @@ -36,13 +40,11 @@ import spack.spec from spack.virtual import ProviderIndex from spack.util.naming import mod_to_class, validate_module_name +from sets import Set +from spack.repo_loader import RepoLoader, imported_packages_module, package_file_name -# Name of module under which packages are imported -_imported_packages_module = 'spack.packages' - -# Name of the package file inside a package directory -_package_file_name = 'package.py' - +# Filename for package repo names +_packagerepo_filename = 'reponame' def _autospec(function): """Decorator that automatically converts the argument of a single-arg @@ -55,13 +57,57 @@ def converter(self, spec_like, **kwargs): class PackageDB(object): - def __init__(self, root): + def __init__(self, default_root): """Construct a new package database from a root directory.""" - self.root = root + + #Collect the repos from the config file and read their names from the file system + repo_dirs = self._repo_list_from_config() + repo_dirs.append(default_root) + self.repos = [(self._read_reponame_from_directory(dir), dir) for dir in repo_dirs] + + # Check for duplicate repo names + s = set() + dups = set(r for r in self.repos if r[0] in s or s.add(r[0])) + if dups: + reponame = list(dups)[0][0] + dir1 = list(dups)[0][1] + dir2 = dict(s)[reponame] + tty.die("Package repo %s in directory %s has the same name as the " + "repo in directory %s" % + (reponame, dir1, dir2)) + + # For each repo, create a RepoLoader + self.repo_loaders = dict([(r[0], RepoLoader(r[0], r[1])) for r in self.repos]) + self.instances = {} self.provider_index = None + def _read_reponame_from_directory(self, dir): + """For a packagerepo directory, read the repo name from the dir/reponame file""" + path = os.path.join(dir, 'reponame') + + try: + with closing(open(path, 'r')) as reponame_file: + name = reponame_file.read().lstrip().rstrip() + if not re.match(r'[a-zA-Z][a-zA-Z0-9]+', name): + tty.die("Package repo name '%s', read from %s, is an invalid name. " + "Repo names must began with a letter and only contain letters " + "and numbers." % (name, path)) + return name + except exceptions.IOError, e: + tty.die("Could not read from package repo name file %s" % path) + + + + def _repo_list_from_config(self): + """Read through the spackconfig and return the list of packagerepo directories""" + config = spack.config.get_config() + if not config.has_option('packagerepo', 'directories'): return [] + dir_string = config.get('packagerepo', 'directories') + return dir_string.split(':') + + @_autospec def get(self, spec, **kwargs): if spec.virtual: @@ -130,13 +176,33 @@ def installed_extensions_for(self, extendee_spec): # catching exceptions. - def dirname_for_package_name(self, pkg_name): + def repo_for_package_name(self, pkg_name, packagerepo_name=None): + """Find the dirname for a package and the packagerepo it came from + if packagerepo_name is not None, then search for the package in the + specified packagerepo""" + #Look for an existing package under any matching packagerepos + roots = [pkgrepo for pkgrepo in self.repos + if not packagerepo_name or packagerepo_name == pkgrepo[0]] + + if not roots: + tty.die("Package repo %s does not exist" % packagerepo_name) + + for pkgrepo in roots: + path = join_path(pkgrepo[1], pkg_name) + if os.path.exists(path): + return (pkgrepo[0], path) + + repo_to_add_to = roots[-1] + return (repo_to_add_to[0], join_path(repo_to_add_to[1], pkg_name)) + + + def dirname_for_package_name(self, pkg_name, packagerepo_name=None): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" - return join_path(self.root, pkg_name) + return self.repo_for_package_name(pkg_name, packagerepo_name)[1] - def filename_for_package_name(self, pkg_name): + def filename_for_package_name(self, pkg_name, packagerepo_name=None): """Get the filename for the module we should load for a particular package. Packages for a pacakge DB live in ``$root//package.py`` @@ -144,10 +210,15 @@ def filename_for_package_name(self, pkg_name): This will return a proper package.py path even if the package doesn't exist yet, so callers will need to ensure the package exists before importing. + + If a packagerepo is specified, then return existing + or new paths in the specified packagerepo directory. If no + package repo is supplied, return an existing path from any + package repo, and new paths in the default package repo. """ validate_module_name(pkg_name) - pkg_dir = self.dirname_for_package_name(pkg_name) - return join_path(pkg_dir, _package_file_name) + pkg_dir = self.dirname_for_package_name(pkg_name, packagerepo_name) + return join_path(pkg_dir, package_file_name) def installed_package_specs(self): @@ -176,14 +247,19 @@ def installed_known_package_specs(self): @memoized def all_package_names(self): """Generator function for all packages. This looks for - ``/package.py`` files within the root direcotry""" - all_package_names = [] - for pkg_name in os.listdir(self.root): - pkg_dir = join_path(self.root, pkg_name) - pkg_file = join_path(pkg_dir, _package_file_name) - if os.path.isfile(pkg_file): - all_package_names.append(pkg_name) - all_package_names.sort() + ``/package.py`` files within the repo direcotories""" + all_packages = Set() + for repo in self.repos: + dir = repo[1] + if not os.path.isdir(dir): + continue + for pkg_name in os.listdir(dir): + pkg_dir = join_path(dir, pkg_name) + pkg_file = join_path(pkg_dir, package_file_name) + if os.path.isfile(pkg_file): + all_packages.add(pkg_name) + all_package_names = list(all_packages) + all_package_names.sort() return all_package_names @@ -200,34 +276,13 @@ def exists(self, pkg_name): @memoized def get_class_for_package_name(self, pkg_name): - """Get an instance of the class for a particular package. + """Get an instance of the class for a particular package.""" + repo = self.repo_for_package_name(pkg_name) + module_name = imported_packages_module + '.' + repo[0] + '.' + pkg_name - This method uses Python's ``imp`` package to load python - source from a Spack package's ``package.py`` file. A - normal python import would only load each package once, but - because we do this dynamically, the method needs to be - memoized to ensure there is only ONE package class - instance, per package, per database. - """ - file_path = self.filename_for_package_name(pkg_name) - - if os.path.exists(file_path): - if not os.path.isfile(file_path): - tty.die("Something's wrong. '%s' is not a file!" % file_path) - if not os.access(file_path, os.R_OK): - tty.die("Cannot read '%s'!" % file_path) - else: - raise UnknownPackageError(pkg_name) + module = self.repo_loaders[repo[0]].get_module(pkg_name) class_name = mod_to_class(pkg_name) - try: - module_name = _imported_packages_module + '.' + pkg_name - module = imp.load_source(module_name, file_path) - - except ImportError, e: - tty.die("Error while importing %s from %s:\n%s" % ( - pkg_name, file_path, e.message)) - cls = getattr(module, class_name) if not inspect.isclass(cls): tty.die("%s.%s is not a class" % (pkg_name, class_name)) diff --git a/lib/spack/spack/repo_loader.py b/lib/spack/spack/repo_loader.py new file mode 100644 index 00000000000..57c19a6c281 --- /dev/null +++ b/lib/spack/spack/repo_loader.py @@ -0,0 +1,115 @@ +import spack +import spack.repos +import re +import types +from llnl.util.lang import * + +# Name of module under which packages are imported +imported_packages_module = 'spack.repos' + +# Name of the package file inside a package directory +package_file_name = 'package.py' + +import sys +class LazyLoader: + """The LazyLoader handles cases when repo modules or classes + are imported. It watches for 'spack.repos.*' loads, then + redirects the load to the appropriate module.""" + def find_module(self, fullname, pathname): + if not fullname.startswith(imported_packages_module): + return None + partial_name = fullname[len(imported_packages_module)+1:] + repo = partial_name.split('.')[0] + module = partial_name.split('.')[1] + repo_loader = spack.db.repo_loaders.get(repo) + if repo_loader: + try: + self.mod = repo_loader.get_module(module) + return self + except (ImportError, spack.packages.UnknownPackageError): + return None + + def load_module(self, fullname): + return self.mod + +sys.meta_path.append(LazyLoader()) + +_reponames = {} +class RepoNamespace(types.ModuleType): + """The RepoNamespace holds the repository namespaces under + spack.repos. For example, when accessing spack.repos.original + this class will use __getattr__ to translate the 'original' + into one of spack's known repositories""" + def __init__(self): + import sys + sys.modules[imported_packages_module] = self + + def __getattr__(self, name): + if name in _reponames: + return _reponames[name] + raise AttributeError + + @property + def __file__(self): + return None + + @property + def __path__(self): + return [] + + +class RepoLoader(types.ModuleType): + """Each RepoLoader is associated with a repository, and the RepoLoader is + responsible for loading packages out of that repository. For example, + a RepoLoader may be responsible for spack.repos.original, and when someone + references spack.repos.original.libelf that RepoLoader will load the + libelf package.""" + def __init__(self, reponame, repopath): + self.path = repopath + self.reponame = reponame + self.module_name = imported_packages_module + '.' + reponame + if not reponame in _reponames: + _reponames[reponame] = self + spack.repos.add_repo(reponame, self) + + import sys + sys.modules[self.module_name] = self + + + @property + def __path__(self): + return [ self.path ] + + + def __getattr__(self, name): + if name[0] == '_': + raise AttributeError + return self.get_module(name) + + + @memoized + def get_module(self, pkg_name): + import os + import imp + import llnl.util.tty as tty + + file_path = os.path.join(self.path, pkg_name, package_file_name) + if os.path.exists(file_path): + if not os.path.isfile(file_path): + tty.die("Something's wrong. '%s' is not a file!" % file_path) + if not os.access(file_path, os.R_OK): + tty.die("Cannot read '%s'!" % file_path) + else: + raise spack.packages.UnknownPackageError(pkg_name) + + try: + module_name = imported_packages_module + '.' + self.reponame + '.' + pkg_name + module = imp.load_source(module_name, file_path) + + except ImportError, e: + tty.die("Error while importing %s from %s:\n%s" % ( + pkg_name, file_path, e.message)) + + return module + + diff --git a/var/spack/packages/reponame b/var/spack/packages/reponame new file mode 100644 index 00000000000..4b48deed3a4 --- /dev/null +++ b/var/spack/packages/reponame @@ -0,0 +1 @@ +original From 7ea328659f6bbaff33319427ba7c6321cc33a637 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 20 Apr 2015 13:27:03 -0700 Subject: [PATCH 014/910] Record package repo origins in .spec files --- lib/spack/spack/packages.py | 35 +++++++++++++++++++--------------- lib/spack/spack/repo_loader.py | 2 +- lib/spack/spack/spec.py | 29 ++++++++++++++++++++++++---- 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 79dbd607037..48705948b7b 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -73,12 +73,12 @@ def __init__(self, default_root): dir1 = list(dups)[0][1] dir2 = dict(s)[reponame] tty.die("Package repo %s in directory %s has the same name as the " - "repo in directory %s" % + "repo in directory %s" % (reponame, dir1, dir2)) # For each repo, create a RepoLoader self.repo_loaders = dict([(r[0], RepoLoader(r[0], r[1])) for r in self.repos]) - + self.instances = {} self.provider_index = None @@ -87,13 +87,13 @@ def _read_reponame_from_directory(self, dir): """For a packagerepo directory, read the repo name from the dir/reponame file""" path = os.path.join(dir, 'reponame') - try: + try: with closing(open(path, 'r')) as reponame_file: - name = reponame_file.read().lstrip().rstrip() + name = reponame_file.read().lstrip().rstrip() if not re.match(r'[a-zA-Z][a-zA-Z0-9]+', name): tty.die("Package repo name '%s', read from %s, is an invalid name. " "Repo names must began with a letter and only contain letters " - "and numbers." % (name, path)) + "and numbers." % (name, path)) return name except exceptions.IOError, e: tty.die("Could not read from package repo name file %s" % path) @@ -107,7 +107,7 @@ def _repo_list_from_config(self): dir_string = config.get('packagerepo', 'directories') return dir_string.split(':') - + @_autospec def get(self, spec, **kwargs): if spec.virtual: @@ -118,7 +118,7 @@ def get(self, spec, **kwargs): del self.instances[spec] if not spec in self.instances: - package_class = self.get_class_for_package_name(spec.name) + package_class = self.get_class_for_package_name(spec.name, spec.repo) try: copy = spec.copy() self.instances[copy] = package_class(copy) @@ -191,7 +191,7 @@ def repo_for_package_name(self, pkg_name, packagerepo_name=None): path = join_path(pkgrepo[1], pkg_name) if os.path.exists(path): return (pkgrepo[0], path) - + repo_to_add_to = roots[-1] return (repo_to_add_to[0], join_path(repo_to_add_to[1], pkg_name)) @@ -259,7 +259,7 @@ def all_package_names(self): if os.path.isfile(pkg_file): all_packages.add(pkg_name) all_package_names = list(all_packages) - all_package_names.sort() + all_package_names.sort() return all_package_names @@ -275,12 +275,12 @@ def exists(self, pkg_name): @memoized - def get_class_for_package_name(self, pkg_name): + def get_class_for_package_name(self, pkg_name, reponame = None): """Get an instance of the class for a particular package.""" - repo = self.repo_for_package_name(pkg_name) - module_name = imported_packages_module + '.' + repo[0] + '.' + pkg_name + (reponame, repodir) = self.repo_for_package_name(pkg_name, reponame) + module_name = imported_packages_module + '.' + reponame + '.' + pkg_name - module = self.repo_loaders[repo[0]].get_module(pkg_name) + module = self.repo_loaders[reponame].get_module(pkg_name) class_name = mod_to_class(pkg_name) cls = getattr(module, class_name) @@ -292,8 +292,13 @@ def get_class_for_package_name(self, pkg_name): class UnknownPackageError(spack.error.SpackError): """Raised when we encounter a package spack doesn't have.""" - def __init__(self, name): - super(UnknownPackageError, self).__init__("Package '%s' not found." % name) + def __init__(self, name, repo=None): + msg = None + if repo: + msg = "Package %s not found in packagerepo %s." % (name, repo) + else: + msg = "Package %s not found." % name + super(UnknownPackageError, self).__init__(msg) self.name = name diff --git a/lib/spack/spack/repo_loader.py b/lib/spack/spack/repo_loader.py index 57c19a6c281..6eaa1eead20 100644 --- a/lib/spack/spack/repo_loader.py +++ b/lib/spack/spack/repo_loader.py @@ -100,7 +100,7 @@ def get_module(self, pkg_name): if not os.access(file_path, os.R_OK): tty.die("Cannot read '%s'!" % file_path) else: - raise spack.packages.UnknownPackageError(pkg_name) + raise spack.packages.UnknownPackageError(pkg_name, self.reponame if self.reponame != 'original' else None) try: module_name = imported_packages_module + '.' + self.reponame + '.' + pkg_name diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index e1fbb844234..972ba9ccbbd 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -112,6 +112,7 @@ from spack.util.string import * from spack.util.prefix import Prefix from spack.virtual import ProviderIndex +from spack.repo_loader import imported_packages_module # Valid pattern for an identifier in Spack identifier_re = r'\w[\w-]*' @@ -412,6 +413,7 @@ def __init__(self, spec_like, *dep_like, **kwargs): self.dependencies = other.dependencies self.variants = other.variants self.variants.spec = self + self.repo = other.repo # Specs are by default not assumed to be normal, but in some # cases we've read them from a file want to assume normal. @@ -1355,6 +1357,7 @@ def _dup(self, other, **kwargs): self.dependencies = DependencyMap() self.variants = other.variants.copy() self.variants.spec = self + self.repo = other.repo # If we copy dependencies, preserve DAG structure in the new spec if kwargs.get('deps', True): @@ -1503,6 +1506,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): in the format string. The format strings you can provide are:: $_ Package name + $. Long package name $@ Version $% Compiler $%@ Compiler & compiler version @@ -1550,6 +1554,9 @@ def write(s, c): if c == '_': out.write(fmt % self.name) + elif c == '.': + longname = '%s.%s.%s' % (imported_packages_module, self.repo, self.name) if self.repo else self.name + out.write(fmt % longname) elif c == '@': if self.versions and self.versions != _any_version: write(fmt % (c + str(self.versions)), c) @@ -1698,17 +1705,29 @@ def parse_compiler(self, text): def spec(self): """Parse a spec out of the input. If a spec is supplied, then initialize and return it instead of creating a new one.""" - self.check_identifier() + + spec_name = None + spec_repo = None + if self.token.value.startswith(imported_packages_module): + lst = self.token.value.split('.') + spec_name = lst[-1] + spec_repo = lst[-2] + else: + spec_name = self.token.value + (spec_repo, repodir) = spack.db.repo_for_package_name(spec_name) + + self.check_identifier(spec_name) # This will init the spec without calling __init__. spec = Spec.__new__(Spec) - spec.name = self.token.value + spec.name = spec_name spec.versions = VersionList() spec.variants = VariantMap(spec) spec.architecture = None spec.compiler = None spec.dependents = DependencyMap() spec.dependencies = DependencyMap() + spec.repo = spec_repo spec._normal = False spec._concrete = False @@ -1802,12 +1821,14 @@ def compiler(self): return compiler - def check_identifier(self): + def check_identifier(self, id=None): """The only identifiers that can contain '.' are versions, but version ids are context-sensitive so we have to check on a case-by-case basis. Call this if we detect a version id where it shouldn't be. """ - if '.' in self.token.value: + if not id: + id = self.token.value + if '.' in id: self.last_token_error("Identifier cannot contain '.'") From e58ee88a632a0855a984cba8f0faa725b8f2eddf Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 20 Apr 2015 13:50:09 -0700 Subject: [PATCH 015/910] Add 'spack packagerepo create' command --- lib/spack/spack/cmd/packagerepo.py | 51 ++++++++++++++++++++++++++---- lib/spack/spack/packages.py | 4 +-- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/cmd/packagerepo.py b/lib/spack/spack/cmd/packagerepo.py index 66bad0ecbf3..2819d0f9806 100644 --- a/lib/spack/spack/cmd/packagerepo.py +++ b/lib/spack/spack/cmd/packagerepo.py @@ -23,15 +23,20 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from external import argparse - import llnl.util.tty as tty from llnl.util.tty.color import colorize from llnl.util.tty.colify import colify from llnl.util.lang import index_by +from llnl.util.filesystem import join_path, mkdirp import spack.spec import spack.config from spack.util.environment import get_path +from spack.packages import packagerepo_filename + +import os +import exceptions +from contextlib import closing description = "Manage package sources" @@ -41,6 +46,10 @@ def setup_parser(subparser): add_parser = sp.add_parser('add', help=packagerepo_add.__doc__) add_parser.add_argument('directory', help="Directory containing the packages.") + + create_parser = sp.add_parser('create', help=packagerepo_create.__doc__) + create_parser.add_argument('directory', help="Directory containing the packages.") + create_parser.add_argument('name', help="Name of new package repository.") remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__) remove_parser.add_argument('name') @@ -48,19 +57,48 @@ def setup_parser(subparser): list_parser = sp.add_parser('list', help=packagerepo_list.__doc__) -def packagerepo_add(args): - """Add package sources to the Spack configuration.""" +def add_to_config(dir): config = spack.config.get_config() user_config = spack.config.get_config('user') orig = None if config.has_value('packagerepo', '', 'directories'): orig = config.get_value('packagerepo', '', 'directories') - if orig and args.directory in orig.split(':'): - tty.die('Repo directory %s already exists in the repo list' % args.directory) + if orig and dir in orig.split(':'): + return False - newsetting = orig + ':' + args.directory if orig else args.directory + newsetting = orig + ':' + dir if orig else dir user_config.set_value('packagerepo', '', 'directories', newsetting) user_config.write() + return True + + +def packagerepo_add(args): + """Add package sources to the Spack configuration.""" + if not add_to_config(args.directory): + tty.die('Repo directory %s already exists in the repo list' % dir) + + +def packagerepo_create(args): + """Create a new package repo at a directory and name""" + dir = args.directory + name = args.name + + if os.path.exists(dir) and not os.path.isdir(dir): + tty.die('File %s already exists and is not a directory' % dir) + if not os.path.exists(dir): + try: + mkdirp(dir) + except exceptions.OSError, e: + tty.die('Failed to create new directory %s' % dir) + path = os.path.join(dir, packagerepo_filename) + try: + with closing(open(path, 'w')) as repofile: + repofile.write(name + '\n') + except exceptions.IOError, e: + tty.die('Could not create new file %s' % path) + + if not add_to_config(args.directory): + tty.warn('Repo directory %s already exists in the repo list' % dir) def packagerepo_remove(args): @@ -80,6 +118,7 @@ def packagerepo_list(args): def packagerepo(parser, args): action = { 'add' : packagerepo_add, + 'create' : packagerepo_create, 'remove' : packagerepo_remove, 'list' : packagerepo_list } action[args.packagerepo_command](args) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 48705948b7b..3b9d74dd6e9 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -44,7 +44,7 @@ from spack.repo_loader import RepoLoader, imported_packages_module, package_file_name # Filename for package repo names -_packagerepo_filename = 'reponame' +packagerepo_filename = 'reponame' def _autospec(function): """Decorator that automatically converts the argument of a single-arg @@ -85,7 +85,7 @@ def __init__(self, default_root): def _read_reponame_from_directory(self, dir): """For a packagerepo directory, read the repo name from the dir/reponame file""" - path = os.path.join(dir, 'reponame') + path = os.path.join(dir, packagerepo_filename) try: with closing(open(path, 'r')) as reponame_file: From da98b07624e2403807166e8a9d0dac3752f75c0f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 6 Jul 2015 19:57:31 -0700 Subject: [PATCH 016/910] Add more options to `spack edit` --- lib/spack/spack/cmd/edit.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index b8764ba391e..9081d12516d 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -78,9 +78,18 @@ def setup_parser(subparser): subparser.add_argument( '-f', '--force', dest='force', action='store_true', help="Open a new file in $EDITOR even if package doesn't exist.") - subparser.add_argument( - '-c', '--command', dest='edit_command', action='store_true', - help="Edit the command with the supplied name instead of a package.") + + filetypes = subparser.add_mutually_exclusive_group() + filetypes.add_argument( + '-c', '--command', dest='path', action='store_const', + const=spack.cmd.command_path, help="Edit the command with the supplied name.") + filetypes.add_argument( + '-t', '--test', dest='path', action='store_const', + const=spack.test_path, help="Edit the test with the supplied name.") + filetypes.add_argument( + '-m', '--module', dest='path', action='store_const', + const=spack.module_path, help="Edit the main spack module with the supplied name.") + subparser.add_argument( 'name', nargs='?', default=None, help="name of package to edit") @@ -88,19 +97,17 @@ def setup_parser(subparser): def edit(parser, args): name = args.name - if args.edit_command: - if not name: - path = spack.cmd.command_path - else: - path = join_path(spack.cmd.command_path, name + ".py") - if not os.path.exists(path): + path = spack.packages_path + if args.path: + path = args.path + if name: + path = join_path(path, name + ".py") + if not args.force and not os.path.exists(path): tty.die("No command named '%s'." % name) spack.editor(path) + elif name: + edit_package(name, args.force) else: # By default open the directory where packages or commands live. - if not name: - path = spack.packages_path - spack.editor(path) - else: - edit_package(name, args.force) + spack.editor(path) From 92f398a897949e3b586abc38a934ae9d5cf1163c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 7 Jul 2015 00:25:20 -0700 Subject: [PATCH 017/910] Better `@memoized` decorator. --- lib/spack/llnl/util/lang.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 9e1bef18ca0..be6dad867ed 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -26,6 +26,7 @@ import re import sys import functools +import collections import inspect # Ignore emacs backups when listing modules @@ -170,16 +171,32 @@ def has_method(cls, name): return False -def memoized(obj): +class memoized(object): """Decorator that caches the results of a function, storing them in an attribute of that function.""" - cache = obj.cache = {} - @functools.wraps(obj) - def memoizer(*args, **kwargs): - if args not in cache: - cache[args] = obj(*args, **kwargs) - return cache[args] - return memoizer + def __init__(self, func): + self.func = func + self.cache = {} + + + def __call__(self, *args): + if not isinstance(args, collections.Hashable): + # Not hashable, so just call the function. + return self.func(*args) + + if args not in self.cache: + self.cache[args] = self.func(*args) + return self.cache[args] + + + def __get__(self, obj, objtype): + """Support instance methods.""" + return functools.partial(self.__call__, obj) + + + def clear(self): + """Expunge cache so that self.func will be called again.""" + self.cache.clear() def list_modules(directory, **kwargs): From a2f2e6a4ff60882d8a93754ee10e8e75245cf430 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 19 Jul 2015 21:55:27 -0700 Subject: [PATCH 018/910] Save changes to external repo integration --- lib/spack/spack/__init__.py | 16 +- .../spack/cmd/{packagerepo.py => repo.py} | 43 ++-- lib/spack/spack/config.py | 206 +++++++++++------- lib/spack/spack/packages.py | 113 ++++++---- lib/spack/spack/repo_loader.py | 35 ++- lib/spack/spack/test/config.py | 84 +++++-- var/spack/mock_packages/repo.yaml | 2 + var/spack/packages/repo.yaml | 2 + var/spack/packages/reponame | 1 - 9 files changed, 324 insertions(+), 178 deletions(-) rename lib/spack/spack/cmd/{packagerepo.py => repo.py} (78%) create mode 100644 var/spack/mock_packages/repo.yaml create mode 100644 var/spack/packages/repo.yaml delete mode 100644 var/spack/packages/reponame diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 1d67b453417..09bc9ca52a9 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -47,11 +47,17 @@ share_path = join_path(prefix, "share", "spack") # -# Set up the packages database. +# Setup the spack.repos namespace +# +from spack.repo_loader import RepoNamespace +repos = RepoNamespace() + +# +# Set up the default packages database. # from spack.packages import PackageDB packages_path = join_path(var_path, "packages") -db = PackageDB(packages_path) +db = PackageDB() # # Paths to mock files for testing. @@ -62,12 +68,6 @@ mock_site_config = join_path(mock_config_path, "site_spackconfig") mock_user_config = join_path(mock_config_path, "user_spackconfig") -# -# Setup the spack.repos namespace -# -from spack.repo_loader import RepoNamespace -repos = RepoNamespace() - # # This controls how spack lays out install prefixes and # stage directories. diff --git a/lib/spack/spack/cmd/packagerepo.py b/lib/spack/spack/cmd/repo.py similarity index 78% rename from lib/spack/spack/cmd/packagerepo.py rename to lib/spack/spack/cmd/repo.py index 2819d0f9806..1261c7ada9d 100644 --- a/lib/spack/spack/cmd/packagerepo.py +++ b/lib/spack/spack/cmd/repo.py @@ -32,7 +32,7 @@ import spack.spec import spack.config from spack.util.environment import get_path -from spack.packages import packagerepo_filename +from spack.packages import repo_config import os import exceptions @@ -42,43 +42,48 @@ def setup_parser(subparser): sp = subparser.add_subparsers( - metavar='SUBCOMMAND', dest='packagerepo_command') + metavar='SUBCOMMAND', dest='repo_command') - add_parser = sp.add_parser('add', help=packagerepo_add.__doc__) + add_parser = sp.add_parser('add', help=repo_add.__doc__) add_parser.add_argument('directory', help="Directory containing the packages.") - create_parser = sp.add_parser('create', help=packagerepo_create.__doc__) + create_parser = sp.add_parser('create', help=repo_create.__doc__) create_parser.add_argument('directory', help="Directory containing the packages.") create_parser.add_argument('name', help="Name of new package repository.") +<<<<<<< HEAD:lib/spack/spack/cmd/packagerepo.py remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__) +======= + + remove_parser = sp.add_parser('remove', help=repo_remove.__doc__) +>>>>>>> Save changes to external repo integration:lib/spack/spack/cmd/repo.py remove_parser.add_argument('name') - list_parser = sp.add_parser('list', help=packagerepo_list.__doc__) + list_parser = sp.add_parser('list', help=repo_list.__doc__) def add_to_config(dir): config = spack.config.get_config() user_config = spack.config.get_config('user') orig = None - if config.has_value('packagerepo', '', 'directories'): - orig = config.get_value('packagerepo', '', 'directories') + if config.has_value('repo', '', 'directories'): + orig = config.get_value('repo', '', 'directories') if orig and dir in orig.split(':'): return False newsetting = orig + ':' + dir if orig else dir - user_config.set_value('packagerepo', '', 'directories', newsetting) + user_config.set_value('repo', '', 'directories', newsetting) user_config.write() return True -def packagerepo_add(args): +def repo_add(args): """Add package sources to the Spack configuration.""" if not add_to_config(args.directory): tty.die('Repo directory %s already exists in the repo list' % dir) -def packagerepo_create(args): +def repo_create(args): """Create a new package repo at a directory and name""" dir = args.directory name = args.name @@ -90,7 +95,7 @@ def packagerepo_create(args): mkdirp(dir) except exceptions.OSError, e: tty.die('Failed to create new directory %s' % dir) - path = os.path.join(dir, packagerepo_filename) + path = os.path.join(dir, repo_config) try: with closing(open(path, 'w')) as repofile: repofile.write(name + '\n') @@ -101,12 +106,12 @@ def packagerepo_create(args): tty.warn('Repo directory %s already exists in the repo list' % dir) -def packagerepo_remove(args): +def repo_remove(args): """Remove a package source from the Spack configuration""" pass -def packagerepo_list(args): +def repo_list(args): """List package sources and their mnemoics""" root_names = spack.db.repos max_len = max(len(s[0]) for s in root_names) @@ -116,9 +121,9 @@ def packagerepo_list(args): -def packagerepo(parser, args): - action = { 'add' : packagerepo_add, - 'create' : packagerepo_create, - 'remove' : packagerepo_remove, - 'list' : packagerepo_list } - action[args.packagerepo_command](args) +def repo(parser, args): + action = { 'add' : repo_add, + 'create' : repo_create, + 'remove' : repo_remove, + 'list' : repo_list } + action[args.repo_command](args) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3e91958c2c2..dc59f9a5a32 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -45,11 +45,11 @@ Configuration file format =============================== -Configuration files are formatted using YAML syntax. -This format is implemented by Python's -yaml class, and it's easy to read and versatile. +Configuration files are formatted using YAML syntax. This format is +implemented by libyaml (included with Spack as an external module), +and it's easy to read and versatile. -The config files are structured as trees, like this ``compiler`` section:: +Config files are structured as trees, like this ``compiler`` section:: compilers: chaos_5_x86_64_ib: @@ -83,62 +83,73 @@ } } -Some routines, like get_mirrors_config and get_compilers_config may strip -off the top-levels of the tree and return subtrees. +Some convenience functions, like get_mirrors_config and +``get_compilers_config`` may strip off the top-levels of the tree and +return subtrees. + """ import os -import exceptions import sys - -from external.ordereddict import OrderedDict -from llnl.util.lang import memoized -import spack.error - +import copy from external import yaml from external.yaml.error import MarkedYAMLError + import llnl.util.tty as tty from llnl.util.filesystem import mkdirp +from llnl.util.lang import memoized + +import spack + _config_sections = {} class _ConfigCategory: name = None filename = None merge = True - def __init__(self, n, f, m): - self.name = n - self.filename = f - self.merge = m + def __init__(self, name, filename, merge, strip): + self.name = name + self.filename = filename + self.merge = merge + self.strip = strip self.files_read_from = [] self.result_dict = {} - _config_sections[n] = self + _config_sections[name] = self -_ConfigCategory('compilers', 'compilers.yaml', True) -_ConfigCategory('mirrors', 'mirrors.yaml', True) -_ConfigCategory('view', 'views.yaml', True) -_ConfigCategory('order', 'orders.yaml', True) +_ConfigCategory('config', 'config.yaml', True, False) +_ConfigCategory('compilers', 'compilers.yaml', True, True) +_ConfigCategory('mirrors', 'mirrors.yaml', True, True) +_ConfigCategory('view', 'views.yaml', True, True) +_ConfigCategory('order', 'orders.yaml', True, True) """Names of scopes and their corresponding configuration files.""" config_scopes = [('site', os.path.join(spack.etc_path, 'spack')), ('user', os.path.expanduser('~/.spack'))] _compiler_by_arch = {} -_read_config_file_result = {} + +@memoized def _read_config_file(filename): - """Read a given YAML configuration file""" - global _read_config_file_result - if filename in _read_config_file_result: - return _read_config_file_result[filename] + """Read a YAML configuration file""" + + # Ignore nonexisting files. + if not os.path.exists(filename): + return None + + elif not os.path.isfile(filename): + tty.die("Invlaid configuration. %s exists but is not a file." % filename) + + elif not os.access(filename, os.R_OK): + tty.die("Configuration file %s is not readable." % filename) try: with open(filename) as f: - ydict = yaml.load(f) + return yaml.load(f) + except MarkedYAMLError, e: tty.die("Error parsing yaml%s: %s" % (str(e.context_mark), e.problem)) - except exceptions.IOError, e: - _read_config_file_result[filename] = None - return None - _read_config_file_result[filename] = ydict - return ydict + + except IOError, e: + tty.die("Error reading configuration file %s: %s" % (filename, str(e))) def clear_config_caches(): @@ -147,41 +158,66 @@ def clear_config_caches(): for key,s in _config_sections.iteritems(): s.files_read_from = [] s.result_dict = {} - spack.config._read_config_file_result = {} + + _read_config_file.clear() spack.config._compiler_by_arch = {} spack.compilers._cached_default_compiler = None -def _merge_dicts(d1, d2): - """Recursively merges two configuration trees, with entries - in d2 taking precedence over d1""" - if not d1: - return d2.copy() - if not d2: - return d1 +def _merge_yaml(dest, source): + """Merges source into dest; entries in source take precedence over dest. - for key2, val2 in d2.iteritems(): - if not key2 in d1: - d1[key2] = val2 - continue - val1 = d1[key2] - if isinstance(val1, dict) and isinstance(val2, dict): - d1[key2] = _merge_dicts(val1, val2) - continue - if isinstance(val1, list) and isinstance(val2, list): - val1.extend(val2) - seen = set() - d1[key2] = [ x for x in val1 if not (x in seen or seen.add(x)) ] - continue - d1[key2] = val2 - return d1 + Config file authors can optionally end any attribute in a dict + with `::` instead of `:`, and the key will override that of the + parent instead of merging. + """ + def they_are(t): + return isinstance(dest, t) and isinstance(source, t) + + # If both are None, handle specially and return None. + if source is None and dest is None: + return None + + # If source is None, overwrite with source. + elif source is None: + return None + + # Source list is prepended (for precedence) + if they_are(list): + seen = set(source) + dest[:] = source + [x for x in dest if x not in seen] + return dest + + # Source dict is merged into dest. Extra ':' means overwrite. + elif they_are(dict): + for sk, sv in source.iteritems(): + # allow total override with, e.g., repos:: + override = sk.endswith(':') + if override: + sk = sk.rstrip(':') + + if override or not sk in dest: + dest[sk] = copy.copy(sv) + else: + dest[sk] = _merge_yaml(dest[sk], source[sk]) + return dest + + # In any other case, overwrite with a copy of the source value. + else: + return copy.copy(source) -def get_config(category_name): - """Get the confguration tree for the names category. Strips off the - top-level category entry from the dict""" - global config_scopes - category = _config_sections[category_name] +def substitute_spack_prefix(path): + """Replaces instances of $spack with Spack's prefix.""" + return path.replace('$spack', spack.prefix) + + +def get_config(category='config'): + """Get the confguration tree for a category. + + Strips off the top-level category entry from the dict + """ + category = _config_sections[category] if category.result_dict: return category.result_dict @@ -191,14 +227,18 @@ def get_config(category_name): result = _read_config_file(path) if not result: continue - if not category_name in result: - continue + + if category.strip: + if not category.name in result: + continue + result = result[category.name] + category.files_read_from.insert(0, path) - result = result[category_name] if category.merge: - category.result_dict = _merge_dicts(category.result_dict, result) + category.result_dict = _merge_yaml(category.result_dict, result) else: category.result_dict = result + return category.result_dict @@ -215,7 +255,7 @@ def get_compilers_config(arch=None): cc_config = get_config('compilers') if arch in cc_config and 'all' in cc_config: arch_compiler = dict(cc_config[arch]) - _compiler_by_arch[arch] = _merge_dict(arch_compiler, cc_config['all']) + _compiler_by_arch[arch] = _merge_yaml(arch_compiler, cc_config['all']) elif arch in cc_config: _compiler_by_arch[arch] = cc_config[arch] elif 'all' in cc_config: @@ -225,6 +265,13 @@ def get_compilers_config(arch=None): return _compiler_by_arch[arch] +def get_repos_config(): + config = get_config() + if 'repos' not in config: + return [] + return config['repos'] + + def get_mirror_config(): """Get the mirror configuration from config files""" return get_config('mirrors') @@ -232,7 +279,6 @@ def get_mirror_config(): def get_config_scope_dirname(scope): """For a scope return the config directory""" - global config_scopes for s,p in config_scopes: if s == scope: return p @@ -251,16 +297,16 @@ def get_config_scope_filename(scope, category_name): def add_to_config(category_name, addition_dict, scope=None): """Merge a new dict into a configuration tree and write the new configuration to disk""" - global _read_config_file_result get_config(category_name) category = _config_sections[category_name] - #If scope is specified, use it. Otherwise use the last config scope that - #we successfully parsed data from. + # If scope is specified, use it. Otherwise use the last config scope that + # we successfully parsed data from. file = None path = None if not scope and not category.files_read_from: scope = 'user' + if scope: try: dir = get_config_scope_dirname(scope) @@ -268,32 +314,37 @@ def add_to_config(category_name, addition_dict, scope=None): mkdirp(dir) path = os.path.join(dir, category.filename) file = open(path, 'w') - except exceptions.IOError, e: + except IOError, e: pass else: for p in category.files_read_from: try: file = open(p, 'w') - except exceptions.IOError, e: + except IOError, e: pass if file: path = p break; + if not file: tty.die('Unable to write to config file %s' % path) - #Merge the new information into the existing file info, then write to disk - new_dict = _read_config_file_result[path] + # Merge the new information into the existing file info, then write to disk + new_dict = _read_config_file(path) + if new_dict and category_name in new_dict: new_dict = new_dict[category_name] - new_dict = _merge_dicts(new_dict, addition_dict) + + new_dict = _merge_yaml(new_dict, addition_dict) new_dict = { category_name : new_dict } - _read_config_file_result[path] = new_dict + + # Install new dict as memoized value, and dump to disk + _read_config_file.cache[path] = new_dict yaml.dump(new_dict, stream=file, default_flow_style=False) file.close() - #Merge the new information into the cached results - category.result_dict = _merge_dicts(category.result_dict, addition_dict) + # Merge the new information into the cached results + category.result_dict = _merge_yaml(category.result_dict, addition_dict) def add_to_mirror_config(addition_dict, scope=None): @@ -311,7 +362,6 @@ def add_to_compiler_config(addition_dict, scope=None, arch=None): def remove_from_config(category_name, key_to_rm, scope=None): """Remove a configuration key and write a new configuration to disk""" - global config_scopes get_config(category_name) scopes_to_rm_from = [scope] if scope else [s for s,p in config_scopes] category = _config_sections[category_name] diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 3b9d74dd6e9..c4142343860 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -30,7 +30,9 @@ import imp import spack.config import re -from contextlib import closing +import itertools +import traceback +from external import yaml import llnl.util.tty as tty from llnl.util.filesystem import join_path @@ -44,7 +46,7 @@ from spack.repo_loader import RepoLoader, imported_packages_module, package_file_name # Filename for package repo names -packagerepo_filename = 'reponame' +repo_config = 'repo.yaml' def _autospec(function): """Decorator that automatically converts the argument of a single-arg @@ -56,56 +58,85 @@ def converter(self, spec_like, **kwargs): return converter +def sliding_window(seq, n): + it = iter(seq) + result = tuple(itertools.islice(it, n)) + if len(result) == n: + yield result + for elem in it: + result = result[1:] + (elem,) + yield result + + class PackageDB(object): - def __init__(self, default_root): - """Construct a new package database from a root directory.""" + def __init__(self, *repo_dirs): + """Construct a new package database from a list of directories. - #Collect the repos from the config file and read their names from the file system - repo_dirs = self._repo_list_from_config() - repo_dirs.append(default_root) - self.repos = [(self._read_reponame_from_directory(dir), dir) for dir in repo_dirs] + Args: + repo_dirs List of directories containing packages. - # Check for duplicate repo names - s = set() - dups = set(r for r in self.repos if r[0] in s or s.add(r[0])) - if dups: - reponame = list(dups)[0][0] - dir1 = list(dups)[0][1] - dir2 = dict(s)[reponame] - tty.die("Package repo %s in directory %s has the same name as the " - "repo in directory %s" % - (reponame, dir1, dir2)) + If ``repo_dirs`` is empty, gets repository list from Spack configuration. + """ + if not repo_dirs: + repo_dirs = spack.config.get_repos_config() + if not repo_dirs: + tty.die("Spack configuration contains no package repositories.") + + # Collect the repos from the config file and read their names + # from the file system + repo_dirs = [spack.config.substitute_spack_prefix(rd) for rd in repo_dirs] + + self.repos = [] + for rdir in repo_dirs: + rname = self._read_reponame_from_directory(rdir) + if rname: + self.repos.append((self._read_reponame_from_directory(rdir), rdir)) + + + by_path = sorted(self.repos, key=lambda r:r[1]) + by_name = sorted(self.repos, key=lambda r:r[0]) + + for r1, r2 in by_path: + if r1[1] == r2[1]: + tty.die("Package repos are the same:", + " %20s %s" % r1, " %20s %s" % r2) + + for r1, r2 in by_name: + if r1[0] == r2[0]: + tty.die("Package repos cannot have the same name:", + " %20s %s" % r1, " %20s %s" % r2) # For each repo, create a RepoLoader - self.repo_loaders = dict([(r[0], RepoLoader(r[0], r[1])) for r in self.repos]) + self.repo_loaders = dict((name, RepoLoader(name, path)) + for name, path in self.repos) self.instances = {} self.provider_index = None def _read_reponame_from_directory(self, dir): - """For a packagerepo directory, read the repo name from the dir/reponame file""" - path = os.path.join(dir, packagerepo_filename) + """For a packagerepo directory, read the repo name from the + $root/repo.yaml file""" + path = os.path.join(dir, repo_config) try: - with closing(open(path, 'r')) as reponame_file: - name = reponame_file.read().lstrip().rstrip() - if not re.match(r'[a-zA-Z][a-zA-Z0-9]+', name): - tty.die("Package repo name '%s', read from %s, is an invalid name. " - "Repo names must began with a letter and only contain letters " - "and numbers." % (name, path)) + with open(path) as reponame_file: + yaml_data = yaml.load(reponame_file) + + if (not yaml_data or + 'repo' not in yaml_data or + 'namespace' not in yaml_data['repo']): + tty.die("Invalid %s in %s" % (repo_config, dir)) + + name = yaml_data['repo']['namespace'] + if not re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', name): + tty.die( + "Package repo name '%s', read from %s, is an invalid name. " + "Repo names must began with a letter and only contain " + "letters and numbers." % (name, path)) return name except exceptions.IOError, e: - tty.die("Could not read from package repo name file %s" % path) - - - - def _repo_list_from_config(self): - """Read through the spackconfig and return the list of packagerepo directories""" - config = spack.config.get_config() - if not config.has_option('packagerepo', 'directories'): return [] - dir_string = config.get('packagerepo', 'directories') - return dir_string.split(':') + tty.die("Error reading %s when opening %s" % (repo_config, dir)) @_autospec @@ -125,7 +156,7 @@ def get(self, spec, **kwargs): except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) - raise FailedConstructorError(spec.name, e) + raise FailedConstructorError(spec.name, *sys.exc_info()) return self.instances[spec] @@ -304,8 +335,10 @@ def __init__(self, name, repo=None): class FailedConstructorError(spack.error.SpackError): """Raised when a package's class constructor fails.""" - def __init__(self, name, reason): + def __init__(self, name, exc_type, exc_obj, exc_tb): super(FailedConstructorError, self).__init__( "Class constructor failed for package '%s'." % name, - str(reason)) + '\nCaused by:\n' + + ('%s: %s\n' % (exc_type.__name__, exc_obj)) + + ''.join(traceback.format_tb(exc_tb))) self.name = name diff --git a/lib/spack/spack/repo_loader.py b/lib/spack/spack/repo_loader.py index 6eaa1eead20..92da1cf7098 100644 --- a/lib/spack/spack/repo_loader.py +++ b/lib/spack/spack/repo_loader.py @@ -1,8 +1,10 @@ -import spack -import spack.repos import re +import sys import types +import traceback + from llnl.util.lang import * +import spack # Name of module under which packages are imported imported_packages_module = 'spack.repos' @@ -13,14 +15,30 @@ import sys class LazyLoader: """The LazyLoader handles cases when repo modules or classes - are imported. It watches for 'spack.repos.*' loads, then + are imported. It watches for 'spack.repos.*' loads, then redirects the load to the appropriate module.""" def find_module(self, fullname, pathname): if not fullname.startswith(imported_packages_module): return None + + print "HERE ===" + print + for line in traceback.format_stack(): + print " ", line.strip() + print + print "full: ", fullname + print "path: ", pathname + print + partial_name = fullname[len(imported_packages_module)+1:] - repo = partial_name.split('.')[0] - module = partial_name.split('.')[1] + + print "partial: ", partial_name + print + + last_dot = partial_name.rfind('.') + repo = partial_name[:last_dot] + module = partial_name[last_dot+1:] + repo_loader = spack.db.repo_loaders.get(repo) if repo_loader: try: @@ -43,7 +61,7 @@ class RepoNamespace(types.ModuleType): def __init__(self): import sys sys.modules[imported_packages_module] = self - + def __getattr__(self, name): if name in _reponames: return _reponames[name] @@ -62,7 +80,7 @@ class RepoLoader(types.ModuleType): """Each RepoLoader is associated with a repository, and the RepoLoader is responsible for loading packages out of that repository. For example, a RepoLoader may be responsible for spack.repos.original, and when someone - references spack.repos.original.libelf that RepoLoader will load the + references spack.repos.original.libelf that RepoLoader will load the libelf package.""" def __init__(self, reponame, repopath): self.path = repopath @@ -70,7 +88,6 @@ def __init__(self, reponame, repopath): self.module_name = imported_packages_module + '.' + reponame if not reponame in _reponames: _reponames[reponame] = self - spack.repos.add_repo(reponame, self) import sys sys.modules[self.module_name] = self @@ -111,5 +128,3 @@ def get_module(self, pkg_name): pkg_name, file_path, e.message)) return module - - diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 790b22f3b0a..eed182a2573 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -30,45 +30,85 @@ from spack.packages import PackageDB from spack.test.mock_packages_test import * +# Some sample compiler config data +a_comps = { + "gcc@4.7.3" : { + "cc" : "/gcc473", + "cxx" : "/g++473", + "f77" : None, + "f90" : None }, + "gcc@4.5.0" : { + "cc" : "/gcc450", + "cxx" : "/g++450", + "f77" : "/gfortran", + "f90" : "/gfortran" }, + "clang@3.3" : { + "cc" : "", + "cxx" : "", + "f77" : "", + "f90" : "" } +} + +b_comps = { + "icc@10.0" : { + "cc" : "/icc100", + "cxx" : "/icc100", + "f77" : None, + "f90" : None }, + "icc@11.1" : { + "cc" : "/icc111", + "cxx" : "/icp111", + "f77" : "/ifort", + "f90" : "/ifort" }, + "clang@3.3" : { + "cc" : "/clang", + "cxx" : "/clang++", + "f77" : None, + "f90" : None} +} + class ConfigTest(MockPackagesTest): def setUp(self): - self.initmock() + super(ConfigTest, self).setUp() self.tmp_dir = mkdtemp('.tmp', 'spack-config-test-') - spack.config.config_scopes = [('test_low_priority', os.path.join(self.tmp_dir, 'low')), - ('test_high_priority', os.path.join(self.tmp_dir, 'high'))] + spack.config.config_scopes = [ + ('test_low_priority', os.path.join(self.tmp_dir, 'low')), + ('test_high_priority', os.path.join(self.tmp_dir, 'high'))] + def tearDown(self): - self.cleanmock() + super(ConfigTest, self).tearDown() shutil.rmtree(self.tmp_dir, True) - def check_config(self, comps): + + def check_config(self, comps, *compiler_names): + """Check that named compilers in comps match Spack's config.""" config = spack.config.get_compilers_config() compiler_list = ['cc', 'cxx', 'f77', 'f90'] - for key in comps: + for key in compiler_names: for c in compiler_list: - if comps[key][c] == '/bad': - continue self.assertEqual(comps[key][c], config[key][c]) - def test_write_key(self): - a_comps = {"gcc@4.7.3" : { "cc" : "/gcc473", "cxx" : "/g++473", "f77" : None, "f90" : None }, - "gcc@4.5.0" : { "cc" : "/gcc450", "cxx" : "/g++450", "f77" : "/gfortran", "f90" : "/gfortran" }, - "clang@3.3" : { "cc" : "/bad", "cxx" : "/bad", "f77" : "/bad", "f90" : "/bad" }} - - b_comps = {"icc@10.0" : { "cc" : "/icc100", "cxx" : "/icc100", "f77" : None, "f90" : None }, - "icc@11.1" : { "cc" : "/icc111", "cxx" : "/icp111", "f77" : "/ifort", "f90" : "/ifort" }, - "clang@3.3" : { "cc" : "/clang", "cxx" : "/clang++", "f77" : None, "f90" : None}} - + def test_write_key_in_memory(self): + # Write b_comps "on top of" a_comps. spack.config.add_to_compiler_config(a_comps, 'test_low_priority') spack.config.add_to_compiler_config(b_comps, 'test_high_priority') - self.check_config(a_comps) - self.check_config(b_comps) + # Make sure the config looks how we expect. + self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') + self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') + + def test_write_key_to_disk(self): + # Write b_comps "on top of" a_comps. + spack.config.add_to_compiler_config(a_comps, 'test_low_priority') + spack.config.add_to_compiler_config(b_comps, 'test_high_priority') + + # Clear caches so we're forced to read from disk. spack.config.clear_config_caches() - self.check_config(a_comps) - self.check_config(b_comps) - + # Same check again, to ensure consistency. + self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') + self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') diff --git a/var/spack/mock_packages/repo.yaml b/var/spack/mock_packages/repo.yaml new file mode 100644 index 00000000000..d0658960065 --- /dev/null +++ b/var/spack/mock_packages/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: mock diff --git a/var/spack/packages/repo.yaml b/var/spack/packages/repo.yaml new file mode 100644 index 00000000000..4a371e1cad4 --- /dev/null +++ b/var/spack/packages/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: gov.llnl.spack diff --git a/var/spack/packages/reponame b/var/spack/packages/reponame deleted file mode 100644 index 4b48deed3a4..00000000000 --- a/var/spack/packages/reponame +++ /dev/null @@ -1 +0,0 @@ -original From 8821715377147ae017d428d8c1216576de0b79e1 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Wed, 19 Aug 2015 14:18:02 -0700 Subject: [PATCH 019/910] add version-based url map for cmake, provide md5 for cmake 3.3.1 --- var/spack/packages/cmake/package.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/var/spack/packages/cmake/package.py b/var/spack/packages/cmake/package.py index 9efa370c8bb..ad3b69f76f7 100644 --- a/var/spack/packages/cmake/package.py +++ b/var/spack/packages/cmake/package.py @@ -23,21 +23,30 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * +import llnl.util.tty as tty class Cmake(Package): """A cross-platform, open-source build system. CMake is a family of tools designed to build, test and package software.""" homepage = 'https://www.cmake.org' - version('2.8.10.2', '097278785da7182ec0aea8769d06860c', - url = 'http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz') - - version('3.0.2', 'db4c687a31444a929d2fdc36c4dfb95f', - url = 'http://www.cmake.org/files/v3.0/cmake-3.0.2.tar.gz') - + version('2.8.10.2', '097278785da7182ec0aea8769d06860c') + version('3.0.2', 'db4c687a31444a929d2fdc36c4dfb95f') + version('3.3.1', '52638576f4e1e621fed6c3410d3a1b12') + # version('3.0.1', 'e2e05d84cb44a42f1371d9995631dcf5') # version('3.0.0', '21a1c85e1a3b803c4b48e7ff915a863e') + def url_for_version(self, version): + """Handle CMake's version-based custom URLs.""" + parts = [str(p) for p in Version(version)] + if len(parts) < 3: + tty.error("Version '%s'does not match CMake's version naming scheme (z.y.x)." % version) + version_short = ".".join(parts[:2]) + version_full = ".".join(parts) + return "http://www.cmake.org/files/v%s/cmake-%s.tar.gz" % (version_short,version_full) + + def install(self, spec, prefix): configure('--prefix=' + prefix, '--parallel=' + str(make_jobs)) From 24c609b6df5f7e30df0aa72f4c16f7272f1086cd Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Sat, 22 Aug 2015 09:34:44 -0500 Subject: [PATCH 020/910] Updates to the Krell related product packages. --- var/spack/packages/cbtf-argonavis/package.py | 14 +++---- var/spack/packages/cbtf-krell/package.py | 12 +++--- var/spack/packages/cbtf-lanl/package.py | 9 +++-- var/spack/packages/cbtf/package.py | 11 ++--- var/spack/packages/openspeedshop/package.py | 42 ++++++++++++++------ 5 files changed, 53 insertions(+), 35 deletions(-) diff --git a/var/spack/packages/cbtf-argonavis/package.py b/var/spack/packages/cbtf-argonavis/package.py index 5f76a18dae4..2dccb0e0c6e 100644 --- a/var/spack/packages/cbtf-argonavis/package.py +++ b/var/spack/packages/cbtf-argonavis/package.py @@ -23,14 +23,13 @@ class CbtfArgonavis(Package): libraries that was done as a result of a DOE SBIR grant.""" homepage = "http://sourceforge.net/p/cbtf/wiki/Home/" - url = "http://sourceforge.net/projects/cbtf/files/cbtf-1.5/cbtf-argonavis-1.5.tar.gz/download" - version('1.5', '8708197dbc08980fcd7b1a3a3a40a2c9') - # Mirror access template example - #url = "file:/opt/spack-mirror-2015-02-27/cbtf-argonavis/cbtf-argonavis-1.5.tar.gz" + #url = "file:/g/g24/jeg/cbtf-argonavis-1.5.tar.gz" #version('1.5', '1f7f6512f55409ed2135cfceabe26b82') - depends_on("cmake") + version('1.6', branch='master', git='http://git.code.sf.net/p/cbtf-argonavis/cbtf-argonavis') + + depends_on("cmake@3.0.2:") depends_on("papi") depends_on("cbtf") depends_on("cbtf-krell") @@ -44,8 +43,9 @@ def install(self, spec, prefix): cmake_prefix_path = join_path(spec['cbtf'].prefix) + ':' + join_path(spec['cbtf-krell'].prefix) # FIXME, hard coded for testing purposes, we will alter when the external package feature is available - cuda_prefix_path = "/usr/local/cuda-6.0" - cupti_prefix_path = "/usr/local/cuda-6.0/extras/CUPTI" + cuda_prefix_path = "/opt/cudatoolkit-6.5" + cupti_prefix_path = "/opt/cudatoolkit-6.5/extras/CUPTI" + with working_dir('CUDA'): with working_dir('build', create=True): diff --git a/var/spack/packages/cbtf-krell/package.py b/var/spack/packages/cbtf-krell/package.py index 0661b101a16..3f486b86ced 100644 --- a/var/spack/packages/cbtf-krell/package.py +++ b/var/spack/packages/cbtf-krell/package.py @@ -25,13 +25,12 @@ class CbtfKrell(Package): HPC levels of scale.""" homepage = "http://sourceforge.net/p/cbtf/wiki/Home/" - url = "http://sourceforge.net/projects/cbtf/files/cbtf-1.5/cbtf-krell-1.5.tar.gz/download" - version('1.5', '86cca5aa48a65206d3ac5502fcbd3ef6') - # optional mirror access template - #url = "file:/opt/spack-mirror-2015-02-27/cbtf-krell/cbtf-krell-1.5.tar.gz" + #url = "file:/g/g24/jeg/cbtf-krell-1.5.tar.gz" #version('1.5', 'b13f6df6a93c44149d977773dd776d2f') + version('1.6', branch='master', git='http://git.code.sf.net/p/cbtf-krell/cbtf-krell') + # Dependencies for cbtf-krell @@ -39,7 +38,7 @@ class CbtfKrell(Package): depends_on("binutils@2.24+krellpatch") # collectionTool - depends_on("boost@1.41:") + depends_on("boost@1.42:") depends_on("dyninst@8.2.1:+krelloptions") depends_on("mrnet@4.1.0:+krelloptions") depends_on("xerces-c@3.1.1:") @@ -66,6 +65,8 @@ def install(self, spec, prefix): # FIXME - hard code path until external package support is available # Need to change this path and/or add additional paths for MPI experiment support on different platforms openmpi_prefix_path = "/opt/openmpi-1.8.2" + mvapich_prefix_path = "/usr/local/tools/mvapich-gnu" + # Other possibilities, they will need a -DMVAPICH_DIR=, etc clause in the cmake command to be recognized # mvapich_prefix_path = "" # mvapich2_prefix_path = "" @@ -106,6 +107,7 @@ def install(self, spec, prefix): '-DCUDA_DIR=%s' % cuda_prefix_path, '-DCUPTI_DIR=%s' % cupti_prefix_path, '-DOPENMPI_DIR=%s' % openmpi_prefix_path, + '-DMVAPICH_DIR=%s' % mvapich_prefix_path, *std_cmake_args) make("clean") diff --git a/var/spack/packages/cbtf-lanl/package.py b/var/spack/packages/cbtf-lanl/package.py index b833871df78..25f91d8c2f5 100644 --- a/var/spack/packages/cbtf-lanl/package.py +++ b/var/spack/packages/cbtf-lanl/package.py @@ -22,15 +22,16 @@ class CbtfLanl(Package): """CBTF LANL project contains a memory tool and data center type system command monitoring tool.""" homepage = "http://sourceforge.net/p/cbtf/wiki/Home/" - url = "http://sourceforge.net/projects/cbtf/files/cbtf-1.5/cbtf-lanl-1.5.tar.gz/download" - version('1.5', '78d42050f9ec7127e8d0a93d87a66702') # Mirror access template example - #url = "file:/opt/spack-mirror-2015-02-27/cbtf-lanl/cbtf-lanl-1.5.tar.gz" + #url = "file:/g/g24/jeg/cbtf-lanl-1.5.tar.gz" #version('1.5', 'c3f78f967b0a42c6734ce4be0e602426') + version('1.6', branch='master', git='http://git.code.sf.net/p/cbtf-lanl/cbtf-lanl') + + # Dependencies for cbtf-krell - depends_on("boost@1.41:") + depends_on("boost@1.42:") depends_on("mrnet@4.1.0+krelloptions") depends_on("xerces-c@3.1.1:") depends_on("cbtf") diff --git a/var/spack/packages/cbtf/package.py b/var/spack/packages/cbtf/package.py index d2023ada858..46d47153e62 100644 --- a/var/spack/packages/cbtf/package.py +++ b/var/spack/packages/cbtf/package.py @@ -24,18 +24,15 @@ class Cbtf(Package): networks into sequential and distributed network tools.""" homepage = "http://sourceforge.net/p/cbtf/wiki/Home" - url = "http://sourceforge.net/projects/cbtf/files/cbtf-1.5/cbtf-1.5.tar.gz/download" - version('1.5', '75a97e0161d19b3a12305ed1ffb3d3e2') - # Mirror access template example - #url = "file:/opt/spack-mirror-2015-02-27/cbtf/cbtf-1.5.tar.gz" - #version('1.5', '1ca88a8834759c4c74452cb97fe7b70a') + #url = "file:/g/g24/jeg/cbtf-1.5.tar.gz" + #version('1.6', '1ca88a8834759c4c74452cb97fe7b70a') # Use when the git repository is available - #version('1.5', branch='master', git='http://git.code.sf.net/p/cbtf/cbtf') + version('1.6', branch='master', git='http://git.code.sf.net/p/cbtf/cbtf') depends_on("cmake") - depends_on("boost@1.41:") + depends_on("boost@1.42:") depends_on("mrnet@4.1.0+krelloptions") depends_on("xerces-c@3.1.1:") depends_on("libxml2") diff --git a/var/spack/packages/openspeedshop/package.py b/var/spack/packages/openspeedshop/package.py index 94fec4a21cb..3c24aed8824 100644 --- a/var/spack/packages/openspeedshop/package.py +++ b/var/spack/packages/openspeedshop/package.py @@ -30,11 +30,13 @@ class Openspeedshop(Package): homepage = "http://www.openspeedshop.org" url = "http://sourceforge.net/projects/openss/files/openss/openspeedshop-2.1/openspeedshop-2.1.tar.gz/download" - version('2.1', '85ab0f883f16ea23815a670b5ec8e5d6') + version('2.1', '64ee17166519838c7b94a1adc138e94f') # optional mirror template - #url = "file:/home/jeg/OpenSpeedShop_ROOT/SOURCES/openspeedshop-2.1.tar.gz" - #version('2.1', '85ab0f883f16ea23815a670b5ec8e5d6') + #url = "file:/g/g24/jeg/openspeedshop-2.1.tar.gz" + #version('2.1', '64ee17166519838c7b94a1adc138e94f') + + parallel = False @@ -56,7 +58,7 @@ class Openspeedshop(Package): depends_on("libelf") depends_on("libdwarf") depends_on("sqlite") - depends_on("boost@1.41:") + depends_on("boost@1.42:") depends_on("dyninst@8.2.1:+krelloptions") depends_on("python") depends_on("qt@3.3.8b+krellpatch") @@ -72,14 +74,14 @@ class Openspeedshop(Package): # Dependencies only for the openspeedshop cbtf package. depends_on("cbtf", when='+cbtf') depends_on("cbtf-krell", when='+cbtf') - depends_on("cbtf-argonavis", when='+cbtf') + #depends_on("cbtf-argonavis", when='+cbtf') depends_on("mrnet@4.1.0:+krelloptions", when='+cbtf') def install(self, spec, prefix): openmpi_prefix_path = "/opt/openmpi-1.8.2" + mvapich_prefix_path = "/usr/local/tools/mvapich-gnu" #'-DOPENMPI_DIR=%s' % spec['openmpi'].prefix, - #'-DMPICH_DIR=%s' % spec['mpich'].prefix, # FIXME: How do we make this dynamic in spack? That is, can we specify the paths to cuda dynamically? # WAITING for external package support. @@ -99,6 +101,7 @@ def install(self, spec, prefix): '-DLIBUNWIND_DIR=%s' % spec['libunwind'].prefix, '-DPAPI_DIR=%s' % spec['papi'].prefix, '-DOPENMPI_DIR=%s' % openmpi_prefix_path, + '-DMVAPICH_DIR=%s' % mvapich_prefix_path, *std_cmake_args) make("clean") make() @@ -106,6 +109,8 @@ def install(self, spec, prefix): else: cmake_prefix_path = join_path(spec['dyninst'].prefix) with working_dir('build', create=True): + #python_vers=join_path(spec['python'].version[:2]) + python_vers='%d.%d' % spec['python'].version[:2] cmake('..', '-DCMAKE_INSTALL_PREFIX=%s' % prefix, '-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, @@ -119,10 +124,14 @@ def install(self, spec, prefix): '-DPAPI_DIR=%s' % spec['papi'].prefix, '-DSQLITE3_DIR=%s' % spec['sqlite'].prefix, '-DQTLIB_DIR=%s' % spec['qt'].prefix, - '-DPYTHON_DIR=%s' % spec['python'].prefix, - '-DBOOST_DIR=%s' % spec['boost'].prefix, + '-DPYTHON_EXECUTABLE=%s' % join_path(spec['python'].prefix + '/bin/python'), + '-DPYTHON_INCLUDE_DIR=%s' % join_path(spec['python'].prefix.include) + '/python' + python_vers, + '-DPYTHON_LIBRARY=%s' % join_path(spec['python'].prefix.lib) + '/libpython' + python_vers + '.so', + '-DBoost_NO_SYSTEM_PATHS=TRUE', + '-DBOOST_ROOT=%s' % spec['boost'].prefix, '-DDYNINST_DIR=%s' % spec['dyninst'].prefix, '-DOPENMPI_DIR=%s' % openmpi_prefix_path, + '-DMVAPICH_DIR=%s' % mvapich_prefix_path, *std_cmake_args) make("clean") make() @@ -133,6 +142,7 @@ def install(self, spec, prefix): cmake_prefix_path = join_path(spec['cbtf'].prefix) + ':' + join_path(spec['cbtf-krell'].prefix) + ':' + join_path(spec['dyninst'].prefix) if '+runtime' in spec: with working_dir('build_cbtf_runtime', create=True): + python_vers='%d.%d' % spec['python'].version[:2] cmake('..', '-DCMAKE_INSTALL_PREFIX=%s' % prefix, '-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, @@ -143,8 +153,11 @@ def install(self, spec, prefix): '-DLIBDWARF_DIR=%s' % spec['libdwarf'].prefix, '-DCBTF_DIR=%s' % spec['cbtf'].prefix, '-DCBTF_KRELL_DIR=%s' % spec['cbtf-krell'].prefix, - '-DPYTHON_DIR=%s' % spec['python'].prefix, - '-DBOOST_DIR=%s' % spec['boost'].prefix, + '-DPYTHON_EXECUTABLE=%s' % join_path(spec['python'].prefix + '/bin/python'), + '-DPYTHON_INCLUDE_DIR=%s' % join_path(spec['python'].prefix.include) + '/python' + python_vers, + '-DPYTHON_LIBRARY=%s' % join_path(spec['python'].prefix.lib) + '/libpython' + python_vers + '.so', + '-DBoost_NO_SYSTEM_PATHS=TRUE', + '-DBOOST_ROOT=%s' % spec['boost'].prefix, '-DDYNINST_DIR=%s' % spec['dyninst'].prefix, '-DMRNET_DIR=%s' % spec['mrnet'].prefix, *std_cmake_args) @@ -154,6 +167,8 @@ def install(self, spec, prefix): else: with working_dir('build_cbtf', create=True): + python_vers='%d.%d' % spec['python'].version[:2] + #python_vers=join_path(spec['python'].version[:2]) cmake('..', '-DCMAKE_INSTALL_PREFIX=%s' % prefix, '-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, @@ -166,8 +181,11 @@ def install(self, spec, prefix): '-DCBTF_DIR=%s' % spec['cbtf'].prefix, '-DCBTF_KRELL_DIR=%s' % spec['cbtf-krell'].prefix, '-DQTLIB_DIR=%s' % spec['qt'].prefix, - '-DPYTHON_DIR=%s' % spec['python'].prefix, - '-DBOOST_DIR=%s' % spec['boost'].prefix, + '-DPYTHON_EXECUTABLE=%s' % join_path(spec['python'].prefix + '/bin/python'), + '-DPYTHON_INCLUDE_DIR=%s' % join_path(spec['python'].prefix.include) + '/python' + python_vers, + '-DPYTHON_LIBRARY=%s' % join_path(spec['python'].prefix.lib) + '/libpython' + python_vers + '.so', + '-DBoost_NO_SYSTEM_PATHS=TRUE', + '-DBOOST_ROOT=%s' % spec['boost'].prefix, '-DDYNINST_DIR=%s' % spec['dyninst'].prefix, '-DMRNET_DIR=%s' % spec['mrnet'].prefix, *std_cmake_args) From 360b307f683e146151a65e1d788ce1d154c47ace Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 24 Aug 2015 09:14:16 -0700 Subject: [PATCH 021/910] Save progress. import gov.llnl.spack.mpich works. --- lib/spack/llnl/util/tty/colify.py | 7 + lib/spack/spack/__init__.py | 12 +- lib/spack/spack/cmd/repo.py | 17 +- lib/spack/spack/config.py | 4 +- lib/spack/spack/packages.py | 568 ++++++++++++------- lib/spack/spack/repo_loader.py | 22 +- lib/spack/spack/spec.py | 2 +- lib/spack/spack/test/directory_layout.py | 9 +- lib/spack/spack/test/mock_packages_test.py | 12 +- lib/spack/spack/test/package_sanity.py | 6 +- lib/spack/spack/test/packages.py | 6 +- lib/spack/spack/util/naming.py | 31 + var/spack/mock_packages/_repo.yaml | 2 + var/spack/mock_packages/repo.yaml | 2 - var/spack/packages/{repo.yaml => _repo.yaml} | 0 15 files changed, 452 insertions(+), 248 deletions(-) create mode 100644 var/spack/mock_packages/_repo.yaml delete mode 100644 var/spack/mock_packages/repo.yaml rename var/spack/packages/{repo.yaml => _repo.yaml} (100%) diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index 66c52c39687..acf64c1e13f 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -220,6 +220,13 @@ def colify(elts, **options): def colify_table(table, **options): + """Version of colify() for data expressed in rows, (list of lists). + + Same as regular colify but takes a list of lists, where each + sub-list must be the same length, and each is interpreted as a + row in a table. Regular colify displays a sequential list of + values in columns. + """ if table is None: raise TypeError("Can't call colify_table on NoneType") elif not table or not table[0]: diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 09bc9ca52a9..71e3ac3715f 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -23,8 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import sys import tempfile from llnl.util.filesystem import * +import llnl.util.tty as tty # This lives in $prefix/lib/spack/spack/__file__ prefix = ancestor(__file__, 4) @@ -42,6 +44,7 @@ hooks_path = join_path(module_path, "hooks") var_path = join_path(prefix, "var", "spack") stage_path = join_path(var_path, "stage") +packages_path = join_path(var_path, "packages") opt_path = join_path(prefix, "opt") install_path = join_path(opt_path, "spack") share_path = join_path(prefix, "share", "spack") @@ -55,9 +58,12 @@ # # Set up the default packages database. # -from spack.packages import PackageDB -packages_path = join_path(var_path, "packages") -db = PackageDB() +import spack.packages +_repo_paths = spack.config.get_repos_config() +if not _repo_paths: + tty.die("Spack configuration contains no package repositories.") +db = spack.packages.PackageFinder(*_repo_paths) +sys.meta_path.append(db) # # Paths to mock files for testing. diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 1261c7ada9d..e290f60b7b9 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -32,7 +32,7 @@ import spack.spec import spack.config from spack.util.environment import get_path -from spack.packages import repo_config +from spack.packages import repo_config_filename import os import exceptions @@ -50,13 +50,8 @@ def setup_parser(subparser): create_parser = sp.add_parser('create', help=repo_create.__doc__) create_parser.add_argument('directory', help="Directory containing the packages.") create_parser.add_argument('name', help="Name of new package repository.") -<<<<<<< HEAD:lib/spack/spack/cmd/packagerepo.py - - remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__) -======= remove_parser = sp.add_parser('remove', help=repo_remove.__doc__) ->>>>>>> Save changes to external repo integration:lib/spack/spack/cmd/repo.py remove_parser.add_argument('name') list_parser = sp.add_parser('list', help=repo_list.__doc__) @@ -81,7 +76,7 @@ def repo_add(args): """Add package sources to the Spack configuration.""" if not add_to_config(args.directory): tty.die('Repo directory %s already exists in the repo list' % dir) - + def repo_create(args): """Create a new package repo at a directory and name""" @@ -95,13 +90,13 @@ def repo_create(args): mkdirp(dir) except exceptions.OSError, e: tty.die('Failed to create new directory %s' % dir) - path = os.path.join(dir, repo_config) + path = os.path.join(dir, repo_config_filename) try: with closing(open(path, 'w')) as repofile: repofile.write(name + '\n') except exceptions.IOError, e: tty.die('Could not create new file %s' % path) - + if not add_to_config(args.directory): tty.warn('Repo directory %s already exists in the repo list' % dir) @@ -118,8 +113,8 @@ def repo_list(args): fmt = "%%-%ds%%s" % (max_len + 4) for root in root_names: print fmt % (root[0], root[1]) - - + + def repo(parser, args): action = { 'add' : repo_add, diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index dc59f9a5a32..66da91f6290 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -269,7 +269,9 @@ def get_repos_config(): config = get_config() if 'repos' not in config: return [] - return config['repos'] + + repo_list = config['repos'] + return [substitute_spack_prefix(repo) for repo in repo_list] def get_mirror_config(): diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index c4142343860..df54b12324d 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -28,7 +28,6 @@ import inspect import glob import imp -import spack.config import re import itertools import traceback @@ -41,149 +40,327 @@ import spack.error import spack.spec from spack.virtual import ProviderIndex -from spack.util.naming import mod_to_class, validate_module_name -from sets import Set -from spack.repo_loader import RepoLoader, imported_packages_module, package_file_name +from spack.util.naming import * # Filename for package repo names -repo_config = 'repo.yaml' +repo_config_filename = '_repo.yaml' + +# Filename for packages in repos. +package_file_name = 'package.py' def _autospec(function): """Decorator that automatically converts the argument of a single-arg function to a Spec.""" - def converter(self, spec_like, **kwargs): + def converter(self, spec_like, *args, **kwargs): if not isinstance(spec_like, spack.spec.Spec): spec_like = spack.spec.Spec(spec_like) - return function(self, spec_like, **kwargs) + return function(self, spec_like, *args, **kwargs) return converter -def sliding_window(seq, n): - it = iter(seq) - result = tuple(itertools.islice(it, n)) - if len(result) == n: - yield result - for elem in it: - result = result[1:] + (elem,) - yield result +class NamespaceTrie(object): + def __init__(self): + self._elements = {} -class PackageDB(object): + def __setitem__(self, namespace, repo): + parts = namespace.split('.') + cur = self._elements + for p in parts[:-1]: + if p not in cur: + cur[p] = {} + cur = cur[p] + + cur[parts[-1]] = repo + + + def __getitem__(self, namespace): + parts = namespace.split('.') + cur = self._elements + for p in parts: + if p not in cur: + raise KeyError("Can't find namespace %s in trie" % namespace) + cur = cur[p] + return cur + + + def __contains__(self, namespace): + parts = namespace.split('.') + cur = self._elements + for p in parts: + if not isinstance(cur, dict): + return False + if p not in cur: + return False + cur = cur[p] + return True + + + +class PackageFinder(object): + """A PackageFinder is a wrapper around a list of PackageDBs. + + It functions exactly like a PackageDB, but it operates on the + combined results of the PackageDBs in its list instead of on a + single package repository. + """ def __init__(self, *repo_dirs): - """Construct a new package database from a list of directories. - - Args: - repo_dirs List of directories containing packages. - - If ``repo_dirs`` is empty, gets repository list from Spack configuration. - """ - if not repo_dirs: - repo_dirs = spack.config.get_repos_config() - if not repo_dirs: - tty.die("Spack configuration contains no package repositories.") - - # Collect the repos from the config file and read their names - # from the file system - repo_dirs = [spack.config.substitute_spack_prefix(rd) for rd in repo_dirs] - self.repos = [] - for rdir in repo_dirs: - rname = self._read_reponame_from_directory(rdir) - if rname: - self.repos.append((self._read_reponame_from_directory(rdir), rdir)) + self.by_namespace = NamespaceTrie() + self.by_path = {} + + for root in repo_dirs: + repo = PackageDB(root) + self.put_last(repo) - by_path = sorted(self.repos, key=lambda r:r[1]) - by_name = sorted(self.repos, key=lambda r:r[0]) + def _check_repo(self, repo): + if repo.root in self.by_path: + raise DuplicateRepoError("Package repos are the same", + repo, self.by_path[repo.root]) - for r1, r2 in by_path: - if r1[1] == r2[1]: - tty.die("Package repos are the same:", - " %20s %s" % r1, " %20s %s" % r2) - - for r1, r2 in by_name: - if r1[0] == r2[0]: - tty.die("Package repos cannot have the same name:", - " %20s %s" % r1, " %20s %s" % r2) - - # For each repo, create a RepoLoader - self.repo_loaders = dict((name, RepoLoader(name, path)) - for name, path in self.repos) - - self.instances = {} - self.provider_index = None + if repo.namespace in self.by_namespace: + tty.error("Package repos cannot have the same name", + repo, self.by_namespace[repo.namespace]) - def _read_reponame_from_directory(self, dir): - """For a packagerepo directory, read the repo name from the - $root/repo.yaml file""" - path = os.path.join(dir, repo_config) + def _add(self, repo): + self._check_repo(repo) + self.by_namespace[repo.namespace] = repo + self.by_path[repo.root] = repo - try: - with open(path) as reponame_file: - yaml_data = yaml.load(reponame_file) - if (not yaml_data or - 'repo' not in yaml_data or - 'namespace' not in yaml_data['repo']): - tty.die("Invalid %s in %s" % (repo_config, dir)) + def put_first(self, repo): + self._add(repo) + self.repos.insert(0, repo) - name = yaml_data['repo']['namespace'] - if not re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', name): - tty.die( - "Package repo name '%s', read from %s, is an invalid name. " - "Repo names must began with a letter and only contain " - "letters and numbers." % (name, path)) + + def put_last(self, repo): + self._add(repo) + self.repos.append(repo) + + + def remove(self, repo): + if repo in self.repos: + self.repos.remove(repo) + + + def swap(self, other): + repos = self.repos + by_namespace = self.by_namespace + by_path = self.by_path + + self.repos = other.repos + self.by_namespace = other.by_namespace + self.by_pah = other.by_path + + other.repos = repos + other.by_namespace = by_namespace + other.by_path = by_path + + + def all_package_names(self): + all_pkgs = set() + for repo in self.repos: + all_pkgs.update(set(repo.all_package_names())) + return all_pkgs + + + def all_packages(self): + for name in self.all_package_names(): + yield self.get(name) + + + def providers_for(self, vpkg_name): + # TODO: USE MORE THAN FIRST REPO + return self.repos[0].providers_for(vpkg_name) + + + def _get_spack_pkg_name(self, repo, py_module_name): + """Allow users to import Spack packages using legal Python identifiers. + + A python identifier might map to many different Spack package + names due to hyphen/underscore ambiguity. + + Easy example: + num3proxy -> 3proxy + + Ambiguous: + foo_bar -> foo_bar, foo-bar + + More ambiguous: + foo_bar_baz -> foo_bar_baz, foo-bar-baz, foo_bar-baz, foo-bar_baz + """ + if py_module_name in repo: + return py_module_name + + options = possible_spack_module_names(py_module_name) + options.remove(py_module_name) + for name in options: + if name in repo: return name - except exceptions.IOError, e: - tty.die("Error reading %s when opening %s" % (repo_config, dir)) + + return None + + + def find_module(self, fullname, path=None): + if fullname in self.by_namespace: + return self + + namespace, dot, module_name = fullname.rpartition('.') + if namespace not in self.by_namespace: + return None + + repo = self.by_namespace[namespace] + name = self._get_spack_pkg_name(repo, module_name) + if not name: + return None + + return self + + + def load_module(self, fullname): + if fullname in sys.modules: + return sys.modules[fullname] + + if fullname in self.by_namespace: + ns = self.by_namespace[fullname] + module = imp.new_module(fullname) + module.__file__ = "" + module.__path__ = [] + module.__package__ = fullname + + else: + namespace, dot, module_name = fullname.rpartition('.') + if namespace not in self.by_namespace: + raise ImportError( + "No Spack repository with namespace %s" % namespace) + + repo = self.by_namespace[namespace] + name = self._get_spack_pkg_name(repo, module_name) + if not name: + raise ImportError( + "No module %s in Spack repository %s" % (module_name, repo)) + + fullname = namespace + '.' + name + file_path = os.path.join(repo.root, name, package_file_name) + module = imp.load_source(fullname, file_path) + module.__package__ = namespace + + module.__loader__ = self + sys.modules[fullname] = module + return module @_autospec - def get(self, spec, **kwargs): + def get(self, spec, new=False): + for repo in self.repos: + if spec.name in repo: + return repo.get(spec, new) + raise UnknownPackageError(spec.name) + + + def get_repo(self, namespace): + if namespace in self.by_namespace: + repo = self.by_namespace[namespace] + if isinstance(repo, PackageDB): + return repo + return None + + + def exists(self, pkg_name): + return any(repo.exists(pkg_name) for repo in self.repos) + + + def __contains__(self, pkg_name): + return self.exists(pkg_name) + + + +class PackageDB(object): + """Class representing a package repository in the filesystem. + + Each package repository must have a top-level configuration file + called `_repo.yaml`. + + Currently, `_repo.yaml` this must define: + + `namespace`: + A Python namespace where the repository's packages should live. + + """ + def __init__(self, root): + """Instantiate a package repository from a filesystem path.""" + # Root directory, containing _repo.yaml and package dirs + self.root = root + + # Config file in /_repo.yaml + self.config_file = os.path.join(self.root, repo_config_filename) + + # Read configuration from _repo.yaml + config = self._read_config() + if not 'namespace' in config: + tty.die('Package repo in %s must define a namespace in %s.' + % (self.root, repo_config_filename)) + + # Check namespace in the repository configuration. + self.namespace = config['namespace'] + if not re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', self.namespace): + tty.die(("Invalid namespace '%s' in '%s'. Namespaces must be " + "valid python identifiers separated by '.'") + % (self.namespace, self.root)) + + # These are internal cache variables. + self._instances = {} + self._provider_index = None + + + def _read_config(self): + """Check for a YAML config file in this db's root directory.""" + try: + with open(self.config_file) as reponame_file: + yaml_data = yaml.load(reponame_file) + + if (not yaml_data or 'repo' not in yaml_data or + not isinstance(yaml_data['repo'], dict)): + tty.die("Invalid %s in repository %s" + % (repo_config_filename, self.root)) + + return yaml_data['repo'] + + except exceptions.IOError, e: + tty.die("Error reading %s when opening %s" + % (self.config_file, self.root)) + + + @_autospec + def get(self, spec, new=False): if spec.virtual: raise UnknownPackageError(spec.name) - if kwargs.get('new', False): - if spec in self.instances: - del self.instances[spec] + if new: + if spec in self._instances: + del self._instances[spec] - if not spec in self.instances: + if not spec in self._instances: package_class = self.get_class_for_package_name(spec.name, spec.repo) try: copy = spec.copy() - self.instances[copy] = package_class(copy) + self._instances[copy] = package_class(copy) except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) raise FailedConstructorError(spec.name, *sys.exc_info()) - return self.instances[spec] - - - @_autospec - def delete(self, spec): - """Force a package to be recreated.""" - del self.instances[spec] - - - def purge(self): - """Clear entire package instance cache.""" - self.instances.clear() - - - @_autospec - def get_installed(self, spec): - """Get all the installed specs that satisfy the provided spec constraint.""" - return [s for s in self.installed_package_specs() if s.satisfies(spec)] + return self._instances[spec] @_autospec def providers_for(self, vpkg_spec): - if self.provider_index is None: - self.provider_index = ProviderIndex(self.all_package_names()) + if self._provider_index is None: + self._provider_index = ProviderIndex(self.all_package_names()) - providers = self.provider_index.providers_for(vpkg_spec) + providers = self._provider_index.providers_for(vpkg_spec) if not providers: raise UnknownPackageError(vpkg_spec.name) return providers @@ -194,6 +371,97 @@ def extensions_for(self, extendee_spec): return [p for p in self.all_packages() if p.extends(extendee_spec)] + def dirname_for_package_name(self, pkg_name): + """Get the directory name for a particular package. This is the + directory that contains its package.py file.""" + return join_path(self.root, pkg_name) + + + def filename_for_package_name(self, pkg_name): + """Get the filename for the module we should load for a particular + package. Packages for a pacakge DB live in + ``$root//package.py`` + + This will return a proper package.py path even if the + package doesn't exist yet, so callers will need to ensure + the package exists before importing. + """ + validate_module_name(pkg_name) + pkg_dir = self.dirname_for_package_name(pkg_name) + return join_path(pkg_dir, package_file_name) + + + @memoized + def all_package_names(self): + """Generator function for all packages. This looks for + ``/package.py`` files within the repo direcotories""" + all_package_names = [] + + for pkg_name in os.listdir(self.root): + pkg_dir = join_path(self.root, pkg_name) + pkg_file = join_path(pkg_dir, package_file_name) + if os.path.isfile(pkg_file): + all_package_names.append(pkg_name) + + return sorted(all_package_names) + + + def all_packages(self): + for name in self.all_package_names(): + yield self.get(name) + + + @memoized + def exists(self, pkg_name): + """Whether a package with the supplied name exists.""" + return os.path.exists(self.filename_for_package_name(pkg_name)) + + + @memoized + def get_class_for_package_name(self, pkg_name, reponame = None): + """Get an instance of the class for a particular package.""" + file_path = self.filename_for_package_name(pkg_name) + + if os.path.exists(file_path): + if not os.path.isfile(file_path): + tty.die("Something's wrong. '%s' is not a file!" % file_path) + if not os.access(file_path, os.R_OK): + tty.die("Cannot read '%s'!" % file_path) + else: + raise UnknownPackageError(pkg_name, self.namespace) + + class_name = mod_to_class(pkg_name) + module = __import__(self.namespace + '.' + pkg_name, fromlist=[class_name]) + cls = getattr(module, class_name) + if not inspect.isclass(cls): + tty.die("%s.%s is not a class" % (pkg_name, class_name)) + + return cls + + + def __str__(self): + return "" % (self.namespace, self.root) + + + def __repr__(self): + return self.__str__() + + + def __contains__(self, pkg_name): + return self.exists(pkg_name) + + + # + # Below functions deal with installed packages, and should be + # moved to some other part of Spack (conbine with + # directory_layout?) + # + @_autospec + def get_installed(self, spec): + """Get all the installed specs that satisfy the provided spec constraint.""" + return [s for s in self.installed_package_specs() if s.satisfies(spec)] + + @_autospec def installed_extensions_for(self, extendee_spec): for s in self.installed_package_specs(): @@ -203,53 +471,6 @@ def installed_extensions_for(self, extendee_spec): except UnknownPackageError, e: # Skip packages we know nothing about continue - # TODO: add some conditional way to do this instead of - # catching exceptions. - - - def repo_for_package_name(self, pkg_name, packagerepo_name=None): - """Find the dirname for a package and the packagerepo it came from - if packagerepo_name is not None, then search for the package in the - specified packagerepo""" - #Look for an existing package under any matching packagerepos - roots = [pkgrepo for pkgrepo in self.repos - if not packagerepo_name or packagerepo_name == pkgrepo[0]] - - if not roots: - tty.die("Package repo %s does not exist" % packagerepo_name) - - for pkgrepo in roots: - path = join_path(pkgrepo[1], pkg_name) - if os.path.exists(path): - return (pkgrepo[0], path) - - repo_to_add_to = roots[-1] - return (repo_to_add_to[0], join_path(repo_to_add_to[1], pkg_name)) - - - def dirname_for_package_name(self, pkg_name, packagerepo_name=None): - """Get the directory name for a particular package. This is the - directory that contains its package.py file.""" - return self.repo_for_package_name(pkg_name, packagerepo_name)[1] - - - def filename_for_package_name(self, pkg_name, packagerepo_name=None): - """Get the filename for the module we should load for a particular - package. Packages for a pacakge DB live in - ``$root//package.py`` - - This will return a proper package.py path even if the - package doesn't exist yet, so callers will need to ensure - the package exists before importing. - - If a packagerepo is specified, then return existing - or new paths in the specified packagerepo directory. If no - package repo is supplied, return an existing path from any - package repo, and new paths in the default package repo. - """ - validate_module_name(pkg_name) - pkg_dir = self.dirname_for_package_name(pkg_name, packagerepo_name) - return join_path(pkg_dir, package_file_name) def installed_package_specs(self): @@ -275,52 +496,6 @@ def installed_known_package_specs(self): yield spec - @memoized - def all_package_names(self): - """Generator function for all packages. This looks for - ``/package.py`` files within the repo direcotories""" - all_packages = Set() - for repo in self.repos: - dir = repo[1] - if not os.path.isdir(dir): - continue - for pkg_name in os.listdir(dir): - pkg_dir = join_path(dir, pkg_name) - pkg_file = join_path(pkg_dir, package_file_name) - if os.path.isfile(pkg_file): - all_packages.add(pkg_name) - all_package_names = list(all_packages) - all_package_names.sort() - return all_package_names - - - def all_packages(self): - for name in self.all_package_names(): - yield self.get(name) - - - @memoized - def exists(self, pkg_name): - """Whether a package with the supplied name exists .""" - return os.path.exists(self.filename_for_package_name(pkg_name)) - - - @memoized - def get_class_for_package_name(self, pkg_name, reponame = None): - """Get an instance of the class for a particular package.""" - (reponame, repodir) = self.repo_for_package_name(pkg_name, reponame) - module_name = imported_packages_module + '.' + reponame + '.' + pkg_name - - module = self.repo_loaders[reponame].get_module(pkg_name) - - class_name = mod_to_class(pkg_name) - cls = getattr(module, class_name) - if not inspect.isclass(cls): - tty.die("%s.%s is not a class" % (pkg_name, class_name)) - - return cls - - class UnknownPackageError(spack.error.SpackError): """Raised when we encounter a package spack doesn't have.""" def __init__(self, name, repo=None): @@ -333,6 +508,13 @@ def __init__(self, name, repo=None): self.name = name +class DuplicateRepoError(spack.error.SpackError): + """Raised when duplicate repos are added to a PackageFinder.""" + def __init__(self, msg, repo1, repo2): + super(UnknownPackageError, self).__init__( + "%s: %s, %s" % (msg, repo1, repo2)) + + class FailedConstructorError(spack.error.SpackError): """Raised when a package's class constructor fails.""" def __init__(self, name, exc_type, exc_obj, exc_tb): diff --git a/lib/spack/spack/repo_loader.py b/lib/spack/spack/repo_loader.py index 92da1cf7098..441011cf986 100644 --- a/lib/spack/spack/repo_loader.py +++ b/lib/spack/spack/repo_loader.py @@ -12,7 +12,6 @@ # Name of the package file inside a package directory package_file_name = 'package.py' -import sys class LazyLoader: """The LazyLoader handles cases when repo modules or classes are imported. It watches for 'spack.repos.*' loads, then @@ -21,15 +20,6 @@ def find_module(self, fullname, pathname): if not fullname.startswith(imported_packages_module): return None - print "HERE ===" - print - for line in traceback.format_stack(): - print " ", line.strip() - print - print "full: ", fullname - print "path: ", pathname - print - partial_name = fullname[len(imported_packages_module)+1:] print "partial: ", partial_name @@ -50,7 +40,7 @@ def find_module(self, fullname, pathname): def load_module(self, fullname): return self.mod -sys.meta_path.append(LazyLoader()) +#sys.meta_path.append(LazyLoader()) _reponames = {} class RepoNamespace(types.ModuleType): @@ -59,7 +49,6 @@ class RepoNamespace(types.ModuleType): this class will use __getattr__ to translate the 'original' into one of spack's known repositories""" def __init__(self): - import sys sys.modules[imported_packages_module] = self def __getattr__(self, name): @@ -89,7 +78,6 @@ def __init__(self, reponame, repopath): if not reponame in _reponames: _reponames[reponame] = self - import sys sys.modules[self.module_name] = self @@ -110,14 +98,6 @@ def get_module(self, pkg_name): import imp import llnl.util.tty as tty - file_path = os.path.join(self.path, pkg_name, package_file_name) - if os.path.exists(file_path): - if not os.path.isfile(file_path): - tty.die("Something's wrong. '%s' is not a file!" % file_path) - if not os.access(file_path, os.R_OK): - tty.die("Cannot read '%s'!" % file_path) - else: - raise spack.packages.UnknownPackageError(pkg_name, self.reponame if self.reponame != 'original' else None) try: module_name = imported_packages_module + '.' + self.reponame + '.' + pkg_name diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 972ba9ccbbd..16664575028 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1714,7 +1714,7 @@ def spec(self): spec_repo = lst[-2] else: spec_name = self.token.value - (spec_repo, repodir) = spack.db.repo_for_package_name(spec_name) + spec_repo = 'gov.llnl.spack' self.check_identifier(spec_name) diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index b3ad8efec4b..55b3f0b18f9 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -34,7 +34,7 @@ import spack from spack.spec import Spec -from spack.packages import PackageDB +from spack.packages import PackageFinder from spack.directory_layout import YamlDirectoryLayout # number of packages to test (to reduce test time) @@ -123,7 +123,7 @@ def test_handle_unknown_package(self): information about installed packages' specs to uninstall or query them again if the package goes away. """ - mock_db = PackageDB(spack.mock_packages_path) + mock_db = PackageFinder(spack.mock_packages_path) not_in_mock = set.difference( set(spack.db.all_package_names()), @@ -145,8 +145,7 @@ def test_handle_unknown_package(self): self.layout.create_install_directory(spec) installed_specs[spec] = self.layout.path_for_spec(spec) - tmp = spack.db - spack.db = mock_db + spack.db.swap(mock_db) # Now check that even without the package files, we know # enough to read a spec from the spec file. @@ -161,7 +160,7 @@ def test_handle_unknown_package(self): self.assertTrue(spec.eq_dag(spec_from_file)) self.assertEqual(spec.dag_hash(), spec_from_file.dag_hash()) - spack.db = tmp + spack.db.swap(mock_db) def test_find(self): diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 00f81114af3..1f46d65790b 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -22,11 +22,12 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import sys import unittest import spack import spack.config -from spack.packages import PackageDB +from spack.packages import PackageFinder from spack.spec import Spec @@ -43,8 +44,8 @@ def initmock(self): # Use the mock packages database for these tests. This allows # us to set up contrived packages that don't interfere with # real ones. - self.real_db = spack.db - spack.db = PackageDB(spack.mock_packages_path) + self.db = PackageFinder(spack.mock_packages_path) + spack.db.swap(self.db) spack.config.clear_config_caches() self.real_scopes = spack.config.config_scopes @@ -55,7 +56,8 @@ def initmock(self): def cleanmock(self): """Restore the real packages path after any test.""" - spack.db = self.real_db + spack.db.swap(self.db) + spack.config.config_scopes = self.real_scopes spack.config.clear_config_caches() @@ -66,5 +68,3 @@ def setUp(self): def tearDown(self): self.cleanmock() - - diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index 6222e7b5f8b..70b5d6a478f 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -47,10 +47,10 @@ def test_get_all_packages(self): def test_get_all_mock_packages(self): """Get the mock packages once each too.""" - tmp = spack.db - spack.db = PackageDB(spack.mock_packages_path) + db = PackageFinder(spack.mock_packages_path) + spack.db.swap(db) self.check_db() - spack.db = tmp + spack.db.swap(db) def test_url_versions(self): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index a8183cf6a65..42bd91ec5ca 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -44,7 +44,8 @@ def test_package_name(self): def test_package_filename(self): - filename = spack.db.filename_for_package_name('mpich') + repo = spack.db.get_repo('gov.llnl.spack.mock') + filename = repo.filename_for_package_name('mpich') self.assertEqual(filename, join_path(spack.mock_packages_path, 'mpich', 'package.py')) @@ -54,7 +55,8 @@ def test_package_name(self): def test_nonexisting_package_filename(self): - filename = spack.db.filename_for_package_name('some-nonexisting-package') + repo = spack.db.get_repo('gov.llnl.spack.mock') + filename = repo.filename_for_package_name('some-nonexisting-package') self.assertEqual(filename, join_path(spack.mock_packages_path, 'some-nonexisting-package', 'package.py')) diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py index 782afbd4bbd..a7b6e2b4361 100644 --- a/lib/spack/spack/util/naming.py +++ b/lib/spack/spack/util/naming.py @@ -1,10 +1,14 @@ # Need this because of spack.util.string from __future__ import absolute_import import string +import itertools import re import spack +__all__ = ['mod_to_class', 'spack_module_to_python_module', 'valid_module_name', + 'validate_module_name', 'possible_spack_module_names'] + # Valid module names can contain '-' but can't start with it. _valid_module_re = r'^\w[\w-]*$' @@ -42,6 +46,33 @@ def mod_to_class(mod_name): return class_name +def spack_module_to_python_module(mod_name): + """Given a Spack module name, returns the name by which it can be + imported in Python. + """ + if re.match(r'[0-9]', mod_name): + mod_name = 'num' + mod_name + + return mod_name.replace('-', '_') + + +def possible_spack_module_names(python_mod_name): + """Given a Python module name, return a list of all possible spack module + names that could correspond to it.""" + mod_name = re.sub(r'^num(\d)', r'\1', python_mod_name) + + parts = re.split(r'(_)', mod_name) + options = [['_', '-']] * mod_name.count('_') + + results = [] + for subs in itertools.product(*options): + s = list(parts) + s[1::2] = subs + results.append(''.join(s)) + + return results + + def valid_module_name(mod_name): """Return whether the mod_name is valid for use in Spack.""" return bool(re.match(_valid_module_re, mod_name)) diff --git a/var/spack/mock_packages/_repo.yaml b/var/spack/mock_packages/_repo.yaml new file mode 100644 index 00000000000..b97b978de3a --- /dev/null +++ b/var/spack/mock_packages/_repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: gov.llnl.spack.mock diff --git a/var/spack/mock_packages/repo.yaml b/var/spack/mock_packages/repo.yaml deleted file mode 100644 index d0658960065..00000000000 --- a/var/spack/mock_packages/repo.yaml +++ /dev/null @@ -1,2 +0,0 @@ -repo: - namespace: mock diff --git a/var/spack/packages/repo.yaml b/var/spack/packages/_repo.yaml similarity index 100% rename from var/spack/packages/repo.yaml rename to var/spack/packages/_repo.yaml From 6dff42be0973c5949d836eab51cfdffb0eda0a69 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 17 Sep 2015 00:21:33 -0700 Subject: [PATCH 022/910] WIP for Matt's branch --- bin/spack | 4 +-- lib/spack/spack/directives.py | 4 +-- lib/spack/spack/package.py | 6 ++++ lib/spack/spack/packages.py | 18 +++++++---- lib/spack/spack/patch.py | 6 +++- lib/spack/spack/spec.py | 21 +++++-------- lib/spack/spack/test/mock_packages_test.py | 35 ++++++++++++++++------ lib/spack/spack/test/package_sanity.py | 2 +- lib/spack/spack/test/spec_dag.py | 12 ++++---- 9 files changed, 68 insertions(+), 40 deletions(-) diff --git a/bin/spack b/bin/spack index 5c042edd2d7..baf08d1481b 100755 --- a/bin/spack +++ b/bin/spack @@ -113,8 +113,8 @@ def main(): spack.spack_working_dir = working_dir if args.mock: - from spack.packages import PackageDB - spack.db = PackageDB(spack.mock_packages_path) + from spack.packages import PackageFinder + spack.db.swap(PackageFinder(spack.mock_packages_path)) # If the user asked for it, don't check ssl certs. if args.insecure: diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 9297d6dac36..aa2cfd28469 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -240,11 +240,11 @@ def patch(pkg, url_or_filename, level=1, when=None): when_spec = parse_anonymous_spec(when, pkg.name) if when_spec not in pkg.patches: - pkg.patches[when_spec] = [Patch(pkg.name, url_or_filename, level)] + pkg.patches[when_spec] = [Patch(pkg, pkg.name, url_or_filename, level)] else: # if this spec is identical to some other, then append this # patch to the existing list. - pkg.patches[when_spec].append(Patch(pkg.name, url_or_filename, level)) + pkg.patches[when_spec].append(Patch(pkg, pkg.name, url_or_filename, level)) @directive('variants') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 3507807373e..090349685b1 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -712,6 +712,12 @@ def do_patch(self): tty.msg("Patched %s" % self.name) + @property + def namespace(self): + namespace, dot, module = self.__module__.rpartition('.') + return namespace + + def do_fake_install(self): """Make a fake install directory contaiing a 'fake' file in bin.""" mkdirp(self.prefix.bin) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index df54b12324d..b21316ebf76 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -172,7 +172,7 @@ def all_packages(self): def providers_for(self, vpkg_name): - # TODO: USE MORE THAN FIRST REPO + # TODO: THIS IS WRONG; shoudl use more than first repo return self.repos[0].providers_for(vpkg_name) @@ -252,14 +252,22 @@ def load_module(self, fullname): return module - @_autospec - def get(self, spec, new=False): + def _find_repo_for_spec(self, spec): + """Find a repo that contains the supplied spec's package. + + Raises UnknownPackageErrorfor if not found. + """ for repo in self.repos: if spec.name in repo: - return repo.get(spec, new) + return repo raise UnknownPackageError(spec.name) + @_autospec + def get(self, spec, new=False): + return self._find_repo_for_spec(spec).get(spec, new) + + def get_repo(self, namespace): if namespace in self.by_namespace: repo = self.by_namespace[namespace] @@ -343,7 +351,7 @@ def get(self, spec, new=False): del self._instances[spec] if not spec in self._instances: - package_class = self.get_class_for_package_name(spec.name, spec.repo) + package_class = self.get_class_for_package_name(spec.name, spec.namespace) try: copy = spec.copy() self._instances[copy] = package_class(copy) diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index b1b6e077385..e89cf11b2ff 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -41,7 +41,11 @@ class Patch(object): """This class describes a patch to be applied to some expanded source code.""" - def __init__(self, pkg_name, path_or_url, level): + def __init__(self, pkg, pkg_name, path_or_url, level): + print pkg, pkg.name, type(pkg) + print "pkg:", dir(pkg.__module__) + print "NAMESPACE", pkg.namespace() + self.pkg_name = pkg_name self.path_or_url = path_or_url self.path = None diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 16664575028..0d49b1fa950 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -112,7 +112,6 @@ from spack.util.string import * from spack.util.prefix import Prefix from spack.virtual import ProviderIndex -from spack.repo_loader import imported_packages_module # Valid pattern for an identifier in Spack identifier_re = r'\w[\w-]*' @@ -413,7 +412,7 @@ def __init__(self, spec_like, *dep_like, **kwargs): self.dependencies = other.dependencies self.variants = other.variants self.variants.spec = self - self.repo = other.repo + self.namespace = other.namespace # Specs are by default not assumed to be normal, but in some # cases we've read them from a file want to assume normal. @@ -1357,7 +1356,7 @@ def _dup(self, other, **kwargs): self.dependencies = DependencyMap() self.variants = other.variants.copy() self.variants.spec = self - self.repo = other.repo + self.namespace = other.namespace # If we copy dependencies, preserve DAG structure in the new spec if kwargs.get('deps', True): @@ -1555,7 +1554,7 @@ def write(s, c): if c == '_': out.write(fmt % self.name) elif c == '.': - longname = '%s.%s.%s' % (imported_packages_module, self.repo, self.name) if self.repo else self.name + longname = '%s.%s.%s' % (self.namespace, self.name) if self.namespace else self.name out.write(fmt % longname) elif c == '@': if self.versions and self.versions != _any_version: @@ -1706,15 +1705,9 @@ def spec(self): """Parse a spec out of the input. If a spec is supplied, then initialize and return it instead of creating a new one.""" - spec_name = None - spec_repo = None - if self.token.value.startswith(imported_packages_module): - lst = self.token.value.split('.') - spec_name = lst[-1] - spec_repo = lst[-2] - else: - spec_name = self.token.value - spec_repo = 'gov.llnl.spack' + spec_namespace, dot, spec_name = self.token.value.rpartition('.') + if not spec_namespace: + spec_namespace = None self.check_identifier(spec_name) @@ -1727,7 +1720,7 @@ def spec(self): spec.compiler = None spec.dependents = DependencyMap() spec.dependencies = DependencyMap() - spec.repo = spec_repo + spec.namespace = spec_namespace spec._normal = False spec._concrete = False diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 1f46d65790b..071c21b7e08 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -31,14 +31,6 @@ from spack.spec import Spec -def set_pkg_dep(pkg, spec): - """Alters dependence information for a package. - Use this to mock up constraints. - """ - spec = Spec(spec) - spack.db.get(pkg).dependencies[spec.name] = { Spec(pkg) : spec } - - class MockPackagesTest(unittest.TestCase): def initmock(self): # Use the mock packages database for these tests. This allows @@ -53,14 +45,39 @@ def initmock(self): ('site', spack.mock_site_config), ('user', spack.mock_user_config)] + # Store changes to the package's dependencies so we can + # restore later. + self.saved_deps = {} + + + def set_pkg_dep(self, pkg_name, spec): + """Alters dependence information for a package. + + Adds a dependency on to pkg. + Use this to mock up constraints. + """ + spec = Spec(spec) + + # Save original dependencies before making any changes. + pkg = spack.db.get(pkg_name) + if pkg_name not in self.saved_deps: + self.saved_deps[pkg_name] = (pkg, pkg.dependencies.copy()) + + # Change dep spec + pkg.dependencies[spec.name] = { Spec(pkg_name) : spec } + def cleanmock(self): """Restore the real packages path after any test.""" spack.db.swap(self.db) - spack.config.config_scopes = self.real_scopes spack.config.clear_config_caches() + # Restore dependency changes that happened during the test + for pkg_name, (pkg, deps) in self.saved_deps.items(): + pkg.dependencies.clear() + pkg.dependencies.update(deps) + def setUp(self): self.initmock() diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index 70b5d6a478f..a5925ea066d 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -29,7 +29,7 @@ import spack import spack.url as url -from spack.packages import PackageDB +from spack.packages import PackageFinder class PackageSanityTest(unittest.TestCase): diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 549f829d3eb..a71026d1838 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -40,8 +40,8 @@ class SpecDagTest(MockPackagesTest): def test_conflicting_package_constraints(self): - set_pkg_dep('mpileaks', 'mpich@1.0') - set_pkg_dep('callpath', 'mpich@2.0') + self.set_pkg_dep('mpileaks', 'mpich@1.0') + self.set_pkg_dep('callpath', 'mpich@2.0') spec = Spec('mpileaks ^mpich ^callpath ^dyninst ^libelf ^libdwarf') @@ -223,25 +223,25 @@ def test_dependents_and_dependencies_are_correct(self): def test_unsatisfiable_version(self): - set_pkg_dep('mpileaks', 'mpich@1.0') + self.set_pkg_dep('mpileaks', 'mpich@1.0') spec = Spec('mpileaks ^mpich@2.0 ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableVersionSpecError, spec.normalize) def test_unsatisfiable_compiler(self): - set_pkg_dep('mpileaks', 'mpich%gcc') + self.set_pkg_dep('mpileaks', 'mpich%gcc') spec = Spec('mpileaks ^mpich%intel ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableCompilerSpecError, spec.normalize) def test_unsatisfiable_compiler_version(self): - set_pkg_dep('mpileaks', 'mpich%gcc@4.6') + self.set_pkg_dep('mpileaks', 'mpich%gcc@4.6') spec = Spec('mpileaks ^mpich%gcc@4.5 ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableCompilerSpecError, spec.normalize) def test_unsatisfiable_architecture(self): - set_pkg_dep('mpileaks', 'mpich=bgqos_0') + self.set_pkg_dep('mpileaks', 'mpich=bgqos_0') spec = Spec('mpileaks ^mpich=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) From 3691201b4f44da9fc373c8368cc6efaeae7b6387 Mon Sep 17 00:00:00 2001 From: Justin Too Date: Mon, 21 Sep 2015 11:03:49 -0700 Subject: [PATCH 023/910] (Package) Add Spot C++ library (v1.99.3) --- var/spack/packages/spot/package.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 var/spack/packages/spot/package.py diff --git a/var/spack/packages/spot/package.py b/var/spack/packages/spot/package.py new file mode 100644 index 00000000000..9e539277ae6 --- /dev/null +++ b/var/spack/packages/spot/package.py @@ -0,0 +1,18 @@ +from spack import * +import os + +class Spot(Package): + """Spot is a C++11 library for omega-automata manipulation and model checking.""" + homepage = "https://spot.lrde.epita.fr/index.html" + url = "http://www.lrde.epita.fr/dload/spot/spot-1.99.3.tar.gz" + + version('1.99.3', 'd53adcb2d0fe7c69f45d4e595a58254e') + + #depends_on("gcc@4.8:") + depends_on("python@3.2:") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") From 59f89dd3bee0ee2b668554c537d1d18404108ec4 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Fri, 29 May 2015 11:04:57 -0700 Subject: [PATCH 024/910] Allow long names in format string variables --- lib/spack/spack/spec.py | 71 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index e1fbb844234..3b5d16c7a75 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1503,14 +1503,28 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): in the format string. The format strings you can provide are:: $_ Package name - $@ Version - $% Compiler - $%@ Compiler & compiler version - $+ Options - $= Architecture - $# 7-char prefix of DAG hash + $@ Version with '@' prefix + $% Compiler with '%' prefix + $%@ Compiler with '%' prefix & compiler version with '@' prefix + $+ Options + $= Architecture with '=' prefix + $# 7-char prefix of DAG hash with '-' prefix $$ $ + You can also use full-string versions, which leave off the prefixes: + + ${PACKAGE} Package name + ${VERSION} Version + ${COMPILER} Full compiler string + ${COMPILERNAME} Compiler name + ${COMPILERVER} Compiler version + ${OPTIONS} Options + ${ARCHITECTURE} Architecture + ${SHA1} Dependencies 8-char sha1 prefix + + ${SPACK_ROOT} The spack root directory + ${SPACK_INSTALL} The default spack install directory, ${SPACK_PREFIX}/opt + Optionally you can provide a width, e.g. $20_ for a 20-wide name. Like printf, you can provide '-' for left justification, e.g. $-20_ for a left-justified name. @@ -1526,7 +1540,8 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): color = kwargs.get('color', False) length = len(format_string) out = StringIO() - escape = compiler = False + named = escape = compiler = False + named_str = fmt = '' def write(s, c): if color: @@ -1566,9 +1581,12 @@ def write(s, c): elif c == '#': out.write('-' + fmt % (self.dag_hash(7))) elif c == '$': - if fmt != '': + if fmt != '%s': raise ValueError("Can't use format width with $$.") out.write('$') + elif c == '{': + named = True + named_str = '' escape = False elif compiler: @@ -1582,6 +1600,43 @@ def write(s, c): out.write(c) compiler = False + elif named: + if not c == '}': + if i == length - 1: + raise ValueError("Error: unterminated ${ in format: '%s'" + % format_string) + named_str += c + continue; + if named_str == 'PACKAGE': + write(fmt % self.name, '@') + if named_str == 'VERSION': + if self.versions and self.versions != _any_version: + write(fmt % str(self.versions), '@') + elif named_str == 'COMPILER': + if self.compiler: + write(fmt % self.compiler, '%') + elif named_str == 'COMPILERNAME': + if self.compiler: + write(fmt % self.compiler.name, '%') + elif named_str == 'COMPILERVER': + if self.compiler: + write(fmt % self.compiler.versions, '%') + elif named_str == 'OPTIONS': + if self.variants: + write(fmt % str(self.variants), '+') + elif named_str == 'ARCHITECTURE': + if self.architecture: + write(fmt % str(self.architecture), '=') + elif named_str == 'SHA1': + if self.dependencies: + out.write(fmt % str(self.dep_hash(8))) + elif named_str == 'SPACK_ROOT': + out.write(fmt % spack.prefix) + elif named_str == 'SPACK_INSTALL': + out.write(fmt % spack.install_path) + + named = False + elif c == '$': escape = True if i == length - 1: From b5c597b31864826050162b358998e240761c5d7e Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Fri, 29 May 2015 12:19:57 -0700 Subject: [PATCH 025/910] Allow specs to be sorted based on preferred packages, versions, compilers, variants and dependencies. --- lib/spack/spack/__init__.py | 8 ++ lib/spack/spack/config.py | 12 +- lib/spack/spack/preferred_packages.py | 172 ++++++++++++++++++++++++++ lib/spack/spack/spec.py | 34 +++++ 4 files changed, 222 insertions(+), 4 deletions(-) create mode 100644 lib/spack/spack/preferred_packages.py diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index caa09eb6e00..bd8478fb98e 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -69,6 +69,14 @@ from spack.directory_layout import YamlDirectoryLayout install_layout = YamlDirectoryLayout(install_path) +# +# This controls how packages are sorted when trying to choose +# the most preferred package. More preferred packages are sorted +# first. +# +from spack.preferred_packages import PreferredPackages +pkgsort = PreferredPackages() + # # This controls how things are concretized in spack. # Replace it with a subclass if you want different diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3e91958c2c2..dbe225960a8 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -89,8 +89,8 @@ import os import exceptions import sys +import copy -from external.ordereddict import OrderedDict from llnl.util.lang import memoized import spack.error @@ -114,8 +114,7 @@ def __init__(self, n, f, m): _ConfigCategory('compilers', 'compilers.yaml', True) _ConfigCategory('mirrors', 'mirrors.yaml', True) -_ConfigCategory('view', 'views.yaml', True) -_ConfigCategory('order', 'orders.yaml', True) +_ConfigCategory('preferred', 'preferred.yaml', True) """Names of scopes and their corresponding configuration files.""" config_scopes = [('site', os.path.join(spack.etc_path, 'spack')), @@ -156,7 +155,7 @@ def _merge_dicts(d1, d2): """Recursively merges two configuration trees, with entries in d2 taking precedence over d1""" if not d1: - return d2.copy() + return copy.copy(d2) if not d2: return d1 @@ -230,6 +229,11 @@ def get_mirror_config(): return get_config('mirrors') +def get_preferred_config(): + """Get the preferred configuration from config files""" + return get_config('preferred') + + def get_config_scope_dirname(scope): """For a scope return the config directory""" global config_scopes diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py new file mode 100644 index 00000000000..248508fe809 --- /dev/null +++ b/lib/spack/spack/preferred_packages.py @@ -0,0 +1,172 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import spack +from spack.version import * + +class PreferredPackages(object): + _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] }, #Arbitrary, but consistent + + def __init__(self): + self.preferred = spack.config.get_preferred_config() + self._spec_for_pkgname_cache = {} + + #Given a package name, sort component (e.g, version, compiler, ...), and + # a second_key (used by providers), return the list + def _order_for_package(self, pkgname, component, second_key): + pkglist = [pkgname] + pkglist.append('all') + for pkg in pkglist: + if not pkg in self.preferred: + continue + orders = self.preferred[pkg] + if not type(orders) is dict: + continue + if not component in orders: + continue + order = orders[component] + if type(order) is dict: + if not second_key in order: + continue; + order = order[second_key] + if not type(order) is str: + tty.die('Expected version list in preferred config, but got %s' % str(order)) + order_list = order.split(',') + return [s.strip() for s in order_list] + return [] + + + # A generic sorting function. Given a package name and sort + # component, return less-than-0, 0, or greater-than-0 if + # a is respectively less-than, equal to, or greater than b. + def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key): + orderlist = self._order_for_package(pkgname, component, second_key) + a_in_list = str(a) in orderlist + b_in_list = str(b) in orderlist + if a_in_list and not b_in_list: + return -1 + elif b_in_list and not a_in_list: + return 1 + + cmp_a = None + cmp_b = None + reverse = None + if not a_in_list and not b_in_list: + cmp_a = a + cmp_b = b + reverse = -1 if reverse_natural_compare else 1 + else: + cmp_a = orderlist.index(str(a)) + cmp_b = orderlist.index(str(b)) + reverse = 1 + + if cmp_a < cmp_b: + return -1 * reverse + elif cmp_a > cmp_b: + return 1 * reverse + else: + return 0 + + + # A sorting function for specs. Similar to component_compare, but + # a and b are considered to match entries in the sorting list if they + # satisfy the list component. + def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key): + specs = self._spec_for_pkgname(pkgname, component, second_key) + a_index = None + b_index = None + reverse = -1 if reverse_natural_compare else 1 + for i, cspec in enumerate(specs): + if a_index == None and cspec.satisfies(a): + a_index = i + if b_index: + break + if b_index == None and cspec.satisfies(b): + b_index = i + if a_index: + break + + if a_index != None and b_index == None: return -1 + elif a_index == None and b_index != None: return 1 + elif a_index != None and b_index == a_index: return -1 * cmp(a, b) + elif a_index != None and b_index != None and a_index != b_index: return cmp(a_index, b_index) + elif a < b: return 1 * reverse + elif b < a: return -1 * reverse + else: return 0 + + + # Given a sort order specified by the pkgname/component/second_key, return + # a list of CompilerSpecs, VersionLists, or Specs for that sorting list. + def _spec_for_pkgname(self, pkgname, component, second_key): + key = (pkgname, component, second_key) + if not key in self._spec_for_pkgname_cache: + pkglist = self._order_for_package(pkgname, component, second_key) + if not pkglist: + if component in self._default_order: + pkglist = self._default_order[component] + if component == 'compiler': + self._spec_for_pkgname_cache[key] = [spack.spec.CompilerSpec(s) for s in pkglist] + elif component == 'version': + self._spec_for_pkgname_cache[key] = [VersionList(s) for s in pkglist] + else: + self._spec_for_pkgname_cache[key] = [spack.spec.Spec(s) for s in pkglist] + return self._spec_for_pkgname_cache[key] + + + def provider_compare(self, pkgname, provider_str, a, b): + """Return less-than-0, 0, or greater than 0 if a is respecively less-than, equal-to, or + greater-than b. A and b are possible implementations of provider_str. + One provider is less-than another if it is preferred over the other. + For example, provider_compare('scorep', 'mpi', 'mvapich', 'openmpi') would return -1 if + mvapich should be preferred over openmpi for scorep.""" + return self._spec_compare(pkgname, 'providers', a, b, False, provider_str) + + + def version_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if version a of pkgname is + respecively less-than, equal-to, or greater-than version b of pkgname. + One version is less-than another if it is preferred over the other.""" + return self._spec_compare(pkgname, 'version', a, b, False, None) + + + def variant_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if variant a of pkgname is + respecively less-than, equal-to, or greater-than variant b of pkgname. + One variant is less-than another if it is preferred over the other.""" + return self._component_compare(pkgname, 'variant', a, b, False, None) + + + def architecture_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is + respecively less-than, equal-to, or greater-than architecture b of pkgname. + One architecture is less-than another if it is preferred over the other.""" + return self._component_compare(pkgname, 'architecture', a, b, False, None) + + + def compiler_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if compiler a of pkgname is + respecively less-than, equal-to, or greater-than compiler b of pkgname. + One compiler is less-than another if it is preferred over the other.""" + return self._spec_compare(pkgname, 'compiler', a, b, False, None) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 3b5d16c7a75..83b1416e36d 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1653,6 +1653,40 @@ def dep_string(self): return ''.join("^" + dep.format() for dep in self.sorted_deps()) + def __cmp__(self, other): + #Package name sort order is not configurable, always goes alphabetical + if self.name != other.name: + return cmp(self.name, other.name) + + #Package version is second in compare order + pkgname = self.name + if self.versions != other.versions: + return spack.pkgsort.version_compare(pkgname, + self.versions, other.versions) + + #Compiler is third + if self.compiler != other.compiler: + return spack.pkgsort.compiler_compare(pkgname, + self.compiler, other.compiler) + + #Variants + if self.variants != other.variants: + return spack.pkgsort.variant_compare(pkgname, + self.variants, other.variants) + + #Architecture + if self.architecture != other.architecture: + return spack.pkgsort.architecture_compare(pkgname, + self.architecture, other.architecture) + + #Dependency is not configurable + if self.dep_hash() != other.dep_hash(): + return -1 if self.dep_hash() < other.dep_hash() else 1 + + #Equal specs + return 0 + + def __str__(self): return self.format() + self.dep_string() From 8d7b7e5d5dadcec9b997b94d95898a4134e122b2 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Fri, 29 May 2015 12:20:32 -0700 Subject: [PATCH 026/910] Use preferred package rules when concretize'ing specs --- lib/spack/spack/concretize.py | 68 +++++++++++++++++++---------------- lib/spack/spack/spec.py | 2 +- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 66002492cb8..0f258c90968 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -38,6 +38,7 @@ import spack.architecture import spack.error from spack.version import * +from functools import partial @@ -49,8 +50,8 @@ class DefaultConcretizer(object): def concretize_version(self, spec): """If the spec is already concrete, return. Otherwise take - the most recent available version, and default to the package's - version if there are no avaialble versions. + the preferred version from spackconfig, and default to the package's + version if there are no available versions. TODO: In many cases we probably want to look for installed versions of each package and use an installed version @@ -68,12 +69,14 @@ def concretize_version(self, spec): # If there are known available versions, return the most recent # version that satisfies the spec pkg = spec.package + cmp_versions = partial(spack.pkgsort.version_compare, spec.name) valid_versions = sorted( [v for v in pkg.versions - if any(v.satisfies(sv) for sv in spec.versions)]) + if any(v.satisfies(sv) for sv in spec.versions)], + cmp=cmp_versions) if valid_versions: - spec.versions = ver([valid_versions[-1]]) + spec.versions = ver([valid_versions[0]]) else: # We don't know of any SAFE versions that match the given # spec. Grab the spec's versions and grab the highest @@ -138,10 +141,10 @@ def concretize_compiler(self, spec): """If the spec already has a compiler, we're done. If not, then take the compiler used for the nearest ancestor with a compiler spec and use that. If the ancestor's compiler is not - concrete, then give it a valid version. If there is no - ancestor with a compiler, use the system default compiler. + concrete, then used the preferred compiler as specified in + spackconfig. - Intuition: Use the system default if no package that depends on + Intuition: Use the spackconfig default if no package that depends on this one has a strict compiler requirement. Otherwise, try to build with the compiler that will be used by libraries that link to this one, to maximize compatibility. @@ -153,40 +156,43 @@ def concretize_compiler(self, spec): spec.compiler in all_compilers): return False - try: - nearest = next(p for p in spec.traverse(direction='parents') - if p.compiler is not None).compiler - - if not nearest in all_compilers: - # Take the newest compiler that saisfies the spec - matches = sorted(spack.compilers.find(nearest)) - if not matches: - raise UnavailableCompilerVersionError(nearest) - - # copy concrete version into nearest spec - nearest.versions = matches[-1].versions.copy() - assert(nearest.concrete) - - spec.compiler = nearest.copy() - - except StopIteration: - spec.compiler = spack.compilers.default_compiler().copy() - + # Find the parent spec that has a compiler, or the root if none do + parent_spec = next(p for p in spec.traverse(direction='parents') + if p.compiler is not None or not p.dependents) + parent_compiler = parent_spec.compiler + assert(parent_spec) + + # Check if the compiler is already fully specified + if parent_compiler in all_compilers: + spec.compiler = parent_compiler.copy() + return True + + # Filter the compilers into a sorted list based on the compiler_order from spackconfig + compiler_list = all_compilers if not parent_compiler else spack.compilers.find(parent_compiler) + cmp_compilers = partial(spack.pkgsort.compiler_compare, parent_spec.name) + matches = sorted(compiler_list, cmp=cmp_compilers) + if not matches: + raise UnavailableCompilerVersionError(parent_compiler) + + # copy concrete version into parent_compiler + spec.compiler = matches[0].copy() + assert(spec.compiler.concrete) return True # things changed. - def choose_provider(self, spec, providers): + def choose_provider(self, package_spec, spec, providers): """This is invoked for virtual specs. Given a spec with a virtual name, say "mpi", and a list of specs of possible providers of that spec, select a provider and return it. """ assert(spec.virtual) assert(providers) + + provider_cmp = partial(spack.pkgsort.provider_compare, package_spec.name, spec.name) + sorted_providers = sorted(providers, cmp=provider_cmp) + first_key = sorted_providers[0] - index = spack.spec.index_specs(providers) - first_key = sorted(index.keys())[0] - latest_version = sorted(index[first_key])[-1] - return latest_version + return first_key class UnavailableCompilerVersionError(spack.error.SpackError): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 83b1416e36d..41496b0e9d5 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -796,7 +796,7 @@ def _expand_virtual_packages(self): for spec in virtuals: providers = spack.db.providers_for(spec) - concrete = spack.concretizer.choose_provider(spec, providers) + concrete = spack.concretizer.choose_provider(self, spec, providers) concrete = concrete.copy() spec._replace_with(concrete) changed = True From ee68a76a193890231d6df7fa7934d42e3708540b Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Fri, 29 May 2015 14:57:24 -0700 Subject: [PATCH 027/910] Bug fixes from testing spack preferred packages --- lib/spack/spack/preferred_packages.py | 7 +++---- var/spack/mock_packages/mpich/package.py | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index 248508fe809..bc5271f693d 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -112,9 +112,8 @@ def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, secon elif a_index == None and b_index != None: return 1 elif a_index != None and b_index == a_index: return -1 * cmp(a, b) elif a_index != None and b_index != None and a_index != b_index: return cmp(a_index, b_index) - elif a < b: return 1 * reverse - elif b < a: return -1 * reverse - else: return 0 + else: return cmp(a, b) * reverse + # Given a sort order specified by the pkgname/component/second_key, return @@ -148,7 +147,7 @@ def version_compare(self, pkgname, a, b): """Return less-than-0, 0, or greater than 0 if version a of pkgname is respecively less-than, equal-to, or greater-than version b of pkgname. One version is less-than another if it is preferred over the other.""" - return self._spec_compare(pkgname, 'version', a, b, False, None) + return self._spec_compare(pkgname, 'version', a, b, True, None) def variant_compare(self, pkgname, a, b): diff --git a/var/spack/mock_packages/mpich/package.py b/var/spack/mock_packages/mpich/package.py index f77d3efc5d1..e4110ad5301 100644 --- a/var/spack/mock_packages/mpich/package.py +++ b/var/spack/mock_packages/mpich/package.py @@ -38,6 +38,7 @@ class Mpich(Package): version('3.0.2', 'foobarbaz') version('3.0.1', 'foobarbaz') version('3.0', 'foobarbaz') + version('1.0', 'foobarbas') provides('mpi@:3', when='@3:') provides('mpi@:1', when='@:1') From 987cd9e78f99aa6ee6f3a48a4b8f556a68cb2965 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Tue, 7 Jul 2015 16:27:13 -0700 Subject: [PATCH 028/910] Update docs for YAML configuration files and preferred concretization --- lib/spack/docs/basic_usage.rst | 37 +++++++----- lib/spack/docs/mirrors.rst | 9 ++- lib/spack/docs/packaging_guide.rst | 66 ++++++++++++++++++++- lib/spack/docs/site_configuration.rst | 83 --------------------------- 4 files changed, 91 insertions(+), 104 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 0578f0c8db6..5d5438220cc 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -357,7 +357,7 @@ Spack, you can simply run ``spack compiler add`` with the path to where the compiler is installed. For example:: $ spack compiler add /usr/local/tools/ic-13.0.079 - ==> Added 1 new compiler to /Users/gamblin2/.spackconfig + ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml intel@13.0.079 Or you can run ``spack compiler add`` with no arguments to force @@ -367,7 +367,7 @@ installed, but you know that new compilers have been added to your $ module load gcc-4.9.0 $ spack compiler add - ==> Added 1 new compiler to /Users/gamblin2/.spackconfig + ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml gcc@4.9.0 This loads the environment module for gcc-4.9.0 to get it into the @@ -398,27 +398,34 @@ Manual compiler configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If auto-detection fails, you can manually configure a compiler by -editing your ``~/.spackconfig`` file. You can do this by running -``spack config edit``, which will open the file in your ``$EDITOR``. +editing your ``~/.spack/compilers.yaml`` file. You can do this by running +``spack config edit compilers``, which will open the file in your ``$EDITOR``. Each compiler configuration in the file looks like this:: ... - [compiler "intel@15.0.0"] - cc = /usr/local/bin/icc-15.0.024-beta - cxx = /usr/local/bin/icpc-15.0.024-beta - f77 = /usr/local/bin/ifort-15.0.024-beta - fc = /usr/local/bin/ifort-15.0.024-beta - ... + chaos_5_x86_64_ib: + ... + intel@15.0.0: + cc: /usr/local/bin/icc-15.0.024-beta + cxx: /usr/local/bin/icpc-15.0.024-beta + f77: /usr/local/bin/ifort-15.0.024-beta + fc: /usr/local/bin/ifort-15.0.024-beta + ... + +The chaos_5_x86_64_ib string is an architecture string, and multiple +compilers can be listed underneath an architecture. The architecture +string may be replaced with the string 'all' to signify compilers that +work on all architectures. For compilers, like ``clang``, that do not support Fortran, put ``None`` for ``f77`` and ``fc``:: - [compiler "clang@3.3svn"] - cc = /usr/bin/clang - cxx = /usr/bin/clang++ - f77 = None - fc = None + clang@3.3svn: + cc: /usr/bin/clang + cxx: /usr/bin/clang++ + f77: None + fc: None Once you save the file, the configured compilers will show up in the list displayed by ``spack compilers``. diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index d732a3dd54f..7581a0e9ede 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -205,12 +205,11 @@ And, if you want to remove a mirror, just remove it by name:: Mirror precedence ---------------------------- -Adding a mirror really just adds a section in ``~/.spackconfig``:: +Adding a mirror really just adds a section in ``~/.spack/mirrors.yaml``:: - [mirror "local_filesystem"] - url = file:///Users/gamblin2/spack-mirror-2014-06-24 - [mirror "remote_server"] - url = https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 + mirrors: + - local_filesystem: file:///Users/gamblin2/spack-mirror-2014-06-24 + - remote_server: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 If you want to change the order in which mirrors are searched for packages, you can edit this file and reorder the sections. Spack will diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 59ba63fa355..5094f739c45 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -632,7 +632,7 @@ Default revision instead. Revisions - Add ``hg`` and ``revision``parameters: + Add ``hg`` and ``revision`` parameters: .. code-block:: python @@ -1524,6 +1524,70 @@ This is useful when you want to know exactly what Spack will do when you ask for a particular spec. +``Concretization Policies`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A user may have certain perferrences for how packages should +be concretized on their system. For example, one user may prefer packages +built with OpenMPI and the Intel compiler. Another user may prefer +packages be built with MVAPICH and GCC. + +Spack's ``preferred`` configuration can be used to set defaults for sites or users. +Spack uses this configuration to make decisions about which compilers, package +versions, depends_on, and variants it should prefer during concretization. + +The preferred configuration can be controlled by editing the +``~/.spack/preferred.yaml`` file for user configuations, or the + + +Here's an example preferred.yaml file: + +.. code-block:: sh + + preferred: + dyninst: + compiler: gcc@4.9 + variants: +debug + gperftools: + version: 2.2, 2.4, 2.3 + all: + compiler: gcc@4.4.7, gcc@4.6:, intel, clang, pgi + providers: + mpi: mvapich, mpich, openmpi + +At a high level, this example is specifying how packages should be +concretized. The dyninst package should prefer using gcc 4.9 and +be built with debug options. The gperftools package should prefer version +2.2 over 2.4. Every package on the system should prefer mvapich for +its MPI and gcc 4.4.7 (except for Dyninst, which perfers gcc 4.9). +These options are used to fill in implicit defaults. Any of them can be overwritten +on the command line if explicitly requested. + +Each preferred.yaml file begin with the string ``preferred:`` and +each subsequent entry is indented underneath it. The next layer contains +package names or the special string ``all`` (which applies to +every package). Underneath each package name is +one or more components: ``compiler``, ``variants``, ``version``, +or ``providers``. Each component has an ordered list of spec +``constraints``, with earlier entries in the list being prefered over +latter entries. + +Sometimes a package installation may have constraints that forbid +the first concretization rule, in which case Spack will use the first +legal concretization rule. Going back to the example, if a user +requests gperftools 2.3 or latter, then Spack will install version 2.4 +as the 2.4 version of gperftools is preferred over 2.3. + +An explicit concretization rule in the preferred section will always +take preference over unlisted concretizations. In the above example, +xlc isn't listed in the compiler list. Every listed compiler from +gcc to pgi will thus be preferred over the xlc compiler. + +The syntax for the ``providers`` section differs slightly from other +concretization rules. A provider lists a value that packages may +``depend_on`` (e.g, mpi) and a list of rules for fulfilling that +dependency. + .. _install-method: Implementing the ``install`` method diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index b03df29573b..1e6740a434f 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -54,89 +54,6 @@ more elements to the list to indicate where your own site's temporary directory is. -.. _concretization-policies: - -Concretization policies ----------------------------- - -When a user asks for a package like ``mpileaks`` to be installed, -Spack has to make decisions like what version should be installed, -what compiler to use, and how its dependencies should be configured. -This process is called *concretization*, and it's covered in detail in -:ref:`its own section `. - -The default concretization policies are in the -:py:mod:`spack.concretize` module, specifically in the -:py:class:`spack.concretize.DefaultConcretizer` class. These are the -important methods used in the concretization process: - -* :py:meth:`concretize_version(self, spec) ` -* :py:meth:`concretize_architecture(self, spec) ` -* :py:meth:`concretize_compiler(self, spec) ` -* :py:meth:`choose_provider(self, spec, providers) ` - -The first three take a :py:class:`Spec ` object and -modify it by adding constraints for the version. For example, if the -input spec had a version range like `1.0:5.0.3`, then the -``concretize_version`` method should set the spec's version to a -*single* version in that range. Likewise, ``concretize_architecture`` -selects an architecture when the input spec does not have one, and -``concretize_compiler`` needs to set both a concrete compiler and a -concrete compiler version. - -``choose_provider()`` affects how concrete implementations are chosen -based on a virtual dependency spec. The input spec is some virtual -dependency and the ``providers`` index is a :py:class:`ProviderIndex -` object. The ``ProviderIndex`` maps -the virtual spec to specs for possible implementations, and -``choose_provider()`` should simply choose one of these. The -``concretize_*`` methods will be called on the chosen implementation -later, so there is no need to fully concretize the spec when returning -it. - -The ``DefaultConcretizer`` is intended to provide sensible defaults -for each policy, but there are certain choices that it can't know -about. For example, one site might prefer ``OpenMPI`` over ``MPICH``, -or another might prefer an old version of some packages. These types -of special cases can be integrated with custom concretizers. - -Writing a custom concretizer -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To write your own concretizer, you need only subclass -``DefaultConcretizer`` and override the methods you want to change. -For example, you might write a class like this to change *only* the -``concretize_version()`` behavior: - -.. code-block:: python - - from spack.concretize import DefaultConcretizer - - class MyConcretizer(DefaultConcretizer): - def concretize_version(self, spec): - # implement custom logic here. - -Once you have written your custom concretizer, you can make Spack use -it by editing ``globals.py``. Find this part of the file: - -.. code-block:: python - - # - # This controls how things are concretized in spack. - # Replace it with a subclass if you want different - # policies. - # - concretizer = DefaultConcretizer() - -Set concretizer to *your own* class instead of the default: - -.. code-block:: python - - concretizer = MyConcretizer() - -The next time you run Spack, your changes should take effect. - - Profiling ~~~~~~~~~~~~~~~~~~~~~ From 53cde110b1d25f8fcf89d7c26ef02bb073cb6a29 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Tue, 7 Jul 2015 16:27:46 -0700 Subject: [PATCH 029/910] Update Spack mirror command to match docs --- lib/spack/spack/cmd/mirror.py | 12 ++++++------ lib/spack/spack/stage.py | 5 ++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 2356170a9ac..baf64d30fc8 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -75,8 +75,8 @@ def mirror_add(args): if url.startswith('/'): url = 'file://' + url - mirror_dict = { args.name : url } - spack.config.add_to_mirror_config({ args.name : url }) + newmirror = [ { args.name : url } ] + spack.config.add_to_mirror_config(newmirror) def mirror_remove(args): @@ -90,15 +90,15 @@ def mirror_remove(args): def mirror_list(args): """Print out available mirrors to the console.""" - sec_names = spack.config.get_mirror_config() - if not sec_names: + mirrors = spack.config.get_mirror_config() + if not mirrors: tty.msg("No mirrors configured.") return - max_len = max(len(s) for s in sec_names) + max_len = max(len(name) for name,path in mirrors) fmt = "%%-%ds%%s" % (max_len + 4) - for name, val in sec_names.iteritems(): + for name, val in mirrors: print fmt % (name, val) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 008c5f04298..c70c7a84a4e 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -26,6 +26,7 @@ import re import shutil import tempfile +import sys import llnl.util.tty as tty from llnl.util.filesystem import * @@ -344,9 +345,7 @@ def destroy(self): def _get_mirrors(): """Get mirrors from spack configuration.""" - config = spack.config.get_mirror_config() - return [val for name, val in config.iteritems()] - + return [path for name, path in spack.config.get_mirror_config()] def ensure_access(file=spack.stage_path): From 53d70fff0121a05fb21c02363570f81573bbeffa Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Tue, 7 Jul 2015 16:28:51 -0700 Subject: [PATCH 030/910] Fix type error with YAML config when merging lists from different configs. --- lib/spack/spack/config.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index dbe225960a8..f3526b19fa3 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -98,6 +98,7 @@ from external.yaml.error import MarkedYAMLError import llnl.util.tty as tty from llnl.util.filesystem import mkdirp +import copy _config_sections = {} class _ConfigCategory: @@ -159,21 +160,19 @@ def _merge_dicts(d1, d2): if not d2: return d1 - for key2, val2 in d2.iteritems(): - if not key2 in d1: - d1[key2] = val2 - continue - val1 = d1[key2] - if isinstance(val1, dict) and isinstance(val2, dict): - d1[key2] = _merge_dicts(val1, val2) - continue - if isinstance(val1, list) and isinstance(val2, list): - val1.extend(val2) - seen = set() - d1[key2] = [ x for x in val1 if not (x in seen or seen.add(x)) ] - continue - d1[key2] = val2 - return d1 + if (type(d1) is list) and (type(d2) is list): + d1.extend(d2) + return d1 + + if (type(d1) is dict) and (type(d2) is dict): + for key2, val2 in d2.iteritems(): + if not key2 in d1: + d1[key2] = val2 + else: + d1[key2] = _merge_dicts(d1[key2], val2) + return d1 + + return d2 def get_config(category_name): @@ -225,8 +224,8 @@ def get_compilers_config(arch=None): def get_mirror_config(): - """Get the mirror configuration from config files""" - return get_config('mirrors') + """Get the mirror configuration from config files as a list of name/location tuples""" + return [x.items()[0] for x in get_config('mirrors')] def get_preferred_config(): From 650c9d4e36c6a58cf6bca0e6abd580ee54d8e175 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Wed, 16 Sep 2015 10:56:11 -0700 Subject: [PATCH 031/910] Allow spack to build against external non-spack-installed packages. --- lib/spack/spack/__init__.py | 6 ++ lib/spack/spack/abi.py | 128 ++++++++++++++++++++++ lib/spack/spack/concretize.py | 140 ++++++++++++++++++++++--- lib/spack/spack/config.py | 59 ++++++++++- lib/spack/spack/directory_layout.py | 8 ++ lib/spack/spack/package.py | 3 + lib/spack/spack/preferred_packages.py | 10 +- lib/spack/spack/spec.py | 58 +++++----- var/spack/packages/mpich/package.py | 1 + var/spack/packages/mvapich2/package.py | 17 ++- 10 files changed, 387 insertions(+), 43 deletions(-) create mode 100644 lib/spack/spack/abi.py diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index bd8478fb98e..5783005b5b8 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -77,6 +77,12 @@ from spack.preferred_packages import PreferredPackages pkgsort = PreferredPackages() +# +# This tests ABI compatibility between packages +# +from spack.abi import ABI +abi = ABI() + # # This controls how things are concretized in spack. # Replace it with a subclass if you want different diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py new file mode 100644 index 00000000000..f0a997703cd --- /dev/null +++ b/lib/spack/spack/abi.py @@ -0,0 +1,128 @@ +############################################################################## +# Copyright (c) 2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import os +import spack +import spack.spec +from spack.spec import CompilerSpec +from spack.util.executable import Executable, ProcessError +from llnl.util.lang import memoized + +class ABI(object): + """This class provides methods to test ABI compatibility between specs. + The current implementation is rather rough and could be improved.""" + + def architecture_compatible(self, parent, child): + """Returns true iff the parent and child specs have ABI compatible architectures.""" + return not parent.architecture or not child.architecture or parent.architecture == child.architecture + + + @memoized + def _gcc_get_libstdcxx_version(self, version): + """Returns gcc ABI compatibility info by getting the library version of + a compiler's libstdc++.so or libgcc_s.so""" + spec = CompilerSpec("gcc", version) + compilers = spack.compilers.compilers_for_spec(spec) + if not compilers: + return None + compiler = compilers[0] + rungcc = None + libname = None + output = None + if compiler.cxx: + rungcc = Executable(compiler.cxx) + libname = "libstdc++.so" + elif compiler.cc: + rungcc = Executable(compiler.cc) + libname = "libgcc_s.so" + else: + return None + try: + output = rungcc("--print-file-name=%s" % libname, return_output=True) + except ProcessError, e: + return None + if not output: + return None + libpath = os.readlink(output.strip()) + if not libpath: + return None + return os.path.basename(libpath) + + + @memoized + def _gcc_compiler_compare(self, pversion, cversion): + """Returns true iff the gcc version pversion and cversion + are ABI compatible.""" + plib = self._gcc_get_libstdcxx_version(pversion) + clib = self._gcc_get_libstdcxx_version(cversion) + if not plib or not clib: + return False + return plib == clib + + + def _intel_compiler_compare(self, pversion, cversion): + """Returns true iff the intel version pversion and cversion + are ABI compatible""" + + #Test major and minor versions. Ignore build version. + if (len(pversion.version) < 2 or len(cversion.version) < 2): + return False + return (pversion.version[0] == cversion.version[0]) and \ + (pversion.version[1] == cversion.version[1]) + + + def compiler_compatible(self, parent, child, **kwargs): + """Returns true iff the compilers for parent and child specs are ABI compatible""" + if not parent.compiler or not child.compiler: + return True + + if parent.compiler.name != child.compiler.name: + #Different compiler families are assumed ABI incompatible + return False + + if kwargs.get('loose', False): + return True + + for pversion in parent.compiler.versions: + for cversion in child.compiler.versions: + #For a few compilers use specialized comparisons. Otherwise + # match on version match. + if pversion.satisfies(cversion): + return True + elif parent.compiler.name == "gcc" and \ + self._gcc_compiler_compare(pversion, cversion): + return True + elif parent.compiler.name == "intel" and \ + self._intel_compiler_compare(pversion, cversion): + return True + return False + + + def compatible(self, parent, child, **kwargs): + """Returns true iff a parent and child spec are ABI compatible""" + loosematch = kwargs.get('loose', False) + return self.architecture_compatible(parent, child) and \ + self.compiler_compatible(parent, child, loose=loosematch) + diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 0f258c90968..01ff163493b 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -33,13 +33,16 @@ TODO: make this customizable and allow users to configure concretization policies. """ +import spack import spack.spec import spack.compilers import spack.architecture import spack.error from spack.version import * from functools import partial - +from spec import DependencyMap +from itertools import chain +from spack.config import * class DefaultConcretizer(object): @@ -48,6 +51,107 @@ class DefaultConcretizer(object): default concretization strategies, or you can override all of them. """ + def _find_other_spec(self, spec, condition): + """Searches the dag from spec in an intelligent order and looks + for a spec that matches a condition""" + dagiter = chain(spec.traverse(direction='parents'), spec.traverse(direction='children')) + found = next((x for x in dagiter if x is not spec and condition(x)), None) + if found: + return found + dagiter = chain(spec.traverse(direction='parents'), spec.traverse(direction='children')) + searched = list(dagiter) + found = next((x for x in spec.root.traverse() if x not in searched and x is not spec and condition(x)), None) + if found: + return found + if condition(spec): + return spec + return None + + + def _valid_virtuals_and_externals(self, spec): + """Returns a list of spec/external-path pairs for both virtuals and externals + that can concretize this spec.""" + + # Get a list of candidate packages that could satisfy this spec + packages = [] + if spec.virtual: + providers = spack.db.providers_for(spec) + if not providers: + raise UnsatisfiableProviderSpecError(providers[0], spec) + spec_w_preferred_providers = self._find_other_spec(spec, \ + lambda(x): spack.pkgsort.spec_has_preferred_provider(x.name, spec.name)) + if not spec_w_preferred_providers: + spec_w_preferred_providers = spec + provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name) + packages = sorted(providers, cmp=provider_cmp) + else: + if not spec_externals(spec) or spec.external: + return None + packages = [spec] + + # For each candidate package, if it has externals add those to the candidates + # if it's a nobuild, then only add the externals. + result = [] + all_compilers = spack.compilers.all_compilers() + for pkg in packages: + externals = spec_externals(pkg) + buildable = not is_spec_nobuild(pkg) + if buildable: + result.append((pkg, None)) + if externals: + sorted_externals = sorted(externals, cmp=lambda a,b: a[0].__cmp__(b[0])) + for external in sorted_externals: + if external[0].satisfies(spec): + result.append(external) + if not result: + raise NoBuildError(spec) + return result + + + def concretize_virtual_and_external(self, spec): + """From a list of candidate virtual and external packages, concretize to one that + is ABI compatible with the rest of the DAG.""" + candidates = self._valid_virtuals_and_externals(spec) + if not candidates: + return False + + #Find the another spec in the dag that has a compiler. We'll use that + # spec to test compiler compatibility. + other_spec = self._find_other_spec(spec, lambda(x): x.compiler) + if not other_spec: + other_spec = spec.root + + #Choose an ABI-compatible candidate, or the first match otherwise. + candidate = None + if other_spec: + candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec)), None) + if not candidate: + #Try a looser ABI matching + candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec, loose=True)), None) + if not candidate: + #Pick the first choice + candidate = candidates[0] + external = candidate[1] + candidate_spec = candidate[0] + + #Refine this spec to the candidate. + changed = False + if spec.virtual: + spec._replace_with(candidate_spec) + changed = True + if spec._dup(candidate_spec, deps=False, cleardeps=False): + changed = True + if not spec.external and external: + spec.external = external + changed = True + #If we're external then trim the dependencies + if external and spec.dependencies: + changed = True + spec.depencencies = DependencyMap() + + return changed + + def concretize_version(self, spec): """If the spec is already concrete, return. Otherwise take the preferred version from spackconfig, and default to the package's @@ -150,31 +254,32 @@ def concretize_compiler(self, spec): link to this one, to maximize compatibility. """ all_compilers = spack.compilers.all_compilers() - + if (spec.compiler and spec.compiler.concrete and spec.compiler in all_compilers): return False - # Find the parent spec that has a compiler, or the root if none do - parent_spec = next(p for p in spec.traverse(direction='parents') - if p.compiler is not None or not p.dependents) - parent_compiler = parent_spec.compiler - assert(parent_spec) + #Find the another spec that has a compiler, or the root if none do + other_spec = self._find_other_spec(spec, lambda(x) : x.compiler) + if not other_spec: + other_spec = spec.root + other_compiler = other_spec.compiler + assert(other_spec) # Check if the compiler is already fully specified - if parent_compiler in all_compilers: - spec.compiler = parent_compiler.copy() + if other_compiler in all_compilers: + spec.compiler = other_compiler.copy() return True # Filter the compilers into a sorted list based on the compiler_order from spackconfig - compiler_list = all_compilers if not parent_compiler else spack.compilers.find(parent_compiler) - cmp_compilers = partial(spack.pkgsort.compiler_compare, parent_spec.name) + compiler_list = all_compilers if not other_compiler else spack.compilers.find(other_compiler) + cmp_compilers = partial(spack.pkgsort.compiler_compare, other_spec.name) matches = sorted(compiler_list, cmp=cmp_compilers) if not matches: - raise UnavailableCompilerVersionError(parent_compiler) + raise UnavailableCompilerVersionError(other_compiler) - # copy concrete version into parent_compiler + # copy concrete version into other_compiler spec.compiler = matches[0].copy() assert(spec.compiler.concrete) return True # things changed. @@ -210,3 +315,12 @@ class NoValidVersionError(spack.error.SpackError): def __init__(self, spec): super(NoValidVersionError, self).__init__( "There are no valid versions for %s that match '%s'" % (spec.name, spec.versions)) + + +class NoBuildError(spack.error.SpackError): + """Raised when a package is configured with the nobuild option, but + no satisfactory external versions can be found""" + def __init__(self, spec): + super(NoBuildError, self).__init__( + "The spec '%s' is configured as nobuild, and no matching external installs were found" % spec.name) + diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index f3526b19fa3..60577c45b36 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -90,9 +90,12 @@ import exceptions import sys import copy - -from llnl.util.lang import memoized +import inspect +import glob +import imp +import spack.spec import spack.error +from llnl.util.lang import memoized from external import yaml from external.yaml.error import MarkedYAMLError @@ -116,6 +119,9 @@ def __init__(self, n, f, m): _ConfigCategory('compilers', 'compilers.yaml', True) _ConfigCategory('mirrors', 'mirrors.yaml', True) _ConfigCategory('preferred', 'preferred.yaml', True) +_ConfigCategory('view', 'views.yaml', True) +_ConfigCategory('preferred', 'preferred.yaml', True) +_ConfigCategory('packages', 'packages.yaml', True) """Names of scopes and their corresponding configuration files.""" config_scopes = [('site', os.path.join(spack.etc_path, 'spack')), @@ -233,6 +239,55 @@ def get_preferred_config(): return get_config('preferred') +@memoized +def get_packages_config(): + """Get the externals configuration from config files""" + package_config = get_config('packages') + if not package_config: + return {} + indexed_packages = {} + for p in package_config: + package_name = spack.spec.Spec(p.keys()[0]).name + if package_name not in indexed_packages: + indexed_packages[package_name] = [] + indexed_packages[package_name].append({ spack.spec.Spec(key) : val for key, val in p.iteritems() }) + return indexed_packages + + +def is_spec_nobuild(spec): + """Return true if the spec pkgspec is configured as nobuild""" + allpkgs = get_packages_config() + name = spec.name + if not name in allpkgs: + return False + for itm in allpkgs[name]: + for pkg,conf in itm.iteritems(): + if pkg.satisfies(spec): + if conf.get('nobuild', False): + return True + return False + + +def spec_externals(spec): + """Return a list of spec, directory pairs for each external location for spec""" + allpkgs = get_packages_config() + name = spec.name + spec_locations = [] + + if not name in allpkgs: + return [] + for itm in allpkgs[name]: + for pkg,conf in itm.iteritems(): + if not pkg.satisfies(spec): + continue + path = conf.get('path', None) + if not path: + continue + spec_locations.append( (pkg, path) ) + return spec_locations + + + def get_config_scope_dirname(scope): """For a scope return the config directory""" global config_scopes diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 85ecc1ce2b1..83e6eb566a7 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -187,6 +187,14 @@ def hidden_file_paths(self): def relative_path_for_spec(self, spec): _check_concrete(spec) + + if spec.external: + return spec.external + + enabled_variants = ( + '-' + v.name for v in spec.variants.values() + if v.enabled) + dir_name = "%s-%s-%s" % ( spec.name, spec.version, diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 61606d05905..1e2f0378c8e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -752,6 +752,9 @@ def do_install(self, if not self.spec.concrete: raise ValueError("Can only install concrete packages.") + if self.spec.external: + return + if os.path.exists(self.prefix): tty.msg("%s is already installed in %s." % (self.name, self.prefix)) return diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index bc5271f693d..bc2a4ac234d 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -35,9 +35,10 @@ def __init__(self): #Given a package name, sort component (e.g, version, compiler, ...), and # a second_key (used by providers), return the list - def _order_for_package(self, pkgname, component, second_key): + def _order_for_package(self, pkgname, component, second_key, test_all=True): pkglist = [pkgname] - pkglist.append('all') + if test_all: + pkglist.append('all') for pkg in pkglist: if not pkg in self.preferred: continue @@ -143,6 +144,11 @@ def provider_compare(self, pkgname, provider_str, a, b): return self._spec_compare(pkgname, 'providers', a, b, False, provider_str) + def spec_has_preferred_provider(self, pkgname, provider_str): + """Return True iff the named package has a list of preferred provider""" + return bool(self._order_for_package(pkgname, 'providers', provider_str, False)) + + def version_compare(self, pkgname, a, b): """Return less-than-0, 0, or greater than 0 if version a of pkgname is respecively less-than, equal-to, or greater-than version b of pkgname. diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 41496b0e9d5..6984b4a174a 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -419,6 +419,7 @@ def __init__(self, spec_like, *dep_like, **kwargs): # package.py files for. self._normal = kwargs.get('normal', False) self._concrete = kwargs.get('concrete', False) + self.external = None # This allows users to construct a spec DAG with literals. # Note that given two specs a and b, Spec(a) copies a, but @@ -426,7 +427,7 @@ def __init__(self, spec_like, *dep_like, **kwargs): for dep in dep_like: spec = dep if isinstance(dep, Spec) else Spec(dep) self._add_dependency(spec) - + # # Private routines here are called by the parser when building a spec. @@ -751,12 +752,11 @@ def _concretize_helper(self, presets=None, visited=None): # Concretize virtual dependencies last. Because they're added # to presets below, their constraints will all be merged, but we'll # still need to select a concrete package later. - if not self.virtual: - changed |= any( - (spack.concretizer.concretize_architecture(self), - spack.concretizer.concretize_compiler(self), - spack.concretizer.concretize_version(self), - spack.concretizer.concretize_variants(self))) + changed |= any( + (spack.concretizer.concretize_architecture(self), + spack.concretizer.concretize_compiler(self), + spack.concretizer.concretize_version(self), + spack.concretizer.concretize_variants(self))) presets[self.name] = self visited.add(self.name) @@ -789,21 +789,18 @@ def _expand_virtual_packages(self): a problem. """ changed = False - while True: - virtuals =[v for v in self.traverse() if v.virtual] - if not virtuals: - return changed + done = False + while not done: + done = True + for spec in list(self.traverse()): + if spack.concretizer.concretize_virtual_and_external(spec): + done = False + changed = True - for spec in virtuals: - providers = spack.db.providers_for(spec) - concrete = spack.concretizer.choose_provider(self, spec, providers) - concrete = concrete.copy() - spec._replace_with(concrete) - changed = True - - # If there are duplicate providers or duplicate provider deps, this - # consolidates them and merge constraints. - changed |= self.normalize(force=True) + # If there are duplicate providers or duplicate provider deps, this + # consolidates them and merge constraints. + changed |= self.normalize(force=True) + return changed def concretize(self): @@ -830,7 +827,6 @@ def concretize(self): self._concretize_helper()) changed = any(changes) force=True - self._concrete = True @@ -1346,15 +1342,26 @@ def _dup(self, other, **kwargs): Whether deps should be copied too. Set to false to copy a spec but not its dependencies. """ + + # We don't count dependencies as changes here + changed = True + if hasattr(self, 'name'): + changed = (self.name != other.name and self.versions != other.versions and \ + self.architecture != other.architecture and self.compiler != other.compiler and \ + self.variants != other.variants and self._normal != other._normal and \ + self.concrete != other.concrete and self.external != other.external) + # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() self.architecture = other.architecture self.compiler = other.compiler.copy() if other.compiler else None - self.dependents = DependencyMap() - self.dependencies = DependencyMap() + if kwargs.get('cleardeps', True): + self.dependents = DependencyMap() + self.dependencies = DependencyMap() self.variants = other.variants.copy() self.variants.spec = self + self.external = other.external # If we copy dependencies, preserve DAG structure in the new spec if kwargs.get('deps', True): @@ -1372,6 +1379,8 @@ def _dup(self, other, **kwargs): # Since we preserved structure, we can copy _normal safely. self._normal = other._normal self._concrete = other._concrete + self.external = other.external + return changed def copy(self, **kwargs): @@ -1796,6 +1805,7 @@ def spec(self): spec.variants = VariantMap(spec) spec.architecture = None spec.compiler = None + spec.external = None spec.dependents = DependencyMap() spec.dependencies = DependencyMap() diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py index b6b2dfde214..dfff22152da 100644 --- a/var/spack/packages/mpich/package.py +++ b/var/spack/packages/mpich/package.py @@ -45,6 +45,7 @@ def setup_dependent_environment(self, module, spec, dep_spec): os.environ['MPICH_F77'] = 'f77' os.environ['MPICH_F90'] = 'f90' + module.mpicc = join_path(self.prefix.bin, 'mpicc') def install(self, spec, prefix): config_args = ["--prefix=" + prefix, diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py index ca0b1287c1c..93bce011b71 100644 --- a/var/spack/packages/mvapich2/package.py +++ b/var/spack/packages/mvapich2/package.py @@ -11,10 +11,17 @@ class Mvapich2(Package): version('2.0', '9fbb68a4111a8b6338e476dc657388b4', url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.0.tar.gz') + + version('2.1', '0095ceecb19bbb7fb262131cb9c2cdd6', + url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.1.tar.gz') provides('mpi@:2.2', when='@1.9') # MVAPICH2-1.9 supports MPI 2.2 provides('mpi@:3.0', when='@2.0') # MVAPICH2-2.0 supports MPI 3.0 + variant('psm', default=False, description="build with psm") + + variant('pmi', default=False, description="build with pmi") + depends_on('pmgr_collective', when='+pmi') def install(self, spec, prefix): # we'll set different configure flags depending on our environment @@ -80,7 +87,13 @@ def install(self, spec, prefix): configure_args.append("--with-device=ch3:psm") else: # throw this flag on IB systems - configure_args.append("--with-device=ch3:mrail", "--with-rdma=gen2") + configure_args.append("--with-device=ch3:mrail") + configure_args.append("--with-rdma=gen2") + + if "+pmi" in spec: + configure_args.append("--with-pmi=pmgr_collective" % spec['pmgr_collective'].prefix) + else: + configure_args.append("--with-pmi=slurm") # TODO: shared-memory build @@ -93,7 +106,7 @@ def install(self, spec, prefix): "--enable-f77", "--enable-fc", "--enable-cxx", "--enable-shared", "--enable-sharedlibs=gcc", "--enable-debuginfo", - "--with-pm=no", "--with-pmi=slurm", + "--with-pm=no", "--enable-romio", "--with-file-system=lustre+nfs+ufs", "--disable-mpe", "--without-mpe", "--disable-silent-rules", From e4d2ba30b57618da388a1a990381d149b33d7aba Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Fri, 2 Oct 2015 11:00:41 -0700 Subject: [PATCH 032/910] Fix failure in spack.test.config.ConfigTest from incorrect compiler config merging --- lib/spack/spack/config.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 60577c45b36..712a2b78fc0 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -174,8 +174,12 @@ def _merge_dicts(d1, d2): for key2, val2 in d2.iteritems(): if not key2 in d1: d1[key2] = val2 - else: + elif type(d1[key2]) is dict and type(val2) is dict: d1[key2] = _merge_dicts(d1[key2], val2) + elif (type(d1) is list) and (type(d2) is list): + d1.extend(d2) + else: + d1[key2] = val2 return d1 return d2 @@ -360,7 +364,7 @@ def add_to_mirror_config(addition_dict, scope=None): def add_to_compiler_config(addition_dict, scope=None, arch=None): - """Add compilerss to the configuration files""" + """Add compilers to the configuration files""" if not arch: arch = spack.architecture.sys_type() add_to_config('compilers', { arch : addition_dict }, scope) From 18f0b24a7f21ec7b46510f45867386b7600bbc55 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 5 Oct 2015 11:30:48 -0700 Subject: [PATCH 033/910] Add tests for spack external dependencies, plus fixes for issues found by those tests. --- lib/spack/spack/concretize.py | 9 +++-- lib/spack/spack/spec.py | 2 +- lib/spack/spack/test/concretize.py | 28 ++++++++++++++ .../site_spackconfig/packages.yaml | 13 +++++++ .../mock_packages/externalprereq/package.py | 34 +++++++++++++++++ .../mock_packages/externaltest/package.py | 37 +++++++++++++++++++ .../mock_packages/externaltool/package.py | 36 ++++++++++++++++++ .../mock_packages/externalvirtual/package.py | 37 +++++++++++++++++++ 8 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 var/spack/mock_configs/site_spackconfig/packages.yaml create mode 100644 var/spack/mock_packages/externalprereq/package.py create mode 100644 var/spack/mock_packages/externaltest/package.py create mode 100644 var/spack/mock_packages/externaltool/package.py create mode 100644 var/spack/mock_packages/externalvirtual/package.py diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 01ff163493b..c27a023136a 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -85,8 +85,8 @@ def _valid_virtuals_and_externals(self, spec): provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name) packages = sorted(providers, cmp=provider_cmp) else: - if not spec_externals(spec) or spec.external: - return None + if spec.external: + return False packages = [spec] # For each candidate package, if it has externals add those to the candidates @@ -129,7 +129,7 @@ def concretize_virtual_and_external(self, spec): #Try a looser ABI matching candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec, loose=True)), None) if not candidate: - #Pick the first choice + #No ABI matches. Pick the top choice based on the orignal preferences. candidate = candidates[0] external = candidate[1] candidate_spec = candidate[0] @@ -144,10 +144,11 @@ def concretize_virtual_and_external(self, spec): if not spec.external and external: spec.external = external changed = True + #If we're external then trim the dependencies if external and spec.dependencies: changed = True - spec.depencencies = DependencyMap() + spec.dependencies = DependencyMap() return changed diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 6984b4a174a..49b67cd3614 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1022,7 +1022,7 @@ def _normalize_helper(self, visited, spec_deps, provider_index): # if we descend into a virtual spec, there's nothing more # to normalize. Concretize will finish resolving it later. - if self.virtual: + if self.virtual or self.external: return False # Combine constraints from package deps with constraints from diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index b3a77d076a1..f81a2f5af81 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -192,3 +192,31 @@ def test_compiler_inheritance(self): # TODO: not exactly the syntax I would like. self.assertTrue(spec['libdwarf'].compiler.satisfies('clang')) self.assertTrue(spec['libelf'].compiler.satisfies('clang')) + + + def test_external_package(self): + spec = Spec('externaltool') + spec.concretize() + + self.assertEqual(spec['externaltool'].external, '/path/to/external_tool') + self.assertFalse('externalprereq' in spec) + self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) + + + def test_nobuild_package(self): + got_error = False + spec = Spec('externaltool%clang') + try: + spec.concretize() + except spack.concretize.NoBuildError: + got_error = True + self.assertTrue(got_error) + + + def test_external_and_virtual(self): + spec = Spec('externaltest') + spec.concretize() + self.assertTrue(spec['externaltool'].external, '/path/to/external_tool') + self.assertTrue(spec['stuff'].external, '/path/to/external_virtual_gcc') + self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) + self.assertTrue(spec['stuff'].compiler.satisfies('gcc')) diff --git a/var/spack/mock_configs/site_spackconfig/packages.yaml b/var/spack/mock_configs/site_spackconfig/packages.yaml new file mode 100644 index 00000000000..eb52c6cf113 --- /dev/null +++ b/var/spack/mock_configs/site_spackconfig/packages.yaml @@ -0,0 +1,13 @@ +packages: + - externaltool: + nobuild: True + - externaltool@1.0%gcc@4.5.0: + path: /path/to/external_tool + - externalvirtual@2.0%clang@3.3: + path: /path/to/external_virtual_clang + nobuild: True + - externalvirtual@1.0%gcc@4.5.0: + path: /path/to/external_virtual_gcc + nobuild: True + + diff --git a/var/spack/mock_packages/externalprereq/package.py b/var/spack/mock_packages/externalprereq/package.py new file mode 100644 index 00000000000..7d639256939 --- /dev/null +++ b/var/spack/mock_packages/externalprereq/package.py @@ -0,0 +1,34 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externalprereq(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/prereq-1.0.tar.gz" + + version('1.4', 'f1234567890abcdef1234567890abcde') + + def install(self, spec, prefix): + pass diff --git a/var/spack/mock_packages/externaltest/package.py b/var/spack/mock_packages/externaltest/package.py new file mode 100644 index 00000000000..c546922f876 --- /dev/null +++ b/var/spack/mock_packages/externaltest/package.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externaltest(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/test-1.0.tar.gz" + + version('1.0', '1234567890abcdef1234567890abcdef') + + depends_on('stuff') + depends_on('externaltool') + + def install(self, spec, prefix): + pass diff --git a/var/spack/mock_packages/externaltool/package.py b/var/spack/mock_packages/externaltool/package.py new file mode 100644 index 00000000000..af902bd70eb --- /dev/null +++ b/var/spack/mock_packages/externaltool/package.py @@ -0,0 +1,36 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externaltool(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/tool-1.0.tar.gz" + + version('1.0', '1234567890abcdef1234567890abcdef') + + depends_on('externalprereq') + + def install(self, spec, prefix): + pass diff --git a/var/spack/mock_packages/externalvirtual/package.py b/var/spack/mock_packages/externalvirtual/package.py new file mode 100644 index 00000000000..722c1e1c532 --- /dev/null +++ b/var/spack/mock_packages/externalvirtual/package.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externalvirtual(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/stuff-1.0.tar.gz" + + version('1.0', '1234567890abcdef1234567890abcdef') + version('2.0', '234567890abcdef1234567890abcdef1') + + provides('stuff') + + def install(self, spec, prefix): + pass From fac4428766fb0a6b6cd357b654215f55df1220d4 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 5 Oct 2015 14:04:33 -0700 Subject: [PATCH 034/910] Documentation for external packages. --- lib/spack/docs/site_configuration.rst | 73 +++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index 1e6740a434f..a7211a9d95b 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -54,6 +54,79 @@ more elements to the list to indicate where your own site's temporary directory is. +External Packages +~~~~~~~~~~~~~~~~~~~~~ +It's possible for Spack to use certain externally-installed +packages rather than always rebuilding packages. This may be desirable +if machines ship with system packages, such as a customized MPI +that should be used instead of Spack building its own MPI. + +External packages are configured through the ``packages.yaml`` file found +in a Spack installation's ``etc/spack/`` or a user's ``~/.spack/`` +directory. Here's an example of an external configuration:: + +.. code-block:: yaml + + packages: + - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: + path: /opt/openmpi-1.4.3 + - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: + path: /opt/openmpi-1.4.3-debug + - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: + path: /opt/openmpi-1.6.5-intel + +This example lists three installations of OpenMPI, one built with gcc, +one built with gcc and debug information, and another built with OpenMPI. +If Spack is asked to build a package that uses one of these MPIs as a +dependency, it link the package to the pre-installed OpenMPI in +the given directory. + +Each ``packages.yaml`` should begin with a ``packages:`` token, followed +by a list of package specs. Specs in the ``packages.yaml`` have at most +one ``path`` tag, which specifies the top-level directory where the +spec is installed. + +Each spec should be as well-defined as reasonably possible. If a +package lacks a spec component, such as missing a compiler or +package version, then Spack will guess the missing component based +on its most-favored packages, and it may guess incorrectly. + +All package versions and compilers listed in ``packages.yaml`` should +have entries in Spack's packages and compiler configuration, even +the package and compiler may not actually be used. + +The packages configuration can tell Spack to use an external location +for certain package versions, but it does not restrict Spack to using +external packages. In the above example, if an OpenMPI 1.8.4 became +available Spack may choose to start building and linking with that version +rather than continue using the pre-installed OpenMPI versions. + +To prevent this, the ``packages.yaml`` configuration also allows packages +to be flagged as non-buildable. The previous example could be modified to +be:: + +.. code-block:: yaml + + packages: + - openmpi: + nobuild: True + - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: + path: /opt/openmpi-1.4.3 + - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: + path: /opt/openmpi-1.4.3-debug + - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: + path: /opt/openmpi-1.6.5-intel + +The addition of the ``nobuild`` flag tells Spack that it should never build +its own version of OpenMPI, and it will instead always rely on a pre-built +OpenMPI. Similar to ``path``, ``nobuild`` is specified as a property under +a spec and will prevent building of anything that satisfies that spec. + +The ``nobuild`` does not need to be paired with external packages. +It could also be used alone to forbid versions of packages that may be +buggy or otherwise undesirable. + + Profiling ~~~~~~~~~~~~~~~~~~~~~ From 16f0ece75bdbcbf5c85af532d928227818766077 Mon Sep 17 00:00:00 2001 From: Paul Dapolito Date: Sun, 1 Nov 2015 19:55:14 -0800 Subject: [PATCH 035/910] added zlib dependency to glib package specification --- var/spack/packages/glib/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/glib/package.py b/var/spack/packages/glib/package.py index 178f0b9df51..baca1a5a459 100644 --- a/var/spack/packages/glib/package.py +++ b/var/spack/packages/glib/package.py @@ -11,6 +11,7 @@ class Glib(Package): version('2.42.1', '89c4119e50e767d3532158605ee9121a') depends_on("libffi") + depends_on("zlib") def install(self, spec, prefix): configure("--prefix=%s" % prefix) From d6a2ab08acd8f4cbedca20ac182329d1090af056 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Fri, 6 Nov 2015 11:47:00 -0800 Subject: [PATCH 036/910] Match up with the new binutils package file. It does what OSS needs w.r.t. libiberty. --- var/spack/packages/binutils/package.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py index 43a670585e6..29b8de462b4 100644 --- a/var/spack/packages/binutils/package.py +++ b/var/spack/packages/binutils/package.py @@ -13,19 +13,22 @@ class Binutils(Package): variant('krellpatch', default=False, description="build with openspeedshop based patch.") patch('binutilskrell-2.24.patch', when='@2.24+krellpatch') + variant('libiberty', default=False, description='Also install libiberty.') + def install(self, spec, prefix): - # Add additional configuration options for use in the OpenSpeedShop project - if '+krellpatch' in spec: - configure('--prefix', self.prefix, - '--libdir', self.prefix.lib, - '--enable-shared', - '--enable-install-libiberty', - '--disable-multilib' - ) - else: - configure("--prefix=%s" % prefix) + configure_args = [ + '--prefix=%s' % prefix, + '--disable-dependency-tracking', + '--enable-interwork', + '--enable-multilib', + '--enable-shared', + '--enable-64-bit-bfd', + '--enable-targets=all'] + if '+libiberty' in spec: + configure_args.append('--enable-install-libiberty') + configure(*configure_args) make() make("install") From 7e6b7795f1cd1e73591c3fba5602220d20d453e6 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Fri, 6 Nov 2015 11:53:53 -0800 Subject: [PATCH 037/910] Match up with the new binutils package file. It does what OSS needs w.r.t. libiberty. --- var/spack/packages/binutils/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py index 29b8de462b4..5ef9a80cca9 100644 --- a/var/spack/packages/binutils/package.py +++ b/var/spack/packages/binutils/package.py @@ -15,6 +15,8 @@ class Binutils(Package): variant('libiberty', default=False, description='Also install libiberty.') + variant('libiberty', default=False, description='Also install libiberty.') + def install(self, spec, prefix): configure_args = [ From aa9b14aff47864c04f99bd4c83fb10dc6a9c49ad Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Fri, 6 Nov 2015 11:55:20 -0800 Subject: [PATCH 038/910] Match up with the new binutils package file. It does what OSS needs w.r.t. libiberty. --- var/spack/packages/binutils/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py index 5ef9a80cca9..c939822cdb5 100644 --- a/var/spack/packages/binutils/package.py +++ b/var/spack/packages/binutils/package.py @@ -17,8 +17,9 @@ class Binutils(Package): variant('libiberty', default=False, description='Also install libiberty.') - def install(self, spec, prefix): + variant('libiberty', default=False, description='Also install libiberty.') + def install(self, spec, prefix): configure_args = [ '--prefix=%s' % prefix, '--disable-dependency-tracking', From 2aa501bf3db5c3962b6f2a0932e21afe03db9329 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Fri, 6 Nov 2015 14:19:00 -0800 Subject: [PATCH 039/910] Update the checksum value for the new tarball --- var/spack/packages/openspeedshop/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/openspeedshop/package.py b/var/spack/packages/openspeedshop/package.py index 3c24aed8824..93f356fb924 100644 --- a/var/spack/packages/openspeedshop/package.py +++ b/var/spack/packages/openspeedshop/package.py @@ -30,7 +30,7 @@ class Openspeedshop(Package): homepage = "http://www.openspeedshop.org" url = "http://sourceforge.net/projects/openss/files/openss/openspeedshop-2.1/openspeedshop-2.1.tar.gz/download" - version('2.1', '64ee17166519838c7b94a1adc138e94f') + version('2.1', 'bdaa57c1a0db9d0c3e0303fd8496c507') # optional mirror template #url = "file:/g/g24/jeg/openspeedshop-2.1.tar.gz" From 0ced060b87b129c47666ef91f79d52d94e79e291 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Fri, 6 Nov 2015 15:15:38 -0800 Subject: [PATCH 040/910] cbtf did not compile with boost 1.59, so reverted to boost 1.50 temporarily until we fix the cbtf/boost compile issues. --- var/spack/packages/cbtf/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/cbtf/package.py b/var/spack/packages/cbtf/package.py index 46d47153e62..6f1e49deed2 100644 --- a/var/spack/packages/cbtf/package.py +++ b/var/spack/packages/cbtf/package.py @@ -32,7 +32,8 @@ class Cbtf(Package): version('1.6', branch='master', git='http://git.code.sf.net/p/cbtf/cbtf') depends_on("cmake") - depends_on("boost@1.42:") + #depends_on("boost@1.42:") + depends_on("boost@1.50") depends_on("mrnet@4.1.0+krelloptions") depends_on("xerces-c@3.1.1:") depends_on("libxml2") From 076eaba6e32f15031840c83a2ea019c14db5f6f2 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 27 Oct 2015 13:23:15 +0100 Subject: [PATCH 041/910] mvapich2 : fixed some issues with the package. Needs further improvment with respect to network and pm --- var/spack/packages/mvapich2/package.py | 174 +++++++++++++------------ 1 file changed, 88 insertions(+), 86 deletions(-) diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py index ca0b1287c1c..1937564fc88 100644 --- a/var/spack/packages/mvapich2/package.py +++ b/var/spack/packages/mvapich2/package.py @@ -1,104 +1,106 @@ -import os from spack import * + class Mvapich2(Package): """mvapich2 is an MPI implmenetation for infiniband networks.""" homepage = "http://mvapich.cse.ohio-state.edu/" - version('1.9', '5dc58ed08fd3142c260b70fe297e127c', - url="http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz") - patch('ad_lustre_rwcontig_open_source.patch', when='@1.9') + version('2.2a', 'b8ceb4fc5f5a97add9b3ff1b9cbe39d2', + url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.2a.tar.gz') version('2.0', '9fbb68a4111a8b6338e476dc657388b4', url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.0.tar.gz') - provides('mpi@:2.2', when='@1.9') # MVAPICH2-1.9 supports MPI 2.2 - provides('mpi@:3.0', when='@2.0') # MVAPICH2-2.0 supports MPI 3.0 + version('1.9', '5dc58ed08fd3142c260b70fe297e127c', + url="http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz") + patch('ad_lustre_rwcontig_open_source.patch', when='@1.9') + provides('mpi@:2.2', when='@1.9') # MVAPICH2-1.9 supports MPI 2.2 + provides('mpi@:3.0', when='@2.0:') # MVAPICH2-2.0 supports MPI 3.0 + + + variant('slurm', default=True, description='Sets SLURM as the process manager') + + # FIXME: those variants are mutually exclusive. A variant enum would fit here. + variant('psm', default=False, description='Configures a build for QLogic PSM-CH3') + variant('sock', default=False, description='Configures a build for TCP/IP-CH3') + # TODO : a lot of network variants are still missing. + # See http://mvapich.cse.ohio-state.edu/static/media/mvapich/mvapich2-2.0-userguide.html + + def set_build_type_flags(self, spec, configure_args): + """ + Appends to configure_args the flags that depends only on the build type (i.e. release or debug) + + :param spec: spec + :param configure_args: list of current configure arguments + """ + if '+debug' in spec: + build_type_options = [ + "--disable-fast", + "--enable-error-checking=runtime", + "--enable-error-messages=all" + ] + + else: + build_type_options = ["--enable-fast=all"] + configure_args.extend(build_type_options) + + def set_process_manager(self, spec, configure_args): + """ + Appends to configure_args the flags that will select the process manager + + :param spec: spec + :param configure_args: list of current configure arguments + """ + # FIXME : supports only slurm so far + if '+slurm' in spec: + process_manager_options = [ + "--with-pm=no", + "--with-pmi=slurm" + ] + configure_args.extend(process_manager_options) + + def set_network_type(self, spec, configure_args): + # Check that at most one variant has been activated + # FIXME : ugly, as it does not scale at all (and is full of conditionals) + count = 0 + if '+psm' in spec: + count += 1 + if '+sock' in spec: + count += 1 + if count > 1: + raise RuntimeError('MVAPICH2 variants are mutually exclusive : only one can be selected at a time') + + # From here on I can suppose that ony one variant has been selected + if '+psm' in spec: + network_options = ["--with-device=ch3:psm"] + elif '+sock' in spec: + network_options = ["--with-device=ch3:sock"] + else: + network_options = ["--with-device=ch3:mrail", "--with-rdma=gen2"] + + configure_args.extend(network_options) def install(self, spec, prefix): # we'll set different configure flags depending on our environment - configure_args = [] - - # TODO: The MPICH*_FLAGS have a different name for 1.9 - - if '+debug' in spec: - # set configure flags for debug build - configure_args.append("--disable-fast") - configure_args.append("--enable-g=dbg") - configure_args.append("--enable-error-checking=runtime") - configure_args.append("--enable-error-messages=all") - configure_args.append("--enable-nmpi-as-mpi") - - if "%gnu" in spec: - # set variables for GNU compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O0" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O0" - os.environ['MPICHLIB_FFLAGS'] = "-g -O0 -fno-second-underscore" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O0 -fno-second-underscore" - elif "%intel" in spec: - # set variables for Inel compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O0" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O0" - os.environ['MPICHLIB_FFLAGS'] = "-g -O0" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O0" - elif "%pgi" in spec: - # set variables for PGI compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O0 -fPIC" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O0 -fPIC" - os.environ['MPICHLIB_FFLAGS'] = "-g -O0 -fPIC" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O0 -fPIC" - - else: - # set configure flags for normal optimizations - configure_args.append("--enable-fast=all") - configure_args.append("--enable-g=dbg") - configure_args.append("--enable-nmpi-as-mpi") - - if "%gnu" in spec: - # set variables for what compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O2" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O2" - os.environ['MPICHLIB_FFLAGS'] = "-g -O2 -fno-second-underscore" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O2 -fno-second-underscore" - elif "%intel" in spec: - # set variables for Inel compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O2" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O2" - os.environ['MPICHLIB_FFLAGS'] = "-g -O2" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O2" - elif "%pgi" in spec: - # set variables for PGI compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O2 -fPIC" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O2 -fPIC" - os.environ['MPICHLIB_FFLAGS'] = "-g -O2 -fPIC" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O2 -fPIC" - - # determine network type by variant - if "+psm" in spec: - # throw this flag on QLogic systems to use PSM - configure_args.append("--with-device=ch3:psm") - else: - # throw this flag on IB systems - configure_args.append("--with-device=ch3:mrail", "--with-rdma=gen2") - - # TODO: shared-memory build - - # TODO: CUDA - - # TODO: other file systems like panasis - - configure( - "--prefix=" + prefix, - "--enable-f77", "--enable-fc", "--enable-cxx", - "--enable-shared", "--enable-sharedlibs=gcc", - "--enable-debuginfo", - "--with-pm=no", "--with-pmi=slurm", - "--enable-romio", "--with-file-system=lustre+nfs+ufs", - "--disable-mpe", "--without-mpe", + configure_args = [ + "--prefix=%s" % prefix, + "--enable-shared", + "--enable-romio", "--disable-silent-rules", - *configure_args) + "--enable-debuginfo", + "--enable-g=dbg" + ] + if not self.compiler.f77 and not self.compiler.fc: + configure_args.append("--enable-fortran=none") + # Set flags that depend only on the type of the build (debug, release) + self.set_build_type_flags(spec, configure_args) + # Set the process manager + self.set_process_manager(spec, configure_args) + # Determine network type by variant + self.set_network_type(spec, configure_args) + + configure(*configure_args) make() - make("install") From 4a72bd791981ba799936a3f5fb9c843401036c8e Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 29 Oct 2015 10:45:29 +0100 Subject: [PATCH 042/910] mvapich2 : fixed typos. Added FIXME --- var/spack/packages/mvapich2/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py index 1937564fc88..94da920c7cc 100644 --- a/var/spack/packages/mvapich2/package.py +++ b/var/spack/packages/mvapich2/package.py @@ -2,7 +2,7 @@ class Mvapich2(Package): - """mvapich2 is an MPI implmenetation for infiniband networks.""" + """MVAPICH2 is an MPI implementation for Infiniband networks.""" homepage = "http://mvapich.cse.ohio-state.edu/" version('2.2a', 'b8ceb4fc5f5a97add9b3ff1b9cbe39d2', @@ -13,6 +13,7 @@ class Mvapich2(Package): version('1.9', '5dc58ed08fd3142c260b70fe297e127c', url="http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz") + # FIXME: the statement doesn't seem to take 'when' into account when applying patches patch('ad_lustre_rwcontig_open_source.patch', when='@1.9') provides('mpi@:2.2', when='@1.9') # MVAPICH2-1.9 supports MPI 2.2 From 0ae05fdf99a8e22b2abb266831b3ea77bbd201b9 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Mon, 9 Nov 2015 13:58:34 +0100 Subject: [PATCH 043/910] mvapich2 : added variants for different process managers --- var/spack/packages/mvapich2/package.py | 39 ++++++++++++++++++++------ 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py index 94da920c7cc..3747724915e 100644 --- a/var/spack/packages/mvapich2/package.py +++ b/var/spack/packages/mvapich2/package.py @@ -13,14 +13,21 @@ class Mvapich2(Package): version('1.9', '5dc58ed08fd3142c260b70fe297e127c', url="http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz") - # FIXME: the statement doesn't seem to take 'when' into account when applying patches patch('ad_lustre_rwcontig_open_source.patch', when='@1.9') provides('mpi@:2.2', when='@1.9') # MVAPICH2-1.9 supports MPI 2.2 provides('mpi@:3.0', when='@2.0:') # MVAPICH2-2.0 supports MPI 3.0 + variant('debug', default=False, description='Enables debug information and error messages at run-time') - variant('slurm', default=True, description='Sets SLURM as the process manager') + ########## + # TODO : Process managers should be grouped into the same variant, as soon as variant capabilities will be extended + # See https://groups.google.com/forum/#!topic/spack/F8-f8B4_0so + variant('slurm', default=False, description='Sets slurm as the only process manager') + variant('hydra', default=False, description='Sets hydra as one of the process managers') + variant('gforker', default=False, description='Sets gforker as one of the process managers') + variant('remshell', default=False, description='Sets remshell as one of the process managers') + ########## # FIXME: those variants are mutually exclusive. A variant enum would fit here. variant('psm', default=False, description='Configures a build for QLogic PSM-CH3') @@ -41,25 +48,41 @@ def set_build_type_flags(self, spec, configure_args): "--enable-error-checking=runtime", "--enable-error-messages=all" ] - else: build_type_options = ["--enable-fast=all"] + configure_args.extend(build_type_options) def set_process_manager(self, spec, configure_args): """ - Appends to configure_args the flags that will select the process manager + Appends to configure_args the flags that will enable the appropriate process managers :param spec: spec :param configure_args: list of current configure arguments """ - # FIXME : supports only slurm so far + # Check that slurm variant is not activated together with other pm variants + has_slurm_incompatible_variant = any((x in spec for x in ['+hydra', '+gforker', '+remshell'])) + if '+slurm' in spec and has_slurm_incompatible_variant: + raise RuntimeError(" %s : 'slurm' cannot be activated together with other process managers" % self.name) + + process_manager_options = [] if '+slurm' in spec: process_manager_options = [ - "--with-pm=no", - "--with-pmi=slurm" + "--with-pm=slurm" ] - configure_args.extend(process_manager_options) + elif has_slurm_incompatible_variant: + pm = [] + if '+hydra' in spec: + pm.append('hydra') + if '+gforker' in spec: + pm.append('gforker') + if '+remshell' in spec: + pm.append('remshell') + + process_manager_options = [ + "--with-pm=%s" % ':'.join(pm) + ] + configure_args.extend(process_manager_options) def set_network_type(self, spec, configure_args): # Check that at most one variant has been activated From 79ed9de4c12ac0decfd8a49cfe17fedb96f9e5af Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Mon, 9 Nov 2015 14:35:54 +0100 Subject: [PATCH 044/910] mvapich2 : added variants for different network types --- var/spack/packages/mvapich2/package.py | 28 +++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py index 3747724915e..fb5912fcef3 100644 --- a/var/spack/packages/mvapich2/package.py +++ b/var/spack/packages/mvapich2/package.py @@ -29,11 +29,16 @@ class Mvapich2(Package): variant('remshell', default=False, description='Sets remshell as one of the process managers') ########## - # FIXME: those variants are mutually exclusive. A variant enum would fit here. + ########## + # TODO : Network types should be grouped into the same variant, as soon as variant capabilities will be extended variant('psm', default=False, description='Configures a build for QLogic PSM-CH3') variant('sock', default=False, description='Configures a build for TCP/IP-CH3') - # TODO : a lot of network variants are still missing. - # See http://mvapich.cse.ohio-state.edu/static/media/mvapich/mvapich2-2.0-userguide.html + variant('nemesisibtcp', default=False, description='Configures a build for both OFA-IB-Nemesis and TCP/IP-Nemesis') + variant('nemesisib', default=False, description='Configures a build for OFA-IB-Nemesis') + variant('nemesis', default=False, description='Configures a build for TCP/IP-Nemesis') + ########## + + # TODO : CUDA support is missing def set_build_type_flags(self, spec, configure_args): """ @@ -46,7 +51,8 @@ def set_build_type_flags(self, spec, configure_args): build_type_options = [ "--disable-fast", "--enable-error-checking=runtime", - "--enable-error-messages=all" + "--enable-error-messages=all", + "--enable-g=dbg", "--enable-debuginfo" # Permits debugging with TotalView ] else: build_type_options = ["--enable-fast=all"] @@ -92,14 +98,26 @@ def set_network_type(self, spec, configure_args): count += 1 if '+sock' in spec: count += 1 + if '+nemesisibtcp' in spec: + count += 1 + if '+nemesisib' in spec: + count += 1 + if '+nemesis' in spec: + count += 1 if count > 1: - raise RuntimeError('MVAPICH2 variants are mutually exclusive : only one can be selected at a time') + raise RuntimeError('MVAPICH2 network variants are mutually exclusive : only one can be selected at a time') # From here on I can suppose that ony one variant has been selected if '+psm' in spec: network_options = ["--with-device=ch3:psm"] elif '+sock' in spec: network_options = ["--with-device=ch3:sock"] + elif '+nemesisibtcp' in spec: + network_options = ["--with-device=ch3:nemesis:ib,tcp"] + elif '+nemesisib' in spec: + network_options = ["--with-device=ch3:nemesis:ib"] + elif '+nemesis' in spec: + network_options = ["--with-device=ch3:nemesis"] else: network_options = ["--with-device=ch3:mrail", "--with-rdma=gen2"] From 42bc552dd74cfe961b3f345321994dfa637549e7 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Mon, 9 Nov 2015 15:45:23 +0100 Subject: [PATCH 045/910] mvapich2 : variant strings are associated with a named variable --- var/spack/packages/mvapich2/package.py | 90 ++++++++++++++------------ 1 file changed, 50 insertions(+), 40 deletions(-) diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py index fb5912fcef3..53090ee05ca 100644 --- a/var/spack/packages/mvapich2/package.py +++ b/var/spack/packages/mvapich2/package.py @@ -23,22 +23,43 @@ class Mvapich2(Package): ########## # TODO : Process managers should be grouped into the same variant, as soon as variant capabilities will be extended # See https://groups.google.com/forum/#!topic/spack/F8-f8B4_0so - variant('slurm', default=False, description='Sets slurm as the only process manager') - variant('hydra', default=False, description='Sets hydra as one of the process managers') - variant('gforker', default=False, description='Sets gforker as one of the process managers') - variant('remshell', default=False, description='Sets remshell as one of the process managers') + SLURM = 'slurm' + HYDRA = 'hydra' + GFORKER = 'gforker' + REMSHELL = 'remshell' + SLURM_INCOMPATIBLE_PMS = (HYDRA, GFORKER, REMSHELL) + variant(SLURM, default=False, description='Sets slurm as the only process manager') + variant(HYDRA, default=False, description='Sets hydra as one of the process managers') + variant(GFORKER, default=False, description='Sets gforker as one of the process managers') + variant(REMSHELL, default=False, description='Sets remshell as one of the process managers') ########## ########## # TODO : Network types should be grouped into the same variant, as soon as variant capabilities will be extended - variant('psm', default=False, description='Configures a build for QLogic PSM-CH3') - variant('sock', default=False, description='Configures a build for TCP/IP-CH3') - variant('nemesisibtcp', default=False, description='Configures a build for both OFA-IB-Nemesis and TCP/IP-Nemesis') - variant('nemesisib', default=False, description='Configures a build for OFA-IB-Nemesis') - variant('nemesis', default=False, description='Configures a build for TCP/IP-Nemesis') + PSM = 'psm' + SOCK = 'sock' + NEMESISIBTCP = 'nemesisibtcp' + NEMESISIB = 'nemesisib' + NEMESIS = 'nemesis' + SUPPORTED_NETWORKS = (PSM, SOCK, NEMESIS, NEMESISIB, NEMESISIBTCP) + variant(PSM, default=False, description='Configures a build for QLogic PSM-CH3') + variant(SOCK, default=False, description='Configures a build for TCP/IP-CH3') + variant(NEMESISIBTCP, default=False, description='Configures a build for both OFA-IB-Nemesis and TCP/IP-Nemesis') + variant(NEMESISIB, default=False, description='Configures a build for OFA-IB-Nemesis') + variant(NEMESIS, default=False, description='Configures a build for TCP/IP-Nemesis') ########## - # TODO : CUDA support is missing + # FIXME : CUDA support is missing + + @staticmethod + def enabled(x): + """ + Given a variant name returns the string that means the variant is enabled + + :param x: variant name + :return: + """ + return '+' + x def set_build_type_flags(self, spec, configure_args): """ @@ -67,56 +88,45 @@ def set_process_manager(self, spec, configure_args): :param configure_args: list of current configure arguments """ # Check that slurm variant is not activated together with other pm variants - has_slurm_incompatible_variant = any((x in spec for x in ['+hydra', '+gforker', '+remshell'])) - if '+slurm' in spec and has_slurm_incompatible_variant: + has_slurm_incompatible_variants = any(self.enabled(x) in spec for x in Mvapich2.SLURM_INCOMPATIBLE_PMS) + if self.enabled(Mvapich2.SLURM) in spec and has_slurm_incompatible_variants: raise RuntimeError(" %s : 'slurm' cannot be activated together with other process managers" % self.name) process_manager_options = [] - if '+slurm' in spec: + if self.enabled(Mvapich2.SLURM) in spec: process_manager_options = [ "--with-pm=slurm" ] - elif has_slurm_incompatible_variant: - pm = [] - if '+hydra' in spec: - pm.append('hydra') - if '+gforker' in spec: - pm.append('gforker') - if '+remshell' in spec: - pm.append('remshell') - + elif has_slurm_incompatible_variants: + pms = [] + # The variant name is equal to the process manager name in the configuration options + for x in Mvapich2.SLURM_INCOMPATIBLE_PMS: + if self.enabled(x) in spec: + pms.append(x) process_manager_options = [ - "--with-pm=%s" % ':'.join(pm) + "--with-pm=%s" % ':'.join(pms) ] configure_args.extend(process_manager_options) def set_network_type(self, spec, configure_args): # Check that at most one variant has been activated - # FIXME : ugly, as it does not scale at all (and is full of conditionals) count = 0 - if '+psm' in spec: - count += 1 - if '+sock' in spec: - count += 1 - if '+nemesisibtcp' in spec: - count += 1 - if '+nemesisib' in spec: - count += 1 - if '+nemesis' in spec: - count += 1 + for x in Mvapich2.SUPPORTED_NETWORKS: + if self.enabled(x) in spec: + count += 1 if count > 1: - raise RuntimeError('MVAPICH2 network variants are mutually exclusive : only one can be selected at a time') + raise RuntimeError('network variants are mutually exclusive (only one can be selected at a time)') # From here on I can suppose that ony one variant has been selected - if '+psm' in spec: + if self.enabled(Mvapich2.PSM) in spec: network_options = ["--with-device=ch3:psm"] - elif '+sock' in spec: + elif self.enabled(Mvapich2.SOCK) in spec: network_options = ["--with-device=ch3:sock"] - elif '+nemesisibtcp' in spec: + elif self.enabled(Mvapich2.NEMESISIBTCP) in spec: network_options = ["--with-device=ch3:nemesis:ib,tcp"] - elif '+nemesisib' in spec: + elif self.enabled(Mvapich2.NEMESISIB) in spec: network_options = ["--with-device=ch3:nemesis:ib"] - elif '+nemesis' in spec: + elif self.enabled(Mvapich2.NEMESIS) in spec: network_options = ["--with-device=ch3:nemesis"] else: network_options = ["--with-device=ch3:mrail", "--with-rdma=gen2"] From 27c64f77404ed3f8a201294596a8a607f18bd39b Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Mon, 9 Nov 2015 15:55:44 +0100 Subject: [PATCH 046/910] mvapich2 : changed method name for consistency. Removed possibly duplicated flags --- var/spack/packages/mvapich2/package.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py index 53090ee05ca..dc2b2cb23f7 100644 --- a/var/spack/packages/mvapich2/package.py +++ b/var/spack/packages/mvapich2/package.py @@ -61,7 +61,7 @@ def enabled(x): """ return '+' + x - def set_build_type_flags(self, spec, configure_args): + def set_build_type(self, spec, configure_args): """ Appends to configure_args the flags that depends only on the build type (i.e. release or debug) @@ -140,14 +140,12 @@ def install(self, spec, prefix): "--enable-shared", "--enable-romio", "--disable-silent-rules", - "--enable-debuginfo", - "--enable-g=dbg" ] if not self.compiler.f77 and not self.compiler.fc: configure_args.append("--enable-fortran=none") - # Set flags that depend only on the type of the build (debug, release) - self.set_build_type_flags(spec, configure_args) + # Set the type of the build (debug, release) + self.set_build_type(spec, configure_args) # Set the process manager self.set_process_manager(spec, configure_args) # Determine network type by variant From 666d9e88f5a500691bf55096d73689ef2a893192 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Tue, 10 Nov 2015 15:37:09 -0800 Subject: [PATCH 047/910] Added Python Twisted library --- var/spack/packages/py-twisted/package.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 var/spack/packages/py-twisted/package.py diff --git a/var/spack/packages/py-twisted/package.py b/var/spack/packages/py-twisted/package.py new file mode 100644 index 00000000000..2fdebb6cb9e --- /dev/null +++ b/var/spack/packages/py-twisted/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyTwisted(Package): + """An asynchronous networking framework written in Python""" + homepage = "https://twistedmatrix.com/" + url = "https://pypi.python.org/packages/source/T/Twisted/Twisted-15.3.0.tar.bz2" + + version('15.4.0', '5337ffb6aeeff3790981a2cd56db9655') + version('15.3.0', 'b58e83da2f00b3352afad74d0c5c4599') + + depends_on('py-setuptools') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 679b617649286c95db15207cfdcf5326cbb151cf Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 11 Nov 2015 12:33:08 -0800 Subject: [PATCH 048/910] Add CleverLeaf package --- var/spack/packages/SAMRAI/package.py | 3 +-- var/spack/packages/cleverleaf/package.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 var/spack/packages/cleverleaf/package.py diff --git a/var/spack/packages/SAMRAI/package.py b/var/spack/packages/SAMRAI/package.py index eef041f0d57..5ee90cf400b 100644 --- a/var/spack/packages/SAMRAI/package.py +++ b/var/spack/packages/SAMRAI/package.py @@ -12,6 +12,7 @@ class Samrai(Package): list_url = homepage version('3.9.1', '232d04d0c995f5abf20d94350befd0b2') + version('3.8.0', 'c18fcffa706346bfa5828b36787ce5fe') version('3.7.3', '12d574eacadf8c9a70f1bb4cd1a69df6') version('3.7.2', 'f6a716f171c9fdbf3cb12f71fa6e2737') version('3.6.3-beta', 'ef0510bf2893042daedaca434e5ec6ce') @@ -37,8 +38,6 @@ def install(self, spec, prefix): configure( "--prefix=%s" % prefix, - "--with-CXX=%s" % spec[mpi].prefix.bin + "/mpic++", - "--with-CC=%s" % spec[mpi].prefix.bin + "/mpicc", "--with-hdf5=%s" % spec['hdf5'].prefix, "--with-boost=%s" % spec['boost'].prefix, "--with-zlib=%s" % spec['zlib'].prefix, diff --git a/var/spack/packages/cleverleaf/package.py b/var/spack/packages/cleverleaf/package.py new file mode 100644 index 00000000000..a01a706c169 --- /dev/null +++ b/var/spack/packages/cleverleaf/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Cleverleaf(Package): + homepage = "http://www.example.com" + url = "http://www.example.com/cleverleaf-1.0.tar.gz" + + version('develop', git='git@github.com:UK-MAC/CleverLeaf_ref.git', branch='develop') + + depends_on("SAMRAI@3.8.0") + + def install(self, spec, prefix): + cmake(*std_cmake_args) + make() + make("install") From 6d7b26d4e08c2ca29f2fe6ff32e63ebfd377d164 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 11 Nov 2015 18:04:22 -0800 Subject: [PATCH 049/910] Insert lib/spack/external into sys.path. This avoids cases where the system python install and lib/spack/external have the same library installed. This requires modifying the names of some modules in lib/spack/external in cases where both the system python and backported features of future python versions (i.e. after 2.6) are used (previously distinguished by "from external import X" and "import X"). --- bin/spack | 2 ++ lib/spack/external/{functools.py => functools_backport.py} | 0 .../external/{ordereddict.py => ordereddict_backport.py} | 0 lib/spack/spack/cmd/activate.py | 2 +- lib/spack/spack/cmd/checksum.py | 2 +- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/cmd/compiler.py | 2 +- lib/spack/spack/cmd/config.py | 2 +- lib/spack/spack/cmd/create.py | 2 +- lib/spack/spack/cmd/deactivate.py | 2 +- lib/spack/spack/cmd/dependents.py | 2 +- lib/spack/spack/cmd/diy.py | 2 +- lib/spack/spack/cmd/env.py | 2 +- lib/spack/spack/cmd/extensions.py | 2 +- lib/spack/spack/cmd/fetch.py | 2 +- lib/spack/spack/cmd/find.py | 2 +- lib/spack/spack/cmd/graph.py | 2 +- lib/spack/spack/cmd/install.py | 2 +- lib/spack/spack/cmd/list.py | 2 +- lib/spack/spack/cmd/load.py | 2 +- lib/spack/spack/cmd/location.py | 2 +- lib/spack/spack/cmd/md5.py | 2 +- lib/spack/spack/cmd/mirror.py | 2 +- lib/spack/spack/cmd/module.py | 2 +- lib/spack/spack/cmd/patch.py | 2 +- lib/spack/spack/cmd/pkg.py | 2 +- lib/spack/spack/cmd/providers.py | 2 +- lib/spack/spack/cmd/python.py | 2 +- lib/spack/spack/cmd/reindex.py | 2 +- lib/spack/spack/cmd/restage.py | 2 +- lib/spack/spack/cmd/spec.py | 2 +- lib/spack/spack/cmd/stage.py | 2 +- lib/spack/spack/cmd/test-install.py | 2 +- lib/spack/spack/cmd/uninstall.py | 2 +- lib/spack/spack/cmd/unload.py | 2 +- lib/spack/spack/cmd/unuse.py | 2 +- lib/spack/spack/cmd/use.py | 2 +- lib/spack/spack/config.py | 6 +++--- lib/spack/spack/database.py | 4 ++-- lib/spack/spack/directory_layout.py | 2 +- lib/spack/spack/spec.py | 4 ++-- lib/spack/spack/test/python_version.py | 2 +- lib/spack/spack/version.py | 2 +- 43 files changed, 46 insertions(+), 44 deletions(-) rename lib/spack/external/{functools.py => functools_backport.py} (100%) rename lib/spack/external/{ordereddict.py => ordereddict_backport.py} (100%) diff --git a/bin/spack b/bin/spack index 127a85f6fe4..f587e206db7 100755 --- a/bin/spack +++ b/bin/spack @@ -38,6 +38,8 @@ SPACK_PREFIX = os.path.dirname(os.path.dirname(SPACK_FILE)) # Allow spack libs to be imported in our scripts SPACK_LIB_PATH = os.path.join(SPACK_PREFIX, "lib", "spack") sys.path.insert(0, SPACK_LIB_PATH) +SPACK_EXTERNAL_LIBS = os.path.join(SPACK_LIB_PATH, "external") +sys.path.insert(0, SPACK_EXTERNAL_LIBS) # If there is no working directory, use the spack prefix. try: diff --git a/lib/spack/external/functools.py b/lib/spack/external/functools_backport.py similarity index 100% rename from lib/spack/external/functools.py rename to lib/spack/external/functools_backport.py diff --git a/lib/spack/external/ordereddict.py b/lib/spack/external/ordereddict_backport.py similarity index 100% rename from lib/spack/external/ordereddict.py rename to lib/spack/external/ordereddict_backport.py diff --git a/lib/spack/spack/cmd/activate.py b/lib/spack/spack/cmd/activate.py index 1004f1f8e6e..1e44948d247 100644 --- a/lib/spack/spack/cmd/activate.py +++ b/lib/spack/spack/cmd/activate.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty import spack import spack.cmd diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 8a448450c2a..c940d577811 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -24,7 +24,7 @@ ############################################################################## import os import re -from external import argparse +import argparse import hashlib from pprint import pprint from subprocess import CalledProcessError diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index c20136ebe5d..e303b3d6342 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 2a64dc914ed..3173d110709 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.color import colorize diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index 8c18f88b648..78972a8be0c 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 46e6bcec14a..05cf170e392 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -27,7 +27,7 @@ import hashlib import re -from external.ordereddict import OrderedDict +from ordereddict_backport import OrderedDict import llnl.util.tty as tty from llnl.util.filesystem import mkdirp diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index 1f0e303cdf2..5a2b353fa2d 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty import spack diff --git a/lib/spack/spack/cmd/dependents.py b/lib/spack/spack/cmd/dependents.py index 652f243b988..129a4eeb237 100644 --- a/lib/spack/spack/cmd/dependents.py +++ b/lib/spack/spack/cmd/dependents.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 9f8a6d39db3..d6bd1fbb79a 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -24,7 +24,7 @@ ############################################################################## import sys import os -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index bde76b5daf6..ae8e95491ed 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from external import argparse +import argparse import llnl.util.tty as tty import spack.cmd import spack.build_environment as build_env diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index 7cadc424b00..86dec7b9e7f 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 0ccebd9486f..1dd8703daf2 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack import spack.cmd diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 0b0dd6ef6fc..05fa620c592 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -25,7 +25,7 @@ import sys import collections import itertools -from external import argparse +import argparse from StringIO import StringIO import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/graph.py b/lib/spack/spack/cmd/graph.py index cb93a1b5433..586a02c53b3 100644 --- a/lib/spack/spack/cmd/graph.py +++ b/lib/spack/spack/cmd/graph.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack import spack.cmd diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 836a6260c87..e4338e222f6 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 1f0978a18ed..0b55d9fb7bc 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -24,7 +24,7 @@ ############################################################################## import sys import llnl.util.tty as tty -from external import argparse +import argparse from llnl.util.tty.colify import colify import spack diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py index 06574d9725d..5bc6b15784c 100644 --- a/lib/spack/spack/cmd/load.py +++ b/lib/spack/spack/cmd/load.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.modules description ="Add package to environment using modules." diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index e8e9c3f2778..6fbab9782fa 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -24,7 +24,7 @@ ############################################################################## import os import sys -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.filesystem import join_path diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index dfa1be412bf..8be0a7ad4c8 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -24,7 +24,7 @@ ############################################################################## import os import hashlib -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.filesystem import * diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 2356170a9ac..4599944f1c2 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -26,7 +26,7 @@ import sys from datetime import datetime -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 654b0cb2fa7..c3daed64022 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -25,7 +25,7 @@ import sys import os import shutil -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.lang import partition_list diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index a6556c4828c..2356583b074 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.cmd import spack diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index 055b7c20623..ae5efd9d9c4 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -24,7 +24,7 @@ ############################################################################## import os -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/providers.py b/lib/spack/spack/cmd/providers.py index 2bcdc9fba2a..1a652c82d17 100644 --- a/lib/spack/spack/cmd/providers.py +++ b/lib/spack/spack/cmd/providers.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from external import argparse +import argparse from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/python.py b/lib/spack/spack/cmd/python.py index 7bd2e45ce07..15c45654c2f 100644 --- a/lib/spack/spack/cmd/python.py +++ b/lib/spack/spack/cmd/python.py @@ -25,7 +25,7 @@ import os import sys import code -from external import argparse +import argparse import platform import spack diff --git a/lib/spack/spack/cmd/reindex.py b/lib/spack/spack/cmd/reindex.py index b584729ea45..c0008930c4b 100644 --- a/lib/spack/spack/cmd/reindex.py +++ b/lib/spack/spack/cmd/reindex.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack description = "Rebuild Spack's package database." diff --git a/lib/spack/spack/cmd/restage.py b/lib/spack/spack/cmd/restage.py index e735a12c327..9230cf5a1a0 100644 --- a/lib/spack/spack/cmd/restage.py +++ b/lib/spack/spack/cmd/restage.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/spec.py b/lib/spack/spack/cmd/spec.py index 407519313ce..af7ec1b36c9 100644 --- a/lib/spack/spack/cmd/spec.py +++ b/lib/spack/spack/cmd/spec.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.cmd import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index f3dc97be171..09cf0e1a1c2 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from external import argparse +import argparse import llnl.util.tty as tty import spack diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 68b761d5dc8..ba0cb39baf2 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import xml.etree.ElementTree as ET import itertools import re diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index e80f2d2636d..93575e005dc 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/unload.py b/lib/spack/spack/cmd/unload.py index 6442c48cb15..24e49b3f24a 100644 --- a/lib/spack/spack/cmd/unload.py +++ b/lib/spack/spack/cmd/unload.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.modules description ="Remove package from environment using module." diff --git a/lib/spack/spack/cmd/unuse.py b/lib/spack/spack/cmd/unuse.py index 2a7229a3a01..7f0b384ea05 100644 --- a/lib/spack/spack/cmd/unuse.py +++ b/lib/spack/spack/cmd/unuse.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.modules description ="Remove package from environment using dotkit." diff --git a/lib/spack/spack/cmd/use.py b/lib/spack/spack/cmd/use.py index e34c1947390..4990fea2f82 100644 --- a/lib/spack/spack/cmd/use.py +++ b/lib/spack/spack/cmd/use.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.modules description ="Add package to environment using dotkit." diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3e91958c2c2..41afe8b2327 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -90,12 +90,12 @@ import exceptions import sys -from external.ordereddict import OrderedDict +from ordereddict_backport import OrderedDict from llnl.util.lang import memoized import spack.error -from external import yaml -from external.yaml.error import MarkedYAMLError +import yaml +from yaml.error import MarkedYAMLError import llnl.util.tty as tty from llnl.util.filesystem import mkdirp diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index e0c14a04553..908ffc7fa4c 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -43,8 +43,8 @@ import time import socket -from external import yaml -from external.yaml.error import MarkedYAMLError, YAMLError +import yaml +from yaml.error import MarkedYAMLError, YAMLError import llnl.util.tty as tty from llnl.util.filesystem import * diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index da8f4187cc5..918405cab69 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -29,7 +29,7 @@ import shutil import glob import tempfile -from external import yaml +import yaml import llnl.util.tty as tty from llnl.util.filesystem import join_path, mkdirp diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 7b79feb311a..92880e9cbf8 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -96,8 +96,8 @@ import base64 from StringIO import StringIO from operator import attrgetter -from external import yaml -from external.yaml.error import MarkedYAMLError +import yaml +from yaml.error import MarkedYAMLError import llnl.util.tty as tty from llnl.util.lang import * diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index 5779d31ed2f..ba7bab6f4b8 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -34,7 +34,7 @@ import llnl.util.tty as tty -from external import pyqver2 +import pyqver2 import spack spack_max_version = (2,6) diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 35db05e018a..ffce2d1ff89 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -48,7 +48,7 @@ import re from bisect import bisect_left from functools import wraps -from external.functools import total_ordering +from functools_backport import total_ordering # Valid version characters VALID_VERSION = r'[A-Za-z0-9_.-]' From 9c30e0210b55a6f5bca048359135ae5a83039de5 Mon Sep 17 00:00:00 2001 From: Stephen Herbein Date: Thu, 12 Nov 2015 08:15:38 -0800 Subject: [PATCH 050/910] Fixed -l flag for 'spack extensions' --- lib/spack/spack/cmd/extensions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index 7cadc424b00..99be4482884 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -37,7 +37,7 @@ def setup_parser(subparser): format_group = subparser.add_mutually_exclusive_group() format_group.add_argument( - '-l', '--long', action='store_const', dest='mode', const='long', + '-l', '--long', action='store_true', dest='long', help='Show dependency hashes as well as versions.') format_group.add_argument( '-p', '--paths', action='store_const', dest='mode', const='paths', @@ -95,4 +95,4 @@ def extensions(parser, args): tty.msg("None activated.") return tty.msg("%d currently activated:" % len(activated)) - spack.cmd.find.display_specs(activated.values(), mode=args.mode) + spack.cmd.find.display_specs(activated.values(), mode=args.mode, long=args.long) From 72c9604bcb9f0f28d0e7d63be9500b99960d118b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 12 Nov 2015 15:17:09 -0800 Subject: [PATCH 051/910] Small fix to prevent this test from interfering with others. --- lib/spack/spack/test/unit_install.py | 39 ++++++++++++++++------------ 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index c4b9092f051..71682729979 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -26,16 +26,16 @@ import itertools import spack -test_install = __import__("spack.cmd.test-install", +test_install = __import__("spack.cmd.test-install", fromlist=["BuildId", "create_test_output", "TestResult"]) class MockOutput(object): def __init__(self): self.results = {} - + def add_test(self, buildId, passed=True, buildInfo=None): self.results[buildId] = passed - + def write_to(self, stream): pass @@ -45,14 +45,14 @@ def __init__(self, name, version, hashStr=None): self.name = name self.version = version self.hash = hashStr if hashStr else hash((name, version)) - + def traverse(self, order=None): - allDeps = itertools.chain.from_iterable(i.traverse() for i in + allDeps = itertools.chain.from_iterable(i.traverse() for i in self.dependencies.itervalues()) return set(itertools.chain([self], allDeps)) - + def dag_hash(self): - return self.hash + return self.hash def to_yaml(self): return "<<>>".format(test_install.BuildId(self).stringId()) @@ -75,47 +75,52 @@ class UnitInstallTest(unittest.TestCase): def setUp(self): super(UnitInstallTest, self).setUp() - + pkgX.installed = False pkgY.installed = False + self.saved_db = spack.db pkgDb = MockPackageDb({specX:pkgX, specY:pkgY}) spack.db = pkgDb + def tearDown(self): super(UnitInstallTest, self).tearDown() - + + spack.db = self.saved_db + def test_installing_both(self): mo = MockOutput() - + pkgX.installed = True pkgY.installed = True test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=test_fetch_log) - - self.assertEqual(mo.results, - {bIdX:test_install.TestResult.PASSED, + + self.assertEqual(mo.results, + {bIdX:test_install.TestResult.PASSED, bIdY:test_install.TestResult.PASSED}) + def test_dependency_already_installed(self): mo = MockOutput() - + pkgX.installed = True pkgY.installed = True test_install.create_test_output(specX, [specX], mo, getLogFunc=test_fetch_log) - + self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED}) #TODO: add test(s) where Y fails to install + class MockPackageDb(object): def __init__(self, init=None): self.specToPkg = {} if init: self.specToPkg.update(init) - + def get(self, spec): return self.specToPkg[spec] def test_fetch_log(path): return [] - From 38fdd063d9bc90aff7d934ab2bd2f02df71d9138 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 12 Nov 2015 15:17:39 -0800 Subject: [PATCH 052/910] Fix and move NamespaceTrie to spack.util.naming - fix up routines in namespace trie. - trie can now hold intermediate elements. - trie now has a test case. --- lib/spack/spack/packages.py | 41 +------------ lib/spack/spack/test/__init__.py | 3 +- lib/spack/spack/test/namespace_trie.py | 83 ++++++++++++++++++++++++++ lib/spack/spack/util/naming.py | 69 ++++++++++++++++++++- 4 files changed, 154 insertions(+), 42 deletions(-) create mode 100644 lib/spack/spack/test/namespace_trie.py diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 6005523bc04..3a74ad67908 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -58,45 +58,6 @@ def converter(self, spec_like, *args, **kwargs): return converter -class NamespaceTrie(object): - def __init__(self): - self._elements = {} - - - def __setitem__(self, namespace, repo): - parts = namespace.split('.') - cur = self._elements - for p in parts[:-1]: - if p not in cur: - cur[p] = {} - cur = cur[p] - - cur[parts[-1]] = repo - - - def __getitem__(self, namespace): - parts = namespace.split('.') - cur = self._elements - for p in parts: - if p not in cur: - raise KeyError("Can't find namespace %s in trie" % namespace) - cur = cur[p] - return cur - - - def __contains__(self, namespace): - parts = namespace.split('.') - cur = self._elements - for p in parts: - if not isinstance(cur, dict): - return False - if p not in cur: - return False - cur = cur[p] - return True - - - class PackageFinder(object): """A PackageFinder is a wrapper around a list of PackageDBs. @@ -172,7 +133,7 @@ def all_packages(self): def providers_for(self, vpkg_name): - # TODO: THIS IS WRONG; shoudl use more than first repo + # TODO: THIS IS WRONG; should use more than first repo return self.repos[0].providers_for(vpkg_name) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 0f776bfea46..620d1fd3623 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -59,7 +59,8 @@ 'configure_guess', 'unit_install', 'lock', - 'database'] + 'database', + 'namespace_trie'] def list_tests(): diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py new file mode 100644 index 00000000000..191abbe9e63 --- /dev/null +++ b/lib/spack/spack/test/namespace_trie.py @@ -0,0 +1,83 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import unittest +from spack.util.naming import NamespaceTrie + + +class NamespaceTrieTest(unittest.TestCase): + + def setUp(self): + self.trie = NamespaceTrie() + + + def test_add_single(self): + self.trie['foo'] = 'bar' + self.assertEqual(self.trie['foo'], 'bar') + self.assertTrue('foo' in self.trie) + + + def test_add_multiple(self): + self.trie['foo.bar'] = 'baz' + self.assertEqual(self.trie['foo.bar'], 'baz') + + self.assertFalse('foo' in self.trie) + self.assertFalse('foo.bar.baz' in self.trie) + self.assertTrue('foo.bar' in self.trie) + + + def test_add_three(self): + # add a three-level namespace + self.trie['foo.bar.baz'] = 'quux' + self.assertEqual(self.trie['foo.bar.baz'], 'quux') + + self.assertFalse('foo' in self.trie) + self.assertFalse('foo.bar' in self.trie) + self.assertTrue('foo.bar.baz' in self.trie) + + # Try to add a second element in a higher space + self.trie['foo.bar'] = 'blah' + + self.assertFalse('foo' in self.trie) + + self.assertTrue('foo.bar' in self.trie) + self.assertEqual(self.trie['foo.bar'], 'blah') + + self.assertTrue('foo.bar.baz' in self.trie) + self.assertEqual(self.trie['foo.bar.baz'], 'quux') + + + def test_add_none_single(self): + self.trie['foo'] = None + self.assertEqual(self.trie['foo'], None) + self.assertTrue('foo' in self.trie) + + + def test_add_none_multiple(self): + self.trie['foo.bar'] = None + self.assertEqual(self.trie['foo.bar'], None) + + self.assertFalse('foo' in self.trie) + self.assertFalse('foo.bar.baz' in self.trie) + self.assertTrue('foo.bar' in self.trie) diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py index a7b6e2b4361..475062bb38a 100644 --- a/lib/spack/spack/util/naming.py +++ b/lib/spack/spack/util/naming.py @@ -3,11 +3,12 @@ import string import itertools import re +from StringIO import StringIO import spack __all__ = ['mod_to_class', 'spack_module_to_python_module', 'valid_module_name', - 'validate_module_name', 'possible_spack_module_names'] + 'validate_module_name', 'possible_spack_module_names', 'NamespaceTrie'] # Valid module names can contain '-' but can't start with it. _valid_module_re = r'^\w[\w-]*$' @@ -90,3 +91,69 @@ def __init__(self, name): super(InvalidModuleNameError, self).__init__( "Invalid module name: " + name) self.name = name + + +class NamespaceTrie(object): + class Element(object): + def __init__(self, value): + self.value = value + + + def __init__(self, separator='.'): + self._subspaces = {} + self._value = None + self._sep = separator + + + def __setitem__(self, namespace, value): + first, sep, rest = namespace.partition(self._sep) + + if not first: + self._value = NamespaceTrie.Element(value) + return + + if first not in self._subspaces: + self._subspaces[first] = NamespaceTrie() + + self._subspaces[first][rest] = value + + + def _get_helper(self, namespace, full_name): + first, sep, rest = namespace.partition(self._sep) + if not first: + if not self._value: + raise KeyError("Can't find namespace '%s' in trie" % full_name) + return self._value.value + elif first not in self._subspaces: + raise KeyError("Can't find namespace '%s' in trie" % full_name) + else: + return self._subspaces[first]._get_helper(rest, full_name) + + + def __getitem__(self, namespace): + return self._get_helper(namespace, namespace) + + + def __contains__(self, namespace): + first, sep, rest = namespace.partition(self._sep) + if not first: + return self._value is not None + elif first not in self._subspaces: + return False + else: + return rest in self._subspaces[first] + + + def _str_helper(self, stream, level=0): + indent = (level * ' ') + for name in sorted(self._subspaces): + stream.write(indent + name + '\n') + if self._value: + stream.write(indent + ' ' + repr(self._value.value)) + stream.write(self._subspaces[name]._str_helper(stream, level+1)) + + + def __str__(self): + stream = StringIO() + self._str_helper(stream) + return stream.getvalue() From 8feaefadcfcae04ccaa204b93502e23158f7a7d4 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Fri, 13 Nov 2015 09:39:00 -0800 Subject: [PATCH 053/910] updating lua package to install in correct order --- var/spack/packages/lua/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/lua/package.py b/var/spack/packages/lua/package.py index 57c443cc2d6..fa4b4f9edcb 100644 --- a/var/spack/packages/lua/package.py +++ b/var/spack/packages/lua/package.py @@ -22,5 +22,7 @@ class Lua(Package): def install(self, spec, prefix): make('INSTALL_TOP=%s' % prefix, 'MYLDFLAGS=-L%s/lib' % spec['ncurses'].prefix, - 'linux', + 'linux') + make('INSTALL_TOP=%s' % prefix, + 'MYLDFLAGS=-L%s/lib' % spec['ncurses'].prefix, 'install') From ca01adb50319970066ee62876a9b37481f7bbe22 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 16 Nov 2015 11:30:35 -0500 Subject: [PATCH 054/910] libxml2: make the python dependency optional Also turn it off by default since it is usually not necessary. Anything needing the Python bindings should declare it explicitly. --- var/spack/packages/libxml2/package.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/libxml2/package.py b/var/spack/packages/libxml2/package.py index df311bfabaa..5d84fad22d2 100644 --- a/var/spack/packages/libxml2/package.py +++ b/var/spack/packages/libxml2/package.py @@ -9,13 +9,20 @@ class Libxml2(Package): version('2.9.2', '9e6a9aca9d155737868b3dc5fd82f788') - extends('python') + variant('python', default=False, description='Enable Python support') + + extends('python', when='+python') depends_on('zlib') depends_on('xz') def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-python=%s" % spec['python'].prefix) + if '+python' in spec: + python_arg = "--with-python=%s" % spec['python'].prefix + else: + python_arg = "--without-python" + + configure("--prefix=%s" % prefix, + python_arg) make() make("install") From 45d981caab32024b6d05909f777abd257777ddb7 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 16 Nov 2015 11:45:27 -0500 Subject: [PATCH 055/910] paraview: add dependency on libxml2 back --- var/spack/packages/paraview/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/paraview/package.py b/var/spack/packages/paraview/package.py index a0ff812ca23..b341053dd26 100644 --- a/var/spack/packages/paraview/package.py +++ b/var/spack/packages/paraview/package.py @@ -30,7 +30,7 @@ class Paraview(Package): depends_on('jpeg') depends_on('libpng') depends_on('libtiff') - #depends_on('libxml2') # drags in python + depends_on('libxml2') depends_on('netcdf') #depends_on('protobuf') # version mismatches? #depends_on('sqlite') # external version not supported @@ -63,7 +63,7 @@ def nfeature_to_bool(feature): '-DVTK_USER_SYSTEM_FREETYPE:BOOL=ON', '-DVTK_USER_SYSTEM_HDF5:BOOL=ON', '-DVTK_USER_SYSTEM_JPEG:BOOL=ON', - #'-DVTK_USER_SYSTEM_LIBXML2:BOOL=ON', + '-DVTK_USER_SYSTEM_LIBXML2:BOOL=ON', '-DVTK_USER_SYSTEM_NETCDF:BOOL=ON', '-DVTK_USER_SYSTEM_TIFF:BOOL=ON', '-DVTK_USER_SYSTEM_ZLIB:BOOL=ON', From d3ead7061ef8862c347d7acca24e844ba7546b45 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 16 Nov 2015 17:04:35 -0500 Subject: [PATCH 056/910] paraview: require qt4 Qt5 has bugs which need at least 5.6.0 for a fix. --- var/spack/packages/paraview/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/paraview/package.py b/var/spack/packages/paraview/package.py index a0ff812ca23..4ca819c1ead 100644 --- a/var/spack/packages/paraview/package.py +++ b/var/spack/packages/paraview/package.py @@ -22,7 +22,7 @@ class Paraview(Package): depends_on('py-matplotlib', when='+python+matplotlib') depends_on('tcl', when='+tcl') depends_on('mpi', when='+mpi') - depends_on('qt', when='+qt') + depends_on('qt@:4', when='+qt') depends_on('bzip2') depends_on('freetype') From 485528cffa42960815fb4716d6a468cde43bf501 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 16 Nov 2015 17:05:17 -0500 Subject: [PATCH 057/910] paraview: configure the python and MPI executables --- var/spack/packages/paraview/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/packages/paraview/package.py b/var/spack/packages/paraview/package.py index 4ca819c1ead..1c44a15a1a9 100644 --- a/var/spack/packages/paraview/package.py +++ b/var/spack/packages/paraview/package.py @@ -49,7 +49,11 @@ def nfeature_to_bool(feature): feature_args = std_cmake_args[:] feature_args.append('-DPARAVIEW_BUILD_QT_GUI:BOOL=%s' % feature_to_bool('+qt')) feature_args.append('-DPARAVIEW_ENABLE_PYTHON:BOOL=%s' % feature_to_bool('+python')) + if '+python' in spec: + feature_args.append('-DPYTHON_EXECUTABLE:FILEPATH=%s/bin/python' % spec['python'].prefix) feature_args.append('-DPARAVIEW_USE_MPI:BOOL=%s' % feature_to_bool('+mpi')) + if '+mpi' in spec: + feature_args.append('-DMPIEXEC:FILEPATH=%s/bin/mpiexec' % spec['mpi'].prefix) feature_args.append('-DVTK_ENABLE_TCL_WRAPPING:BOOL=%s' % feature_to_bool('+tcl')) feature_args.append('-DVTK_OPENGL_HAS_OSMESA:BOOL=%s' % feature_to_bool('+osmesa')) feature_args.append('-DVTK_USE_X:BOOL=%s' % nfeature_to_bool('+osmesa')) From d40897404ba9b751cf8fe18152e30eec1afc12c1 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 16 Nov 2015 17:02:21 -0500 Subject: [PATCH 058/910] py-matplotlib: make the gui and ipython support optional ipython, nice as it is, is not required to use matplotlib. The UI bits are also optional and dragging in Qt and Tcl/Tk for the library shouldn't be the default. --- var/spack/packages/py-matplotlib/package.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py index e7ce3dfd24a..a5fee39d42e 100644 --- a/var/spack/packages/py-matplotlib/package.py +++ b/var/spack/packages/py-matplotlib/package.py @@ -9,10 +9,13 @@ class PyMatplotlib(Package): version('1.4.2', '7d22efb6cce475025733c50487bd8898') version('1.4.3', '86af2e3e3c61849ac7576a6f5ca44267') + variant('gui', default=False, description='Enable GUI') + variant('ipython', default=False, description='Enable ipython support') + extends('python', ignore=r'bin/nosetests.*$') - depends_on('py-pyside') - depends_on('py-ipython') + depends_on('py-pyside', when='+gui') + depends_on('py-ipython', when='+ipython') depends_on('py-pyparsing') depends_on('py-six') depends_on('py-dateutil') @@ -20,10 +23,10 @@ class PyMatplotlib(Package): depends_on('py-nose') depends_on('py-numpy') - depends_on('qt') + depends_on('qt', when='+gui') depends_on('bzip2') - depends_on('tcl') - depends_on('tk') + depends_on('tcl', when='+gui') + depends_on('tk', when='+gui') depends_on('qhull') def install(self, spec, prefix): From 5eb8e1eec8adda3d48b762cd224fc86089ba35f0 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 16 Nov 2015 17:03:34 -0500 Subject: [PATCH 059/910] py-pyside: update docstring --- var/spack/packages/py-pyside/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py index bb5da44d027..ffa433e18e9 100644 --- a/var/spack/packages/py-pyside/package.py +++ b/var/spack/packages/py-pyside/package.py @@ -2,7 +2,7 @@ import os class PyPyside(Package): - """array processing for numbers, strings, records, and objects.""" + """Python bindings for Qt.""" homepage = "https://pypi.python.org/pypi/pyside" url = "https://pypi.python.org/packages/source/P/PySide/PySide-1.2.2.tar.gz" From b98c25c1fba8b4e7f11d0c2094cf3f05ef2fe5b6 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 16 Nov 2015 17:03:47 -0500 Subject: [PATCH 060/910] paraview: update matplotlib and numpy deps --- var/spack/packages/paraview/package.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/var/spack/packages/paraview/package.py b/var/spack/packages/paraview/package.py index a0ff812ca23..8c860bf65b8 100644 --- a/var/spack/packages/paraview/package.py +++ b/var/spack/packages/paraview/package.py @@ -7,8 +7,6 @@ class Paraview(Package): version('4.4.0', 'fa1569857dd680ebb4d7ff89c2227378', url='http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz') variant('python', default=False, description='Enable Python support') - variant('matplotlib', default=False, description='Enable Matplotlib support') - variant('numpy', default=False, description='Enable NumPy support') variant('tcl', default=False, description='Enable TCL support') @@ -18,8 +16,8 @@ class Paraview(Package): variant('qt', default=False, description='Enable Qt support') depends_on('python', when='+python') - depends_on('py-numpy', when='+python+numpy') - depends_on('py-matplotlib', when='+python+matplotlib') + depends_on('py-numpy', when='+python') + depends_on('py-matplotlib', when='+python') depends_on('tcl', when='+tcl') depends_on('mpi', when='+mpi') depends_on('qt', when='+qt') From ad0a1bf70dc2776c88115389400fd6958e49ecc8 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 17 Nov 2015 16:47:47 -0500 Subject: [PATCH 061/910] Add rsync package --- var/spack/packages/rsync/package.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 var/spack/packages/rsync/package.py diff --git a/var/spack/packages/rsync/package.py b/var/spack/packages/rsync/package.py new file mode 100644 index 00000000000..726633dd5e7 --- /dev/null +++ b/var/spack/packages/rsync/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Rsync(Package): + """rsync is an open source utility that provides fast incremental file transfer.""" + homepage = "https://rsync.samba.org" + url = "https://download.samba.org/pub/rsync/rsync-3.1.1.tar.gz" + + version('3.1.1', '43bd6676f0b404326eee2d63be3cdcfe') + + # depends_on("foo") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") From 1003b847b66c8e53f70459df58672cf0f1ff368e Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 17 Nov 2015 17:10:04 -0500 Subject: [PATCH 062/910] Remove comment --- var/spack/packages/rsync/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/var/spack/packages/rsync/package.py b/var/spack/packages/rsync/package.py index 726633dd5e7..8ae21b1cb9b 100644 --- a/var/spack/packages/rsync/package.py +++ b/var/spack/packages/rsync/package.py @@ -7,8 +7,6 @@ class Rsync(Package): version('3.1.1', '43bd6676f0b404326eee2d63be3cdcfe') - # depends_on("foo") - def install(self, spec, prefix): configure('--prefix=%s' % prefix) From cc49f9b6576234fbfa2de05df15f1dbb43b4926f Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 17 Nov 2015 18:28:20 -0500 Subject: [PATCH 063/910] Update OpenSSL package --- var/spack/packages/openssl/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index c5a8aeb9dc2..4bf1f832d45 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -10,6 +10,7 @@ class Openssl(Package): url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz" version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') + version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') depends_on("zlib") parallel = False From b2a85bc12ed9c0749f858d95cb81ef733befab8a Mon Sep 17 00:00:00 2001 From: Tru Huynh Date: Wed, 18 Nov 2015 10:02:22 +0100 Subject: [PATCH 064/910] updated to version 8u66 --- var/spack/packages/jdk/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/jdk/package.py b/var/spack/packages/jdk/package.py index 8f8076dd146..f8f5fc21bdd 100644 --- a/var/spack/packages/jdk/package.py +++ b/var/spack/packages/jdk/package.py @@ -14,8 +14,8 @@ class Jdk(Package): in the form of a binary product aimed at Java developers.""" homepage = "http://www.oracle.com/technetwork/java/javase/downloads/index.html" - version('8u25-linux-x64', 'e145c03a7edc845215092786bcfba77e', - url="http://download.oracle.com/otn-pub/java/jdk/8u25-b17/jdk-8u25-linux-x64.tar.gz") + version('8u66-linux-x64', '88f31f3d642c3287134297b8c10e61bf', + url="http://download.oracle.com/otn-pub/java/jdk/8u66-b17/jdk-8u66-linux-x64.tar.gz") # Oracle requires that you accept their License Agreement in order # to access the Java packages in download.oracle.com. In order to From 893e05540c640c4598477a39688a773556bebad9 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 18 Nov 2015 09:55:45 -0600 Subject: [PATCH 065/910] Update HDF5 to 1.8.16 --- var/spack/packages/hdf5/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/hdf5/package.py b/var/spack/packages/hdf5/package.py index 15e0ef93388..7666210d068 100644 --- a/var/spack/packages/hdf5/package.py +++ b/var/spack/packages/hdf5/package.py @@ -11,6 +11,7 @@ class Hdf5(Package): list_url = "http://www.hdfgroup.org/ftp/HDF5/releases" list_depth = 3 + version('1.8.16', 'b8ed9a36ae142317f88b0c7ef4b9c618') version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') version('1.8.13', 'c03426e9e77d7766944654280b467289') From af9fe0a03835401efaa7bee1c4b148e878bad3cc Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 18 Nov 2015 12:37:54 -0500 Subject: [PATCH 066/910] Update hwloc to 1.11.1 --- var/spack/packages/hwloc/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/packages/hwloc/package.py b/var/spack/packages/hwloc/package.py index 31a31f376a2..2f46399afd2 100644 --- a/var/spack/packages/hwloc/package.py +++ b/var/spack/packages/hwloc/package.py @@ -15,6 +15,8 @@ class Hwloc(Package): homepage = "http://www.open-mpi.org/projects/hwloc/" url = "http://www.open-mpi.org/software/hwloc/v1.9/downloads/hwloc-1.9.tar.gz" + version('1.11.1', '002742efd3a8431f98d6315365a2b543', + url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.1.tar.bz2') version('1.9', '1f9f9155682fe8946a97c08896109508') def install(self, spec, prefix): From e36fd7d29f3b2726c77ca6028135ca6d93376cae Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 18 Nov 2015 13:13:13 -0500 Subject: [PATCH 067/910] Update OpenMPI to 1.10.1; correct configure error in OpenMPI --- var/spack/packages/openmpi/configure.patch | 31 ++++++++++++++++++++++ var/spack/packages/openmpi/package.py | 9 +++++++ 2 files changed, 40 insertions(+) create mode 100644 var/spack/packages/openmpi/configure.patch diff --git a/var/spack/packages/openmpi/configure.patch b/var/spack/packages/openmpi/configure.patch new file mode 100644 index 00000000000..18fb42c1d16 --- /dev/null +++ b/var/spack/packages/openmpi/configure.patch @@ -0,0 +1,31 @@ +This patch addresses . +--- a/configure ++++ b/configure +@@ -301130,10 +301130,11 @@ + case ${prev}${p} in + + -L* | -R* | -l*) +- # Some compilers place space between "-{L,R}" and the path. ++ # Some compilers place space between "-{L,R,l}" and the path. + # Remove the space. + if test $p = "-L" || +- test $p = "-R"; then ++ test $p = "-R" || ++ test $p = "-l"; then + prev=$p + continue + fi +@@ -303036,10 +303037,11 @@ + case ${prev}${p} in + + -L* | -R* | -l*) +- # Some compilers place space between "-{L,R}" and the path. ++ # Some compilers place space between "-{L,R,l}" and the path. + # Remove the space. + if test $p = "-L" || +- test $p = "-R"; then ++ test $p = "-R" || ++ test $p = "-l"; then + prev=$p + continue + fi diff --git a/var/spack/packages/openmpi/package.py b/var/spack/packages/openmpi/package.py index 5e429dedf5c..be2202fbbd7 100644 --- a/var/spack/packages/openmpi/package.py +++ b/var/spack/packages/openmpi/package.py @@ -14,6 +14,8 @@ class Openmpi(Package): homepage = "http://www.open-mpi.org" + version('1.10.1', 'f0fcd77ed345b7eafb431968124ba16e', + url = "http://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-1.10.1.tar.bz2") version('1.10.0', '280cf952de68369cebaca886c5ce0304', url = "http://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-1.10.0.tar.bz2") version('1.8.8', '0dab8e602372da1425e9242ae37faf8c', @@ -23,10 +25,15 @@ class Openmpi(Package): patch('ad_lustre_rwcontig_open_source.patch', when="@1.6.5") patch('llnl-platforms.patch', when="@1.6.5") + patch('configure.patch', when="@1.10.0:") provides('mpi@:2.2', when='@1.6.5') # Open MPI 1.6.5 supports MPI-2.2 provides('mpi@:3.0', when='@1.8.8') # Open MPI 1.8.8 supports MPI-3.0 provides('mpi@:3.0', when='@1.10.0') # Open MPI 1.10.0 supports MPI-3.0 + provides('mpi@:3.0', when='@1.10.1') # Open MPI 1.10.1 supports MPI-3.0 + + + depends_on('hwloc') def setup_dependent_environment(self, module, spec, dep_spec): @@ -40,6 +47,8 @@ def setup_dependent_environment(self, module, spec, dep_spec): def install(self, spec, prefix): config_args = ["--prefix=%s" % prefix] + config_args.append("--with-hwloc=%s" % spec['hwloc'].prefix) + # TODO: use variants for this, e.g. +lanl, +llnl, etc. # use this for LANL builds, but for LLNL builds, we need: # "--with-platform=contrib/platform/llnl/optimized" From 511ffc3b44d8ef9f9673c2a37c6cd5385bf9eea2 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 18 Nov 2015 13:33:04 -0500 Subject: [PATCH 068/910] HDF5: Support non-MPICH MPI implementations --- var/spack/packages/hdf5/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/hdf5/package.py b/var/spack/packages/hdf5/package.py index 15e0ef93388..e1cd8fdabb7 100644 --- a/var/spack/packages/hdf5/package.py +++ b/var/spack/packages/hdf5/package.py @@ -25,8 +25,8 @@ def install(self, spec, prefix): "--with-zlib=%s" % spec['zlib'].prefix, "--enable-parallel", "--enable-shared", - "CC=%s" % spec['mpich'].prefix.bin + "/mpicc", - "CXX=%s" % spec['mpich'].prefix.bin + "/mpic++") + "CC=%s" % spec['mpi'].prefix.bin + "/mpicc", + "CXX=%s" % spec['mpi'].prefix.bin + "/mpic++") make() make("install") From cbc552e91e487ad0933e04a0cc73cc7d4f0b4061 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 18 Nov 2015 13:43:49 -0500 Subject: [PATCH 069/910] Add jemalloc package --- var/spack/packages/jemalloc/package.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 var/spack/packages/jemalloc/package.py diff --git a/var/spack/packages/jemalloc/package.py b/var/spack/packages/jemalloc/package.py new file mode 100644 index 00000000000..ff946ae821d --- /dev/null +++ b/var/spack/packages/jemalloc/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Jemalloc(Package): + """jemalloc is a general purpose malloc(3) implementation that emphasizes fragmentation avoidance and scalable concurrency support.""" + homepage = "http://www.canonware.com/jemalloc/" + url = "https://github.com/jemalloc/jemalloc/releases/download/4.0.4/jemalloc-4.0.4.tar.bz2" + + version('4.0.4', '687c5cc53b9a7ab711ccd680351ff988') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") From 2e51e207f9450af257f4795ecf4c405c9b23b3e7 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 18 Nov 2015 15:17:48 -0500 Subject: [PATCH 070/910] Update cmake to 3.4.0 --- var/spack/packages/cmake/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/packages/cmake/package.py b/var/spack/packages/cmake/package.py index 9efa370c8bb..fcfbca07056 100644 --- a/var/spack/packages/cmake/package.py +++ b/var/spack/packages/cmake/package.py @@ -34,6 +34,9 @@ class Cmake(Package): version('3.0.2', 'db4c687a31444a929d2fdc36c4dfb95f', url = 'http://www.cmake.org/files/v3.0/cmake-3.0.2.tar.gz') + + version('3.4.0', 'cd3034e0a44256a0917e254167217fc8', + url = 'https://cmake.org/files/v3.4/cmake-3.4.0.tar.gz') # version('3.0.1', 'e2e05d84cb44a42f1371d9995631dcf5') # version('3.0.0', '21a1c85e1a3b803c4b48e7ff915a863e') From b6d368d7a05551cdd3470ee495293ff7e4f894ba Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 19 Nov 2015 17:50:30 -0500 Subject: [PATCH 071/910] dbus: disable launchd support --- var/spack/packages/dbus/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/dbus/package.py b/var/spack/packages/dbus/package.py index f7c302d611a..294b0de54ea 100644 --- a/var/spack/packages/dbus/package.py +++ b/var/spack/packages/dbus/package.py @@ -22,7 +22,8 @@ class Dbus(Package): def install(self, spec, prefix): configure( "--prefix=%s" % prefix, - "--disable-systemd") + "--disable-systemd", + "--disable-launchd") make() make("install") From 150e43f3fdb99700288ddfda8b42dfb95d476eb7 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 19 Nov 2015 17:51:44 -0500 Subject: [PATCH 072/910] WIP: openssl: set KERNEL_BITS to make ./config work Why OpenSSL refuses to build without this (it explicitly warns about it) is a mystery. --- var/spack/packages/openssl/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index c5a8aeb9dc2..f17d0a2794e 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -15,6 +15,8 @@ class Openssl(Package): parallel = False def install(self, spec, prefix): + env['KERNEL_BITS'] = '64' # Only on 64-bit Mac + config = Executable("./config") config("--prefix=%s" % prefix, "--openssldir=%s/etc/openssl" % prefix, From c023b1000e6f9044e40080f2bf10f0406ba6ec28 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 19 Nov 2015 17:52:22 -0500 Subject: [PATCH 073/910] python: target OS X 10.6 Targeting anything older lacks rpath stuff which configure uses. --- var/spack/packages/python/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 000881a8469..31fb8b3d5ab 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -26,6 +26,7 @@ class Python(Package): def install(self, spec, prefix): # Need this to allow python build to find the Python installation. env['PYTHONHOME'] = prefix + env['MACOSX_DEPLOYMENT_TARGET'] = '10.6' # Rest of install is pretty standard. configure("--prefix=%s" % prefix, From 4edcc59c3dc6f5c5121e0b453f39ab90ec5af3ca Mon Sep 17 00:00:00 2001 From: Abhinav Bhatele Date: Thu, 19 Nov 2015 23:05:21 -0800 Subject: [PATCH 074/910] Add damselfly package --- var/spack/packages/damselfly/package.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 var/spack/packages/damselfly/package.py diff --git a/var/spack/packages/damselfly/package.py b/var/spack/packages/damselfly/package.py new file mode 100644 index 00000000000..54df528c4a7 --- /dev/null +++ b/var/spack/packages/damselfly/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Damselfly(Package): + """Damselfly is a model-based parallel network simulator.""" + homepage = "https://github.com/scalability-llnl/damselfly" + url = "https://github.com/scalability-llnl/damselfly" + + version('1.0', '05cf7e2d8ece4408c0f2abb7ab63fd74c0d62895', git='https://github.com/scalability-llnl/damselfly.git', tag='v1.0') + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('-DCMAKE_BUILD_TYPE=release', '..', *std_cmake_args) + make() + make('install') From 5e93dd65929e97559acdb1227e5418b7896bb244 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Fri, 20 Nov 2015 11:49:46 -0800 Subject: [PATCH 075/910] modified warning to reference spack clean clean --dist has been removed, command fixed to something that works. --- lib/spack/spack/fetch_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index b810023c5a9..5e6850d14b6 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -190,7 +190,7 @@ def fetch(self): if content_types and 'text/html' in content_types[-1]: tty.warn("The contents of " + self.archive_file + " look like HTML.", "The checksum will likely be bad. If it is, you can use", - "'spack clean --dist' to remove the bad archive, then fix", + "'spack clean ' to remove the bad archive, then fix", "your internet gateway issue and install again.") if not self.archive_file: From 34700d560f441d5f2af1fd26bad5f9a6e80fcefe Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Fri, 20 Nov 2015 12:14:21 -0800 Subject: [PATCH 076/910] add judy dynamic array/meta-trie/hash table lib Just adding judy. --- var/spack/packages/judy/package.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 var/spack/packages/judy/package.py diff --git a/var/spack/packages/judy/package.py b/var/spack/packages/judy/package.py new file mode 100644 index 00000000000..b8d8701383c --- /dev/null +++ b/var/spack/packages/judy/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Judy(Package): + """A general-purpose dynamic array, associative array and hash-trie - Judy""" + homepage = "http://judy.sourceforge.net/" + url = "http://downloads.sourceforge.net/project/judy/judy/Judy-1.0.5/Judy-1.0.5.tar.gz" + + version('1.0.5', '115a0d26302676e962ae2f70ec484a54') + parallel = False + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") From 8c06b92225699fdd85185688b88d559a54c8a794 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 22 Nov 2015 20:28:59 -0800 Subject: [PATCH 077/910] Working PackageFinder / Repo --- .mailmap | 2 + lib/spack/spack/__init__.py | 6 - lib/spack/spack/packages.py | 381 +++++++++++++++-------- lib/spack/spack/patch.py | 2 +- lib/spack/spack/test/config.py | 1 - lib/spack/spack/test/namespace_trie.py | 61 +++- lib/spack/spack/test/package_sanity.py | 5 +- lib/spack/spack/test/packages.py | 6 +- lib/spack/spack/test/url_substitution.py | 1 - lib/spack/spack/util/naming.py | 32 +- 10 files changed, 334 insertions(+), 163 deletions(-) diff --git a/.mailmap b/.mailmap index 1cc13c1eb14..1b99da32b5b 100644 --- a/.mailmap +++ b/.mailmap @@ -9,3 +9,5 @@ Saravan Pantham Saravan Pantham Tom Scogland Tom Scogland Tom Scogland Joachim Protze jprotze +Gregory L. Lee Gregory L. Lee +Gregory L. Lee Gregory Lee diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index eccec12d3bd..549d0a9a0fc 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -51,12 +51,6 @@ install_path = join_path(opt_path, "spack") etc_path = join_path(prefix, "etc") -# -# Setup the spack.repos namespace -# -from spack.repo_loader import RepoNamespace -repos = RepoNamespace() - # # Set up the default packages database. # diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 3a74ad67908..8114b7f1aa4 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -31,6 +31,7 @@ import re import itertools import traceback +from bisect import bisect_left from external import yaml import llnl.util.tty as tty @@ -58,11 +59,19 @@ def converter(self, spec_like, *args, **kwargs): return converter -class PackageFinder(object): - """A PackageFinder is a wrapper around a list of PackageDBs. +def _make_namespace_module(ns): + module = imp.new_module(ns) + module.__file__ = "(spack namespace)" + module.__path__ = [] + module.__package__ = ns + return module - It functions exactly like a PackageDB, but it operates on the - combined results of the PackageDBs in its list instead of on a + +class PackageFinder(object): + """A PackageFinder is a wrapper around a list of Repos. + + It functions exactly like a Repo, but it operates on the + combined results of the Repos in its list instead of on a single package repository. """ def __init__(self, *repo_dirs): @@ -70,61 +79,74 @@ def __init__(self, *repo_dirs): self.by_namespace = NamespaceTrie() self.by_path = {} + self._all_package_names = [] + self._provider_index = None + for root in repo_dirs: - repo = PackageDB(root) + repo = Repo(root) self.put_last(repo) - def _check_repo(self, repo): + def swap(self, other): + """Convenience function to make swapping repostiories easier. + + This is currently used by mock tests. + TODO: Maybe there is a cleaner way. + + """ + attrs = ['repos', 'by_namespace', 'by_path', '_all_package_names', '_provider_index'] + for attr in attrs: + tmp = getattr(self, attr) + setattr(self, attr, getattr(other, attr)) + setattr(other, attr, tmp) + + + def _add(self, repo): + """Add a repository to the namespace and path indexes. + + Checks for duplicates -- two repos can't have the same root + directory, and they provide have the same namespace. + + """ if repo.root in self.by_path: raise DuplicateRepoError("Package repos are the same", repo, self.by_path[repo.root]) if repo.namespace in self.by_namespace: - tty.error("Package repos cannot have the same name", - repo, self.by_namespace[repo.namespace]) + raise DuplicateRepoError("Package repos cannot have the same name", + repo, self.by_namespace[repo.namespace]) - - def _add(self, repo): - self._check_repo(repo) + # Add repo to the pkg indexes self.by_namespace[repo.namespace] = repo self.by_path[repo.root] = repo + # add names to the cached name list + new_pkgs = set(repo.all_package_names()) + new_pkgs.update(set(self._all_package_names)) + self._all_package_names = sorted(new_pkgs, key=lambda n:n.lower()) + def put_first(self, repo): + """Add repo first in the search path.""" self._add(repo) self.repos.insert(0, repo) def put_last(self, repo): + """Add repo last in the search path.""" self._add(repo) self.repos.append(repo) def remove(self, repo): + """Remove a repo from the search path.""" if repo in self.repos: self.repos.remove(repo) - def swap(self, other): - repos = self.repos - by_namespace = self.by_namespace - by_path = self.by_path - - self.repos = other.repos - self.by_namespace = other.by_namespace - self.by_pah = other.by_path - - other.repos = repos - other.by_namespace = by_namespace - other.by_path = by_path - - def all_package_names(self): - all_pkgs = set() - for repo in self.repos: - all_pkgs.update(set(repo.all_package_names())) - return all_pkgs + """Return all unique package names in all repositories.""" + return self._all_package_names def all_packages(self): @@ -132,109 +154,82 @@ def all_packages(self): yield self.get(name) - def providers_for(self, vpkg_name): - # TODO: THIS IS WRONG; should use more than first repo - return self.repos[0].providers_for(vpkg_name) + @_autospec + def providers_for(self, vpkg_spec): + if self._provider_index is None: + self._provider_index = ProviderIndex(self.all_package_names()) + + providers = self._provider_index.providers_for(vpkg_spec) + if not providers: + raise UnknownPackageError(vpkg_spec.name) + return providers - def _get_spack_pkg_name(self, repo, py_module_name): - """Allow users to import Spack packages using legal Python identifiers. + def find_module(self, fullname, path=None): + """Implements precedence for overlaid namespaces. - A python identifier might map to many different Spack package - names due to hyphen/underscore ambiguity. + Loop checks each namespace in self.repos for packages, and + also handles loading empty containing namespaces. - Easy example: - num3proxy -> 3proxy - - Ambiguous: - foo_bar -> foo_bar, foo-bar - - More ambiguous: - foo_bar_baz -> foo_bar_baz, foo-bar-baz, foo_bar-baz, foo-bar_baz """ - if py_module_name in repo: - return py_module_name + # namespaces are added to repo, and package modules are leaves. + namespace, dot, module_name = fullname.rpartition('.') - options = possible_spack_module_names(py_module_name) - options.remove(py_module_name) - for name in options: - if name in repo: - return name + # If it's a module in some repo, or if it is the repo's + # namespace, let the repo handle it. + for repo in self.repos: + if namespace == repo.namespace: + if repo.real_name(module_name): + return repo + elif fullname == repo.namespace: + return repo + + # No repo provides the namespace, but it is a valid prefix of + # something in the PackageFinder. + if self.by_namespace.is_prefix(fullname): + return self return None - def find_module(self, fullname, path=None): - if fullname in self.by_namespace: - return self - - namespace, dot, module_name = fullname.rpartition('.') - if namespace not in self.by_namespace: - return None - - repo = self.by_namespace[namespace] - name = self._get_spack_pkg_name(repo, module_name) - if not name: - return None - - return self - - def load_module(self, fullname): + """Loads containing namespaces when necessary. + + See ``Repo`` for how actual package modules are loaded. + """ if fullname in sys.modules: return sys.modules[fullname] - if fullname in self.by_namespace: - ns = self.by_namespace[fullname] - module = imp.new_module(fullname) - module.__file__ = "" - module.__path__ = [] - module.__package__ = fullname + # partition fullname into prefix and module name. + namespace, dot, module_name = fullname.rpartition('.') - else: - namespace, dot, module_name = fullname.rpartition('.') - if namespace not in self.by_namespace: - raise ImportError( - "No Spack repository with namespace %s" % namespace) - - repo = self.by_namespace[namespace] - name = self._get_spack_pkg_name(repo, module_name) - if not name: - raise ImportError( - "No module %s in Spack repository %s" % (module_name, repo)) - - fullname = namespace + '.' + name - file_path = os.path.join(repo.root, name, package_file_name) - module = imp.load_source(fullname, file_path) - module.__package__ = namespace + if not self.by_namespace.is_prefix(fullname): + raise ImportError("No such Spack repo: %s" % fullname) + module = _make_namespace_module(namespace) module.__loader__ = self sys.modules[fullname] = module return module - def _find_repo_for_spec(self, spec): - """Find a repo that contains the supplied spec's package. - - Raises UnknownPackageErrorfor if not found. - """ + def repo_for_pkg(self, pkg_name): for repo in self.repos: - if spec.name in repo: + if pkg_name in repo: return repo - raise UnknownPackageError(spec.name) + raise UnknownPackageError(pkg_name) @_autospec def get(self, spec, new=False): - return self._find_repo_for_spec(spec).get(spec, new) + """Find a repo that contains the supplied spec's package. + + Raises UnknownPackageError if not found. + """ + return self.repo_for_pkg(spec.name).get(spec) - def get_repo(self, namespace): - if namespace in self.by_namespace: - repo = self.by_namespace[namespace] - if isinstance(repo, PackageDB): - return repo - return None + def dirname_for_package_name(self, pkg_name): + return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) def exists(self, pkg_name): @@ -246,7 +241,7 @@ def __contains__(self, pkg_name): -class PackageDB(object): +class Repo(object): """Class representing a package repository in the filesystem. Each package repository must have a top-level configuration file @@ -278,10 +273,107 @@ def __init__(self, root): tty.die(("Invalid namespace '%s' in '%s'. Namespaces must be " "valid python identifiers separated by '.'") % (self.namespace, self.root)) + self._names = self.namespace.split('.') # These are internal cache variables. + self._modules = {} + self._classes = {} self._instances = {} + self._provider_index = None + self._all_package_names = None + + # make sure the namespace for packages in this repo exists. + self._create_namespace() + + + def _create_namespace(self): + """Create this repo's namespace module and insert it into sys.modules. + + Ensures that modules loaded via the repo have a home, and that + we don't get runtime warnings from Python's module system. + + """ + for l in range(1, len(self._names)+1): + ns = '.'.join(self._names[:l]) + if not ns in sys.modules: + sys.modules[ns] = _make_namespace_module(ns) + sys.modules[ns].__loader__ = self + + + def real_name(self, import_name): + """Allow users to import Spack packages using Python identifiers. + + A python identifier might map to many different Spack package + names due to hyphen/underscore ambiguity. + + Easy example: + num3proxy -> 3proxy + + Ambiguous: + foo_bar -> foo_bar, foo-bar + + More ambiguous: + foo_bar_baz -> foo_bar_baz, foo-bar-baz, foo_bar-baz, foo-bar_baz + """ + if import_name in self: + return import_name + + options = possible_spack_module_names(import_name) + options.remove(import_name) + for name in options: + if name in self: + return name + return None + + + def is_prefix(self, fullname): + """True if fullname is a prefix of this Repo's namespace.""" + parts = fullname.split('.') + return self._names[:len(parts)] == parts + + + def find_module(self, fullname, path=None): + """Python find_module import hook. + + Returns this Repo if it can load the module; None if not. + """ + if self.is_prefix(fullname): + return self + + namespace, dot, module_name = fullname.rpartition('.') + if namespace == self.namespace: + if self.real_name(module_name): + return self + + return None + + + def load_module(self, fullname): + """Python importer load hook. + + Tries to load the module; raises an ImportError if it can't. + """ + if fullname in sys.modules: + return sys.modules[fullname] + + namespace, dot, module_name = fullname.rpartition('.') + + if self.is_prefix(fullname): + module = _make_namespace_module(fullname) + + elif namespace == self.namespace: + real_name = self.real_name(module_name) + if not real_name: + raise ImportError("No module %s in repo %s" % (module_name, namespace)) + module = self._get_pkg_module(real_name) + + else: + raise ImportError("No module %s in repo %s" % (fullname, self.namespace)) + + module.__loader__ = self + sys.modules[fullname] = module + return module def _read_config(self): @@ -307,15 +399,14 @@ def get(self, spec, new=False): if spec.virtual: raise UnknownPackageError(spec.name) - if new: - if spec in self._instances: - del self._instances[spec] + if new and spec in self._instances: + del self._instances[spec] if not spec in self._instances: - package_class = self.get_class_for_package_name(spec.name, spec.namespace) + PackageClass = self._get_pkg_class(spec.name) try: copy = spec.copy() - self._instances[copy] = package_class(copy) + self._instances[copy] = PackageClass(copy) except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) @@ -353,7 +444,7 @@ def dirname_for_package_name(self, pkg_name): def filename_for_package_name(self, pkg_name): """Get the filename for the module we should load for a particular - package. Packages for a pacakge DB live in + package. Packages for a Repo live in ``$root//package.py`` This will return a proper package.py path even if the @@ -366,19 +457,20 @@ def filename_for_package_name(self, pkg_name): return join_path(pkg_dir, package_file_name) - @memoized def all_package_names(self): - """Generator function for all packages. This looks for - ``/package.py`` files within the repo direcotories""" - all_package_names = [] + """Returns a sorted list of all package names in the Repo.""" + if self._all_package_names is None: + self._all_package_names = [] - for pkg_name in os.listdir(self.root): - pkg_dir = join_path(self.root, pkg_name) - pkg_file = join_path(pkg_dir, package_file_name) - if os.path.isfile(pkg_file): - all_package_names.append(pkg_name) + for pkg_name in os.listdir(self.root): + pkg_dir = join_path(self.root, pkg_name) + pkg_file = join_path(pkg_dir, package_file_name) + if os.path.isfile(pkg_file): + self._all_package_names.append(pkg_name) - return sorted(all_package_names) + self._all_package_names.sort() + + return self._all_package_names def all_packages(self): @@ -386,27 +478,54 @@ def all_packages(self): yield self.get(name) - @memoized def exists(self, pkg_name): """Whether a package with the supplied name exists.""" - return os.path.exists(self.filename_for_package_name(pkg_name)) + # This does a binary search in the sorted list. + idx = bisect_left(self.all_package_names(), pkg_name) + return self._all_package_names[idx] == pkg_name - @memoized - def get_class_for_package_name(self, pkg_name, reponame = None): - """Get an instance of the class for a particular package.""" - file_path = self.filename_for_package_name(pkg_name) + def _get_pkg_module(self, pkg_name): + """Create a module for a particular package. + + This caches the module within this Repo *instance*. It does + *not* add it to ``sys.modules``. So, you can construct + multiple Repos for testing and ensure that the module will be + loaded once per repo. + + """ + if pkg_name not in self._modules: + file_path = self.filename_for_package_name(pkg_name) + + if not os.path.exists(file_path): + raise UnknownPackageError(pkg_name, self.namespace) - if os.path.exists(file_path): if not os.path.isfile(file_path): tty.die("Something's wrong. '%s' is not a file!" % file_path) + if not os.access(file_path, os.R_OK): tty.die("Cannot read '%s'!" % file_path) - else: - raise UnknownPackageError(pkg_name, self.namespace) + fullname = "%s.%s" % (self.namespace, pkg_name) + + module = imp.load_source(fullname, file_path) + module.__package__ = self.namespace + module.__loader__ = self + self._modules[pkg_name] = module + + return self._modules[pkg_name] + + + def _get_pkg_class(self, pkg_name): + """Get the class for the package out of its module. + + First loads (or fetches from cache) a module for the + package. Then extracts the package class from the module + according to Spack's naming convention. + """ class_name = mod_to_class(pkg_name) - module = __import__(self.namespace + '.' + pkg_name, fromlist=[class_name]) + module = self._get_pkg_module(pkg_name) + cls = getattr(module, class_name) if not inspect.isclass(cls): tty.die("%s.%s is not a class" % (pkg_name, class_name)) @@ -415,7 +534,7 @@ def get_class_for_package_name(self, pkg_name, reponame = None): def __str__(self): - return "" % (self.namespace, self.root) + return "" % (self.namespace, self.root) def __repr__(self): diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index da5fa1646ba..fe6e0a65a38 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -54,7 +54,7 @@ def __init__(self, pkg, path_or_url, level): if '://' in path_or_url: self.url = path_or_url else: - pkg_dir = spack.db.dirname_for_package_name(pkg_name) + pkg_dir = spack.db.dirname_for_package_name(self.pkg_name) self.path = join_path(pkg_dir, path_or_url) if not os.path.isfile(self.path): raise NoSuchPatchFileError(pkg_name, self.path) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index eed182a2573..b1195dfe4e9 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -27,7 +27,6 @@ import os from tempfile import mkdtemp import spack -from spack.packages import PackageDB from spack.test.mock_packages_test import * # Some sample compiler config data diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index 191abbe9e63..2fa13688e65 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -34,50 +34,81 @@ def setUp(self): def test_add_single(self): self.trie['foo'] = 'bar' + + self.assertTrue(self.trie.is_prefix('foo')) + self.assertTrue(self.trie.has_value('foo')) self.assertEqual(self.trie['foo'], 'bar') - self.assertTrue('foo' in self.trie) def test_add_multiple(self): self.trie['foo.bar'] = 'baz' + + self.assertFalse(self.trie.has_value('foo')) + self.assertTrue(self.trie.is_prefix('foo')) + + self.assertTrue(self.trie.is_prefix('foo.bar')) + self.assertTrue(self.trie.has_value('foo.bar')) self.assertEqual(self.trie['foo.bar'], 'baz') - self.assertFalse('foo' in self.trie) - self.assertFalse('foo.bar.baz' in self.trie) - self.assertTrue('foo.bar' in self.trie) + self.assertFalse(self.trie.is_prefix('foo.bar.baz')) + self.assertFalse(self.trie.has_value('foo.bar.baz')) def test_add_three(self): # add a three-level namespace self.trie['foo.bar.baz'] = 'quux' + + self.assertTrue(self.trie.is_prefix('foo')) + self.assertFalse(self.trie.has_value('foo')) + + self.assertTrue(self.trie.is_prefix('foo.bar')) + self.assertFalse(self.trie.has_value('foo.bar')) + + self.assertTrue(self.trie.is_prefix('foo.bar.baz')) + self.assertTrue(self.trie.has_value('foo.bar.baz')) self.assertEqual(self.trie['foo.bar.baz'], 'quux') - self.assertFalse('foo' in self.trie) - self.assertFalse('foo.bar' in self.trie) - self.assertTrue('foo.bar.baz' in self.trie) + self.assertFalse(self.trie.is_prefix('foo.bar.baz.quux')) + self.assertFalse(self.trie.has_value('foo.bar.baz.quux')) - # Try to add a second element in a higher space + # Try to add a second element in a prefix namespace self.trie['foo.bar'] = 'blah' - self.assertFalse('foo' in self.trie) + self.assertTrue(self.trie.is_prefix('foo')) + self.assertFalse(self.trie.has_value('foo')) - self.assertTrue('foo.bar' in self.trie) + self.assertTrue(self.trie.is_prefix('foo.bar')) + self.assertTrue(self.trie.has_value('foo.bar')) self.assertEqual(self.trie['foo.bar'], 'blah') - self.assertTrue('foo.bar.baz' in self.trie) + self.assertTrue(self.trie.is_prefix('foo.bar.baz')) + self.assertTrue(self.trie.has_value('foo.bar.baz')) self.assertEqual(self.trie['foo.bar.baz'], 'quux') + self.assertFalse(self.trie.is_prefix('foo.bar.baz.quux')) + self.assertFalse(self.trie.has_value('foo.bar.baz.quux')) + def test_add_none_single(self): self.trie['foo'] = None + self.assertTrue(self.trie.is_prefix('foo')) + self.assertTrue(self.trie.has_value('foo')) self.assertEqual(self.trie['foo'], None) - self.assertTrue('foo' in self.trie) + + self.assertFalse(self.trie.is_prefix('foo.bar')) + self.assertFalse(self.trie.has_value('foo.bar')) + def test_add_none_multiple(self): self.trie['foo.bar'] = None + + self.assertTrue(self.trie.is_prefix('foo')) + self.assertFalse(self.trie.has_value('foo')) + + self.assertTrue(self.trie.is_prefix('foo.bar')) + self.assertTrue(self.trie.has_value('foo.bar')) self.assertEqual(self.trie['foo.bar'], None) - self.assertFalse('foo' in self.trie) - self.assertFalse('foo.bar.baz' in self.trie) - self.assertTrue('foo.bar' in self.trie) + self.assertFalse(self.trie.is_prefix('foo.bar.baz')) + self.assertFalse(self.trie.has_value('foo.bar.baz')) diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index a5925ea066d..a3985682440 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -28,7 +28,6 @@ import unittest import spack -import spack.url as url from spack.packages import PackageFinder @@ -45,7 +44,7 @@ def test_get_all_packages(self): self.check_db() - def test_get_all_mock_packages(self): + def ztest_get_all_mock_packages(self): """Get the mock packages once each too.""" db = PackageFinder(spack.mock_packages_path) spack.db.swap(db) @@ -53,7 +52,7 @@ def test_get_all_mock_packages(self): spack.db.swap(db) - def test_url_versions(self): + def ztest_url_versions(self): """Check URLs for regular packages, if they are explicitly defined.""" for pkg in spack.db.all_packages(): for v, vattrs in pkg.versions.items(): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 42bd91ec5ca..2d19d9ddc7e 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -27,7 +27,7 @@ from llnl.util.filesystem import join_path import spack -import spack.packages as packages +from spack.packages import Repo from spack.util.naming import mod_to_class from spack.test.mock_packages_test import * @@ -44,7 +44,7 @@ def test_package_name(self): def test_package_filename(self): - repo = spack.db.get_repo('gov.llnl.spack.mock') + repo = Repo(spack.mock_packages_path) filename = repo.filename_for_package_name('mpich') self.assertEqual(filename, join_path(spack.mock_packages_path, 'mpich', 'package.py')) @@ -55,7 +55,7 @@ def test_package_name(self): def test_nonexisting_package_filename(self): - repo = spack.db.get_repo('gov.llnl.spack.mock') + repo = Repo(spack.mock_packages_path) filename = repo.filename_for_package_name('some-nonexisting-package') self.assertEqual(filename, join_path(spack.mock_packages_path, 'some-nonexisting-package', 'package.py')) diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index db7ddd251de..3ff76f63be7 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -29,7 +29,6 @@ import spack import spack.url as url -from spack.packages import PackageDB class PackageSanityTest(unittest.TestCase): diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py index 475062bb38a..26ca86c77fa 100644 --- a/lib/spack/spack/util/naming.py +++ b/lib/spack/spack/util/naming.py @@ -134,14 +134,42 @@ def __getitem__(self, namespace): return self._get_helper(namespace, namespace) - def __contains__(self, namespace): + def is_prefix(self, namespace): + """True if the namespace has a value, or if it's the prefix of one that does.""" + first, sep, rest = namespace.partition(self._sep) + if not first: + return True + elif first not in self._subspaces: + return False + else: + return self._subspaces[first].is_prefix(rest) + + + def is_leaf(self, namespace): + """True if this namespace has no children in the trie.""" + first, sep, rest = namespace.partition(self._sep) + if not first: + return bool(self._subspaces) + elif first not in self._subspaces: + return False + else: + return self._subspaces[first].is_leaf(rest) + + + def has_value(self, namespace): + """True if there is a value set for the given namespace.""" first, sep, rest = namespace.partition(self._sep) if not first: return self._value is not None elif first not in self._subspaces: return False else: - return rest in self._subspaces[first] + return self._subspaces[first].has_value(rest) + + + def __contains__(self, namespace): + """Returns whether a value has been set for the namespace.""" + return self.has_value(namespace) def _str_helper(self, stream, level=0): From 1330c10fc8ed4a4e9ddcb846f0b82644e34fd92d Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 23 Nov 2015 16:00:10 -0500 Subject: [PATCH 078/910] Don't use https for downloading --- var/spack/packages/hypre/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/hypre/package.py b/var/spack/packages/hypre/package.py index 198b3f00dcd..0f7f14dd89f 100644 --- a/var/spack/packages/hypre/package.py +++ b/var/spack/packages/hypre/package.py @@ -5,8 +5,8 @@ class Hypre(Package): features parallel multigrid methods for both structured and unstructured grid problems.""" - homepage = "https://computation.llnl.gov/project/linear_solvers/software.php" - url = "https://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz" + homepage = "http://computation.llnl.gov/project/linear_solvers/software.php" + url = "http://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz" version('2.10.0b', '768be38793a35bb5d055905b271f5b8e') From cb21a5b309b7e1352e7ae75361a2a8931f818d9e Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 23 Nov 2015 16:48:06 -0500 Subject: [PATCH 079/910] Add OpenBLAS --- var/spack/packages/openblas/package.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 var/spack/packages/openblas/package.py diff --git a/var/spack/packages/openblas/package.py b/var/spack/packages/openblas/package.py new file mode 100644 index 00000000000..8cc26d73e60 --- /dev/null +++ b/var/spack/packages/openblas/package.py @@ -0,0 +1,24 @@ +from spack import * + +class Openblas(Package): + """OpenBLAS: An optimized BLAS library""" + homepage = "http://www.openblas.net" + url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + + version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') + + # virtual dependency + provides('blas') + provides('lapack') + + # Doesn't always build correctly in parallel + # parallel = False + + def install(self, spec, prefix): + make('libs', 'netlib', 'shared', 'CC=cc', 'FC=f77') + make('install', "PREFIX='%s'" % prefix) + + # Blas virtual package should provide blas.a and libblas.a + with working_dir(prefix.lib): + symlink('libopenblas.a', 'blas.a') + symlink('libopenblas.a', 'libblas.a') From 60fd7e22685c5948309362bbd5e01ae5f5e7dd02 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 23 Nov 2015 16:50:11 -0500 Subject: [PATCH 080/910] Update PAPI --- var/spack/packages/papi/package.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/var/spack/packages/papi/package.py b/var/spack/packages/papi/package.py index 596f7114d60..097b8c41ec0 100644 --- a/var/spack/packages/papi/package.py +++ b/var/spack/packages/papi/package.py @@ -13,6 +13,7 @@ class Papi(Package): homepage = "http://icl.cs.utk.edu/papi/index.html" url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.3.0.tar.gz" + version('5.4.1', '9134a99219c79767a11463a76b0b01a2') version('5.3.0', '367961dd0ab426e5ae367c2713924ffb') def install(self, spec, prefix): @@ -20,13 +21,10 @@ def install(self, spec, prefix): configure_args=["--prefix=%s" % prefix] - # need to force consistency in the use of compilers - if spec.satisfies('%gcc'): - configure_args.append('CC=gcc') - configure_args.append('MPICH_CC=gcc') - if spec.satisfies('%intel'): - configure_args.append('CC=icc') - configure_args.append('MPICH_CC=icc') + # PAPI uses MPI if MPI is present; since we don't require an + # MPI package, we ensure that all attempts to use MPI fail, so + # that PAPI does not get confused + configure_args.append('MPICC=:') configure(*configure_args) From 38b83362b6caeaab56aa7f9c01889eafb77e6e18 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Mon, 23 Nov 2015 15:58:33 -0800 Subject: [PATCH 081/910] Updated url and description --- var/spack/packages/cleverleaf/package.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/cleverleaf/package.py b/var/spack/packages/cleverleaf/package.py index a01a706c169..ddbe57f0194 100644 --- a/var/spack/packages/cleverleaf/package.py +++ b/var/spack/packages/cleverleaf/package.py @@ -1,10 +1,17 @@ from spack import * class Cleverleaf(Package): - homepage = "http://www.example.com" - url = "http://www.example.com/cleverleaf-1.0.tar.gz" + """ + CleverLeaf is a hydrodynamics mini-app that extends CloverLeaf with Adaptive + Mesh Refinement using the SAMRAI toolkit from Lawrence Livermore National + Laboratory. The primary goal of CleverLeaf is to evaluate the application of + AMR to the Lagrangian-Eulerian hydrodynamics scheme used by CloverLeaf. + """ - version('develop', git='git@github.com:UK-MAC/CleverLeaf_ref.git', branch='develop') + homepage = "http://uk-mac.github.io/CleverLeaf/" + url = "https://github.com/UK-MAC/CleverLeaf/tarball/master" + + version('develop', git='https://github.com/UK-MAC/CleverLeaf_ref.git', branch='develop') depends_on("SAMRAI@3.8.0") From 4dd47fbc78a1d7c16bbd93248bdd9a2a3696f0e9 Mon Sep 17 00:00:00 2001 From: Kevin Huck Date: Mon, 23 Nov 2015 16:16:52 -0800 Subject: [PATCH 082/910] cleaning up commits for merge request --- var/spack/packages/apex/package.py | 7 ++----- var/spack/packages/ompt-openmp/package.py | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/var/spack/packages/apex/package.py b/var/spack/packages/apex/package.py index 6404d5208aa..8769d970567 100644 --- a/var/spack/packages/apex/package.py +++ b/var/spack/packages/apex/package.py @@ -3,12 +3,9 @@ class Apex(Package): homepage = "http://github.com/khuck/xpress-apex" - #url = "http://github.com/khuck/xpress-apex/archive/v0.1-release-candidate.tar.gz" - url = "http://github.com/khuck/xpress-apex" + url = "http://github.com/khuck/xpress-apex/archive/v0.1.tar.gz" - #version('0.1', '6e039c224387348296739f6bf360d081') - #version('master', branch='master', git='https://github.com/khuck/xpress-apex.git') - version('2015-10-21', git='https://github.com/khuck/xpress-apex.git', commit='d2e66ddde689120472fc57fc546d8cd80aab745c') + version('0.1', '8b95f0c0313da1575960d3ad69f18e75') depends_on("binutils+libiberty") depends_on("boost@1.54:") diff --git a/var/spack/packages/ompt-openmp/package.py b/var/spack/packages/ompt-openmp/package.py index 5d380ebd775..9c1b4b95701 100644 --- a/var/spack/packages/ompt-openmp/package.py +++ b/var/spack/packages/ompt-openmp/package.py @@ -3,9 +3,9 @@ class OmptOpenmp(Package): """LLVM/Clang OpenMP runtime with OMPT support. This is a fork of the OpenMPToolsInterface/LLVM-openmp fork of the official LLVM OpenMP mirror. This library provides a drop-in replacement of the OpenMP runtimes for GCC, Intel and LLVM/Clang.""" homepage = "https://github.com/OpenMPToolsInterface/LLVM-openmp" - url = "http://github.com/khuck/LLVM-openmp/archive/v0.1-spack.tar.gz" + url = "http://github.com/khuck/LLVM-openmp/archive/v0.1.tar.gz" - version('spack', '35227b2726e377faa433fc841226e036') + version('0.1', '3375b5ce67a48cae107371fcd811f639') # depends_on("foo") From 1d90d98e406d280e634b168bbea919ab0f764a48 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 23 Nov 2015 20:32:06 -0500 Subject: [PATCH 083/910] Remove commented-out code --- var/spack/packages/openblas/package.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/var/spack/packages/openblas/package.py b/var/spack/packages/openblas/package.py index 8cc26d73e60..e01467c05ae 100644 --- a/var/spack/packages/openblas/package.py +++ b/var/spack/packages/openblas/package.py @@ -11,9 +11,6 @@ class Openblas(Package): provides('blas') provides('lapack') - # Doesn't always build correctly in parallel - # parallel = False - def install(self, spec, prefix): make('libs', 'netlib', 'shared', 'CC=cc', 'FC=f77') make('install', "PREFIX='%s'" % prefix) From 894fcd90e7cc9c080e639f846a02b683ff29c17b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 23 Nov 2015 17:39:59 -0800 Subject: [PATCH 084/910] Add a fix/warning so that stale .pyc files don't kill Spack. - Can't think of a better way to do this. - The externals integration will cause spack to die in weird ways for users who just pull from develop. --- bin/spack | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/bin/spack b/bin/spack index f587e206db7..e92d7cc273f 100755 --- a/bin/spack +++ b/bin/spack @@ -41,6 +41,21 @@ sys.path.insert(0, SPACK_LIB_PATH) SPACK_EXTERNAL_LIBS = os.path.join(SPACK_LIB_PATH, "external") sys.path.insert(0, SPACK_EXTERNAL_LIBS) +# Quick and dirty check to clean orphaned .pyc files left over from +# previous revisions. These files were present in earlier versions of +# Spack, were removed, but shadow system modules that Spack still +# imports. If we leave them, Spack will fail in mysterious ways. +# TODO: more elegant solution for orphaned pyc files. +orphaned_pyc_files = [os.path.join(SPACK_EXTERNAL_LIBS, n) + for n in ('functools.pyc', 'ordereddict.pyc')] +for pyc_file in orphaned_pyc_files: + if not os.path.exists(pyc_file): + continue + try: + os.remove(pyc_file) + except OSError as e: + print "WARNING: Spack may fail mysteriously. Couldn't remove orphaned .pyc file: %s" % pyc + # If there is no working directory, use the spack prefix. try: working_dir = os.getcwd() From 099fa1df34a157a323aa023b69c52399a441b81b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 12 Nov 2015 13:23:19 -0800 Subject: [PATCH 085/910] Use nose to run unit tests. 1. Adding a plugin to keep track of the total number of tests run as well as the number of tests with failures/errors. 2. Some nose plugins (including xunit which will be added in a future commit) assign stdout to a stream object that does not have a .fileno attribute. spack.util.executable.Executable now avoids passing stdout to subprocess (and always uses subprocess.PIPE) TODO: 1. Still need to figure out how to activate the plugin (as of now it is being ignored by nose). Newer versions of nose appear to make this simpler (e.g. the "addplugins" argument to nose.run) 2. Need to include new version of nose in order to use xunit --- lib/spack/spack/test/__init__.py | 27 +++++----- lib/spack/spack/test/tally_plugin.py | 73 ++++++++++++++++++++++++++++ lib/spack/spack/test/unit_install.py | 6 +-- lib/spack/spack/util/executable.py | 7 ++- 4 files changed, 93 insertions(+), 20 deletions(-) create mode 100644 lib/spack/spack/test/tally_plugin.py diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 0f776bfea46..9d577d65397 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -24,7 +24,9 @@ ############################################################################## import sys import unittest +import nose +from spack.test.tally_plugin import Tally import llnl.util.tty as tty from llnl.util.tty.colify import colify @@ -81,28 +83,23 @@ def run(names, verbose=False): "Valid names are:") colify(sorted(test_names), indent=4) sys.exit(1) - - runner = unittest.TextTestRunner(verbosity=verbosity) - - testsRun = errors = failures = 0 + + tally = Tally() + tally.enabled = True for test in names: module = 'spack.test.' + test print module - suite = unittest.defaultTestLoader.loadTestsFromName(module) - + tty.msg("Running test: %s" % test) - result = runner.run(suite) - testsRun += result.testsRun - errors += len(result.errors) - failures += len(result.failures) + result = nose.run(argv=["", module], plugins=[tally]) - succeeded = not errors and not failures + succeeded = not tally.failCount and not tally.errorCount tty.msg("Tests Complete.", - "%5d tests run" % testsRun, - "%5d failures" % failures, - "%5d errors" % errors) + "%5d tests run" % tally.numberOfTests, + "%5d failures" % tally.failCount, + "%5d errors" % tally.errorCount) - if not errors and not failures: + if succeeded: tty.info("OK", format='g') else: tty.info("FAIL", format='r') diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py new file mode 100644 index 00000000000..d8638bf7e3e --- /dev/null +++ b/lib/spack/spack/test/tally_plugin.py @@ -0,0 +1,73 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from nose.plugins import Plugin + +import os + +class Tally(Plugin): + name = 'tally' + + def __init__(self): + super(Tally, self).__init__() + self.successes = set() + self.failures = set() + self.errors = set() + + @property + def successCount(self): + return len(self.successes) + + @property + def failCount(self): + return len(self.failures) + + @property + def errorCount(self): + return len(self.errors) + + @property + def numberOfTests(self): + return self.errorCount + self.failCount + self.successCount + + def options(self, parser, env=os.environ): + super(Tally, self).options(parser, env=env) + + def configure(self, options, conf): + super(Tally, self).configure(options, conf) + + def begin(self): + print ">>> TALLY PLUGIN BEGIN" + + def addSuccess(self, test): + self.successes.add(test) + + def addError(self, test, err): + self.errors.add(test) + + def addFailure(self, test, err): + test.failures.add(test) + + def finalize(self, result): + pass diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index c4b9092f051..0bcb3f37671 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -90,7 +90,7 @@ def test_installing_both(self): pkgX.installed = True pkgY.installed = True - test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=test_fetch_log) + test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=mock_fetch_log) self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED, @@ -101,7 +101,7 @@ def test_dependency_already_installed(self): pkgX.installed = True pkgY.installed = True - test_install.create_test_output(specX, [specX], mo, getLogFunc=test_fetch_log) + test_install.create_test_output(specX, [specX], mo, getLogFunc=mock_fetch_log) self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED}) @@ -116,6 +116,6 @@ def __init__(self, init=None): def get(self, spec): return self.specToPkg[spec] -def test_fetch_log(path): +def mock_fetch_log(path): return [] diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index d1dfb62ffb6..6848a7e2a54 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -95,11 +95,14 @@ def streamify(arg, mode): proc = subprocess.Popen( cmd, stdin=input, - stderr=error, - stdout=subprocess.PIPE if return_output else output) + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) out, err = proc.communicate() self.returncode = proc.returncode + output.write(out) + error.write(err) + rc = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" From 52cce2d71081404261e6eaf753ad754430517ed5 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 24 Nov 2015 00:30:08 -0800 Subject: [PATCH 086/910] Revert "WIP: openssl: set KERNEL_BITS to make ./config work" --- var/spack/packages/openssl/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index 43b8771be0f..4bf1f832d45 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -16,8 +16,6 @@ class Openssl(Package): parallel = False def install(self, spec, prefix): - env['KERNEL_BITS'] = '64' # Only on 64-bit Mac - config = Executable("./config") config("--prefix=%s" % prefix, "--openssldir=%s/etc/openssl" % prefix, From a1553dd12b9388277a3cb80cfa861489bd1c865f Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Tue, 24 Nov 2015 09:01:50 -0800 Subject: [PATCH 087/910] adding google sparsehash and cityhash libraries --- var/spack/packages/cityhash/package.py | 16 ++++++++++++++++ var/spack/packages/jemalloc/package.py | 12 +++++++++++- var/spack/packages/sparsehash/package.py | 14 ++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 var/spack/packages/cityhash/package.py create mode 100644 var/spack/packages/sparsehash/package.py diff --git a/var/spack/packages/cityhash/package.py b/var/spack/packages/cityhash/package.py new file mode 100644 index 00000000000..1643cc3b425 --- /dev/null +++ b/var/spack/packages/cityhash/package.py @@ -0,0 +1,16 @@ +from spack import * +from spack.util.environment import * + +class Cityhash(Package): + homepage = "https://github.com/google/cityhash" + url = "https://github.com/google/cityhash" + + version('2013-07-31', git='https://github.com/google/cityhash.git', commit='8af9b8c2b889d80c22d6bc26ba0df1afb79a30db') + version('master', branch='master', git='https://github.com/google/cityhash.git') + + def install(self, spec, prefix): + configure('--enable-sse4.2', '--prefix=%s' % prefix) + + make() + make("install") + diff --git a/var/spack/packages/jemalloc/package.py b/var/spack/packages/jemalloc/package.py index ff946ae821d..8cec9ea75ba 100644 --- a/var/spack/packages/jemalloc/package.py +++ b/var/spack/packages/jemalloc/package.py @@ -7,8 +7,18 @@ class Jemalloc(Package): version('4.0.4', '687c5cc53b9a7ab711ccd680351ff988') + variant('stats', default=False, description='Enable heap statistics') + variant('prof', default=False, description='Enable heap profiling') + def install(self, spec, prefix): - configure('--prefix=%s' % prefix) + configure_args = ['--prefix=%s' % prefix,] + + if '+stats' in spec: + configure_args.append('--enable-stats') + if '+prof' in spec: + configure_args.append('--enable-prof') + + configure(*configure_args) make() make("install") diff --git a/var/spack/packages/sparsehash/package.py b/var/spack/packages/sparsehash/package.py new file mode 100644 index 00000000000..7decaeb89b1 --- /dev/null +++ b/var/spack/packages/sparsehash/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Sparsehash(Package): + """Sparse and dense hash-tables for C++ by Google""" + homepage = "https://github.com/sparsehash/sparsehash" + url = "https://github.com/sparsehash/sparsehash/archive/sparsehash-2.0.3.tar.gz" + + version('2.0.3', 'd8d5e2538c1c25577b3f066d7a55e99e') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") From 70049185a5b3d132374dcb36138eaa081503f96c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 24 Nov 2015 10:39:42 -0800 Subject: [PATCH 088/910] Activation of Tally plugin to count tests. It doesn't appear to be actually counting the individual unit tests correctly so I need to chase that down. --- lib/spack/spack/test/__init__.py | 5 +++-- lib/spack/spack/test/tally_plugin.py | 3 --- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 9d577d65397..327a4886b91 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -85,13 +85,14 @@ def run(names, verbose=False): sys.exit(1) tally = Tally() - tally.enabled = True for test in names: module = 'spack.test.' + test print module tty.msg("Running test: %s" % test) - result = nose.run(argv=["", module], plugins=[tally]) + + activateTally = "--with-%s" % spack.test.tally_plugin.Tally.name + result = nose.run(argv=["", activateTally, module], addplugins=[tally]) succeeded = not tally.failCount and not tally.errorCount tty.msg("Tests Complete.", diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index d8638bf7e3e..b38f4f31344 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -57,9 +57,6 @@ def options(self, parser, env=os.environ): def configure(self, options, conf): super(Tally, self).configure(options, conf) - def begin(self): - print ">>> TALLY PLUGIN BEGIN" - def addSuccess(self, test): self.successes.add(test) From 5081ba6802818d5a5533985c0e3a0c1fa5b6321b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 24 Nov 2015 10:45:07 -0800 Subject: [PATCH 089/910] It appears the same test object was returned multiple times for different unit tests, so tracking tests with sets wouldn't work unless I extracted the details relevant to the particular test. For now a simple count will work so using a set was unnecessary anyways. --- lib/spack/spack/test/tally_plugin.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index b38f4f31344..e314fae6cef 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -31,21 +31,9 @@ class Tally(Plugin): def __init__(self): super(Tally, self).__init__() - self.successes = set() - self.failures = set() - self.errors = set() - - @property - def successCount(self): - return len(self.successes) - - @property - def failCount(self): - return len(self.failures) - - @property - def errorCount(self): - return len(self.errors) + self.successCount = 0 + self.failCount = 0 + self.errorCount = 0 @property def numberOfTests(self): @@ -58,13 +46,13 @@ def configure(self, options, conf): super(Tally, self).configure(options, conf) def addSuccess(self, test): - self.successes.add(test) + self.successCount += 1 def addError(self, test, err): - self.errors.add(test) + self.errorCount += 1 def addFailure(self, test, err): - test.failures.add(test) + self.failCount += 1 def finalize(self, result): pass From a0d388875e619723920437a22897c2870ad29f81 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Tue, 24 Nov 2015 15:12:31 -0800 Subject: [PATCH 090/910] Fixed Mitos dependency and added new version --- var/spack/packages/Mitos/package.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/Mitos/package.py b/var/spack/packages/Mitos/package.py index e312da3ffc7..0377887943e 100644 --- a/var/spack/packages/Mitos/package.py +++ b/var/spack/packages/Mitos/package.py @@ -7,10 +7,17 @@ class Mitos(Package): homepage = "https://github.com/scalability-llnl/Mitos" url = "https://github.com/scalability-llnl/Mitos" - version('0.9.1', 'c6cb57f3cae54f5157affd97ef7ef79e', git='https://github.com/scalability-llnl/Mitos.git', tag='v0.9.1') + version('0.9.2', + git='https://github.com/scalability-llnl/Mitos.git', + commit='8cb143a2e8c00353ff531a781a9ca0992b0aaa3d') + + version('0.9.1', + git='https://github.com/scalability-llnl/Mitos.git', + tag='v0.9.1') depends_on('dyninst@8.2.1:') depends_on('hwloc') + depends_on('mpi') def install(self, spec, prefix): with working_dir('spack-build', create=True): From 0adf93853d802266e939bffb99b6b9f60a88188e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 24 Nov 2015 17:58:17 -0800 Subject: [PATCH 091/910] Update log_output to handle cases where sys.stdout/stderr streams do not have an associated file descriptor (e.g. holds for nose tests where sys.stdout is assigned to a StringIO object) --- lib/spack/llnl/util/tty/log.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index 5a52d45bc77..b26e0d250bd 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -122,6 +122,10 @@ def __init__(self, stream, echo=False, force_color=False, debug=False): self.force_color = force_color self.debug = debug + # Default is to try file-descriptor reassignment unless the system + # out/err streams do not have an associated file descriptor + self.directAssignment = False + def trace(self, frame, event, arg): """Jumps to __exit__ on the child process.""" raise _SkipWithBlock() @@ -185,13 +189,21 @@ def __enter__(self): # Child: redirect output, execute the with block. os.close(read) - # Save old stdout and stderr - self._stdout = os.dup(sys.stdout.fileno()) - self._stderr = os.dup(sys.stderr.fileno()) + try: + # Save old stdout and stderr + self._stdout = os.dup(sys.stdout.fileno()) + self._stderr = os.dup(sys.stderr.fileno()) - # redirect to the pipe. - os.dup2(write, sys.stdout.fileno()) - os.dup2(write, sys.stderr.fileno()) + # redirect to the pipe. + os.dup2(write, sys.stdout.fileno()) + os.dup2(write, sys.stderr.fileno()) + except AttributeError: + self.directAssignment = True + self._stdout = sys.stdout + self._stderr = sys.stderr + output_redirect = os.fdopen(write, 'w') + sys.stdout = output_redirect + sys.stderr = output_redirect if self.force_color: color._force_color = True @@ -218,8 +230,12 @@ def __exit__(self, exc_type, exception, traceback): # # TODO: think about how this works outside install. # TODO: ideally would propagate exception to parent... - os.dup2(self._stdout, sys.stdout.fileno()) - os.dup2(self._stderr, sys.stderr.fileno()) + if self.directAssignment: + sys.stdout = self._stdout + sys.stderr = self._stderr + else: + os.dup2(self._stdout, sys.stdout.fileno()) + os.dup2(self._stderr, sys.stderr.fileno()) return False From 18dea24df8710bb65fd61278fb6bdf45d28d2aa6 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 24 Nov 2015 19:08:37 -0800 Subject: [PATCH 092/910] Use the xunit nose plugin to generate JUnit XML test results from Spack's unit tests. --- lib/spack/spack/cmd/test.py | 13 ++++++++++++- lib/spack/spack/test/__init__.py | 12 +++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index b1418ac2f1b..001ebfaef0b 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -22,8 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import os from pprint import pprint +from llnl.util.filesystem import join_path, mkdirp from llnl.util.tty.colify import colify from llnl.util.lang import list_modules @@ -37,6 +39,9 @@ def setup_parser(subparser): 'names', nargs='*', help="Names of tests to run.") subparser.add_argument( '-l', '--list', action='store_true', dest='list', help="Show available tests") + # TODO: make XML output optional + subparser.add_argument( + '-d', '--outputDir', dest='outputDir', help="Nose creates XML files in this directory") subparser.add_argument( '-v', '--verbose', action='store_true', dest='verbose', help="verbose output") @@ -48,4 +53,10 @@ def test(parser, args): colify(spack.test.list_tests(), indent=2) else: - spack.test.run(args.names, args.verbose) + if not args.outputDir: + outputDir = join_path(os.getcwd(), "test-output") + else: + outputDir = args.outputDir + if not os.path.exists(outputDir): + mkdirp(outputDir) + spack.test.run(args.names, outputDir, args.verbose) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 327a4886b91..158e8bdfcb3 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -27,6 +27,7 @@ import nose from spack.test.tally_plugin import Tally +from llnl.util.filesystem import join_path import llnl.util.tty as tty from llnl.util.tty.colify import colify @@ -69,7 +70,7 @@ def list_tests(): return test_names -def run(names, verbose=False): +def run(names, outputDir, verbose=False): """Run tests with the supplied names. Names should be a list. If it's empty, run ALL of Spack's tests.""" verbosity = 1 if not verbose else 2 @@ -91,8 +92,13 @@ def run(names, verbose=False): tty.msg("Running test: %s" % test) - activateTally = "--with-%s" % spack.test.tally_plugin.Tally.name - result = nose.run(argv=["", activateTally, module], addplugins=[tally]) + xmlOutputFname = "unittests-{0}.xml".format(test) + xmlOutputPath = join_path(outputDir, xmlOutputFname) + runOpts = ["--with-%s" % spack.test.tally_plugin.Tally.name, + "--with-xunit", + "--xunit-file={0}".format(xmlOutputPath)] + argv = [""] + runOpts + [module] + result = nose.run(argv=argv, addplugins=[tally]) succeeded = not tally.failCount and not tally.errorCount tty.msg("Tests Complete.", From e6d232bfefda069d876466faaa0c8e9e4a11b4f3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 25 Nov 2015 09:58:10 -0800 Subject: [PATCH 093/910] Rename packages.py to repository.py, PackageFinder to RepoPath. --- lib/spack/spack/__init__.py | 4 +- lib/spack/spack/cmd/repo.py | 2 +- lib/spack/spack/cmd/uninstall.py | 4 +- lib/spack/spack/repo_loader.py | 110 ------------------ .../spack/{packages.py => repository.py} | 19 ++- lib/spack/spack/test/directory_layout.py | 4 +- lib/spack/spack/test/mock_packages_test.py | 4 +- lib/spack/spack/test/package_sanity.py | 4 +- lib/spack/spack/test/packages.py | 2 +- 9 files changed, 25 insertions(+), 128 deletions(-) delete mode 100644 lib/spack/spack/repo_loader.py rename lib/spack/spack/{packages.py => repository.py} (97%) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 549d0a9a0fc..20ae8c92727 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -54,11 +54,11 @@ # # Set up the default packages database. # -import spack.packages +import spack.repository _repo_paths = spack.config.get_repos_config() if not _repo_paths: tty.die("Spack configuration contains no package repositories.") -db = spack.packages.PackageFinder(*_repo_paths) +db = spack.repository.RepoPath(*_repo_paths) sys.meta_path.append(db) # diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index e290f60b7b9..87f21b833d1 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -32,7 +32,7 @@ import spack.spec import spack.config from spack.util.environment import get_path -from spack.packages import repo_config_filename +from spack.repository import repo_config_filename import os import exceptions diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index e80f2d2636d..eba76ef71db 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -30,7 +30,7 @@ import spack import spack.cmd -import spack.packages +import spack.repository from spack.cmd.find import display_specs from spack.package import PackageStillNeededError @@ -80,7 +80,7 @@ def uninstall(parser, args): # should work if package is known to spack pkgs.append(s.package) - except spack.packages.UnknownPackageError, e: + except spack.repository.UnknownPackageError, e: # The package.py file has gone away -- but still want to uninstall. spack.Package(s).do_uninstall(force=True) diff --git a/lib/spack/spack/repo_loader.py b/lib/spack/spack/repo_loader.py deleted file mode 100644 index 441011cf986..00000000000 --- a/lib/spack/spack/repo_loader.py +++ /dev/null @@ -1,110 +0,0 @@ -import re -import sys -import types -import traceback - -from llnl.util.lang import * -import spack - -# Name of module under which packages are imported -imported_packages_module = 'spack.repos' - -# Name of the package file inside a package directory -package_file_name = 'package.py' - -class LazyLoader: - """The LazyLoader handles cases when repo modules or classes - are imported. It watches for 'spack.repos.*' loads, then - redirects the load to the appropriate module.""" - def find_module(self, fullname, pathname): - if not fullname.startswith(imported_packages_module): - return None - - partial_name = fullname[len(imported_packages_module)+1:] - - print "partial: ", partial_name - print - - last_dot = partial_name.rfind('.') - repo = partial_name[:last_dot] - module = partial_name[last_dot+1:] - - repo_loader = spack.db.repo_loaders.get(repo) - if repo_loader: - try: - self.mod = repo_loader.get_module(module) - return self - except (ImportError, spack.packages.UnknownPackageError): - return None - - def load_module(self, fullname): - return self.mod - -#sys.meta_path.append(LazyLoader()) - -_reponames = {} -class RepoNamespace(types.ModuleType): - """The RepoNamespace holds the repository namespaces under - spack.repos. For example, when accessing spack.repos.original - this class will use __getattr__ to translate the 'original' - into one of spack's known repositories""" - def __init__(self): - sys.modules[imported_packages_module] = self - - def __getattr__(self, name): - if name in _reponames: - return _reponames[name] - raise AttributeError - - @property - def __file__(self): - return None - - @property - def __path__(self): - return [] - - -class RepoLoader(types.ModuleType): - """Each RepoLoader is associated with a repository, and the RepoLoader is - responsible for loading packages out of that repository. For example, - a RepoLoader may be responsible for spack.repos.original, and when someone - references spack.repos.original.libelf that RepoLoader will load the - libelf package.""" - def __init__(self, reponame, repopath): - self.path = repopath - self.reponame = reponame - self.module_name = imported_packages_module + '.' + reponame - if not reponame in _reponames: - _reponames[reponame] = self - - sys.modules[self.module_name] = self - - - @property - def __path__(self): - return [ self.path ] - - - def __getattr__(self, name): - if name[0] == '_': - raise AttributeError - return self.get_module(name) - - - @memoized - def get_module(self, pkg_name): - import os - import imp - import llnl.util.tty as tty - - - try: - module_name = imported_packages_module + '.' + self.reponame + '.' + pkg_name - module = imp.load_source(module_name, file_path) - - except ImportError, e: - tty.die("Error while importing %s from %s:\n%s" % ( - pkg_name, file_path, e.message)) - - return module diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/repository.py similarity index 97% rename from lib/spack/spack/packages.py rename to lib/spack/spack/repository.py index 8114b7f1aa4..c1545b36547 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/repository.py @@ -67,8 +67,8 @@ def _make_namespace_module(ns): return module -class PackageFinder(object): - """A PackageFinder is a wrapper around a list of Repos. +class RepoPath(object): + """A RepoPath is a list of repos that function as one. It functions exactly like a Repo, but it operates on the combined results of the Repos in its list instead of on a @@ -83,7 +83,10 @@ def __init__(self, *repo_dirs): self._provider_index = None for root in repo_dirs: - repo = Repo(root) + # Try to make it a repo if it's not one. + if not isinstance(root, Repo): + repo = Repo(root) + # Add the repo to the path. self.put_last(repo) @@ -94,7 +97,11 @@ def swap(self, other): TODO: Maybe there is a cleaner way. """ - attrs = ['repos', 'by_namespace', 'by_path', '_all_package_names', '_provider_index'] + attrs = ['repos', + 'by_namespace', + 'by_path', + '_all_package_names', + '_provider_index'] for attr in attrs: tmp = getattr(self, attr) setattr(self, attr, getattr(other, attr)) @@ -185,7 +192,7 @@ def find_module(self, fullname, path=None): return repo # No repo provides the namespace, but it is a valid prefix of - # something in the PackageFinder. + # something in the RepoPath. if self.by_namespace.is_prefix(fullname): return self @@ -603,7 +610,7 @@ def __init__(self, name, repo=None): class DuplicateRepoError(spack.error.SpackError): - """Raised when duplicate repos are added to a PackageFinder.""" + """Raised when duplicate repos are added to a RepoPath.""" def __init__(self, msg, repo1, repo2): super(UnknownPackageError, self).__init__( "%s: %s, %s" % (msg, repo1, repo2)) diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 55b3f0b18f9..580620e0e86 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -34,7 +34,7 @@ import spack from spack.spec import Spec -from spack.packages import PackageFinder +from spack.repository import RepoPath from spack.directory_layout import YamlDirectoryLayout # number of packages to test (to reduce test time) @@ -123,7 +123,7 @@ def test_handle_unknown_package(self): information about installed packages' specs to uninstall or query them again if the package goes away. """ - mock_db = PackageFinder(spack.mock_packages_path) + mock_db = RepoPath(spack.mock_packages_path) not_in_mock = set.difference( set(spack.db.all_package_names()), diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 071c21b7e08..8c6273def00 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -27,7 +27,7 @@ import spack import spack.config -from spack.packages import PackageFinder +from spack.repository import RepoPath from spack.spec import Spec @@ -36,7 +36,7 @@ def initmock(self): # Use the mock packages database for these tests. This allows # us to set up contrived packages that don't interfere with # real ones. - self.db = PackageFinder(spack.mock_packages_path) + self.db = RepoPath(spack.mock_packages_path) spack.db.swap(self.db) spack.config.clear_config_caches() diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index a3985682440..0a132fd7018 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -28,7 +28,7 @@ import unittest import spack -from spack.packages import PackageFinder +from spack.repository import RepoPath class PackageSanityTest(unittest.TestCase): @@ -46,7 +46,7 @@ def test_get_all_packages(self): def ztest_get_all_mock_packages(self): """Get the mock packages once each too.""" - db = PackageFinder(spack.mock_packages_path) + db = RepoPath(spack.mock_packages_path) spack.db.swap(db) self.check_db() spack.db.swap(db) diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 2d19d9ddc7e..8a786e364e4 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -27,7 +27,7 @@ from llnl.util.filesystem import join_path import spack -from spack.packages import Repo +from spack.repository import Repo from spack.util.naming import mod_to_class from spack.test.mock_packages_test import * From 04f032d6e397ce219a673c93277683060def52fd Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 25 Nov 2015 10:01:44 -0800 Subject: [PATCH 094/910] Rename spack.db -> spack.repo Making distinction between install database and package repositories clearer. --- lib/spack/spack/__init__.py | 6 +-- lib/spack/spack/cmd/checksum.py | 2 +- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/cmd/create.py | 4 +- lib/spack/spack/cmd/diy.py | 4 +- lib/spack/spack/cmd/edit.py | 2 +- lib/spack/spack/cmd/extensions.py | 2 +- lib/spack/spack/cmd/fetch.py | 2 +- lib/spack/spack/cmd/find.py | 2 +- lib/spack/spack/cmd/info.py | 2 +- lib/spack/spack/cmd/install.py | 2 +- lib/spack/spack/cmd/list.py | 2 +- lib/spack/spack/cmd/location.py | 6 +-- lib/spack/spack/cmd/mirror.py | 2 +- lib/spack/spack/cmd/package-list.py | 2 +- lib/spack/spack/cmd/patch.py | 2 +- lib/spack/spack/cmd/pkg.py | 2 +- lib/spack/spack/cmd/providers.py | 2 +- lib/spack/spack/cmd/repo.py | 2 +- lib/spack/spack/cmd/restage.py | 2 +- lib/spack/spack/cmd/stage.py | 2 +- lib/spack/spack/cmd/test-install.py | 8 ++-- lib/spack/spack/cmd/urls.py | 2 +- lib/spack/spack/cmd/versions.py | 2 +- lib/spack/spack/database.py | 2 +- lib/spack/spack/graph.py | 2 +- lib/spack/spack/package.py | 4 +- lib/spack/spack/patch.py | 2 +- lib/spack/spack/spec.py | 14 +++---- lib/spack/spack/test/concretize.py | 12 +++--- lib/spack/spack/test/database.py | 2 +- lib/spack/spack/test/directory_layout.py | 12 +++--- lib/spack/spack/test/git_fetch.py | 2 +- lib/spack/spack/test/hg_fetch.py | 2 +- lib/spack/spack/test/install.py | 2 +- lib/spack/spack/test/mirror.py | 2 +- lib/spack/spack/test/mock_packages_test.py | 6 +-- lib/spack/spack/test/multimethod.py | 48 +++++++++++----------- lib/spack/spack/test/package_sanity.py | 10 ++--- lib/spack/spack/test/packages.py | 6 +-- lib/spack/spack/test/python_version.py | 4 +- lib/spack/spack/test/svn_fetch.py | 2 +- lib/spack/spack/test/unit_install.py | 6 +-- 43 files changed, 103 insertions(+), 103 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 20ae8c92727..4f481ce9379 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -58,8 +58,8 @@ _repo_paths = spack.config.get_repos_config() if not _repo_paths: tty.die("Spack configuration contains no package repositories.") -db = spack.repository.RepoPath(*_repo_paths) -sys.meta_path.append(db) +repo = spack.repository.RepoPath(*_repo_paths) +sys.meta_path.append(repo) # # Set up the installed packages database @@ -146,7 +146,7 @@ # When packages call 'from spack import *', this extra stuff is brought in. # # Spack internal code should call 'import spack' and accesses other -# variables (spack.db, paths, etc.) directly. +# variables (spack.repo, paths, etc.) directly. # # TODO: maybe this should be separated out and should go in build_environment.py? # TODO: it's not clear where all the stuff that needs to be included in packages diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 8a448450c2a..a9a5d11ccaa 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -81,7 +81,7 @@ def get_checksums(versions, urls, **kwargs): def checksum(parser, args): # get the package we're going to generate checksums for - pkg = spack.db.get(args.package) + pkg = spack.repo.get(args.package) # If the user asked for specific versions, use those. if args.versions: diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index c20136ebe5d..5546060e09e 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -42,5 +42,5 @@ def clean(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: - package = spack.db.get(spec) + package = spack.repo.get(spec) package.do_clean() diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 1502942f2c9..475c4993b59 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -172,7 +172,7 @@ def create(parser, args): tty.msg("Creating template for package %s" % name) # Create a directory for the new package. - pkg_path = spack.db.filename_for_package_name(name, package_repo) + pkg_path = spack.repo.filename_for_package_name(name, package_repo) if os.path.exists(pkg_path) and not args.force: tty.die("%s already exists." % pkg_path) else: @@ -214,7 +214,7 @@ def create(parser, args): name = 'py-%s' % name # Create a directory for the new package. - pkg_path = spack.db.filename_for_package_name(name) + pkg_path = spack.repo.filename_for_package_name(name) if os.path.exists(pkg_path) and not args.force: tty.die("%s already exists." % pkg_path) else: diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 9f8a6d39db3..9b8d2e1f6fe 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -61,7 +61,7 @@ def diy(self, args): # Take a write lock before checking for existence. with spack.installed_db.write_transaction(): spec = specs[0] - if not spack.db.exists(spec.name): + if not spack.repo.exists(spec.name): tty.warn("No such package: %s" % spec.name) create = tty.get_yes_or_no("Create this package?", default=False) if not create: @@ -76,7 +76,7 @@ def diy(self, args): tty.die("spack diy spec must have a single, concrete version.") spec.concretize() - package = spack.db.get(spec) + package = spack.repo.get(spec) if package.installed: tty.error("Already installed in %s" % package.prefix) diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index 9081d12516d..cb63c6fecda 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -54,7 +54,7 @@ def install(self, spec, prefix): def edit_package(name, force=False): - path = spack.db.filename_for_package_name(name) + path = spack.repo.filename_for_package_name(name) if os.path.exists(path): if not os.path.isfile(path): diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index 7cadc424b00..39c8e1f8c03 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -71,7 +71,7 @@ def extensions(parser, args): args.mode = 'short' # List package names of extensions - extensions = spack.db.extensions_for(spec) + extensions = spack.repo.extensions_for(spec) if not extensions: tty.msg("%s has no extensions." % spec.cshort_spec) return diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 0ccebd9486f..efaa7acc138 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -46,5 +46,5 @@ def fetch(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: - package = spack.db.get(spec) + package = spack.repo.get(spec) package.do_fetch() diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 0b0dd6ef6fc..5c0f2f521e4 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -137,7 +137,7 @@ def find(parser, args): # Filter out specs that don't exist. query_specs = spack.cmd.parse_specs(args.query_specs) query_specs, nonexisting = partition_list( - query_specs, lambda s: spack.db.exists(s.name)) + query_specs, lambda s: spack.repo.exists(s.name)) if nonexisting: msg = "No such package%s: " % ('s' if len(nonexisting) > 1 else '') diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index 085e4db44d8..2b61850c2d1 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -105,5 +105,5 @@ def print_text_info(pkg): def info(parser, args): - pkg = spack.db.get(args.name) + pkg = spack.repo.get(args.name) print_text_info(pkg) diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 836a6260c87..0942a8e3830 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -70,7 +70,7 @@ def install(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: - package = spack.db.get(spec) + package = spack.repo.get(spec) with spack.installed_db.write_transaction(): package.do_install( keep_prefix=args.keep_prefix, diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 1f0978a18ed..d20aded6cae 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -43,7 +43,7 @@ def setup_parser(subparser): def list(parser, args): # Start with all package names. - pkgs = spack.db.all_package_names() + pkgs = spack.repo.all_package_names() # filter if a filter arg was provided if args.filter: diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index e8e9c3f2778..7475c658676 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -72,7 +72,7 @@ def location(parser, args): print spack.prefix elif args.packages: - print spack.db.root + print spack.repo.root elif args.stages: print spack.stage_path @@ -94,12 +94,12 @@ def location(parser, args): if args.package_dir: # This one just needs the spec name. - print join_path(spack.db.root, spec.name) + print join_path(spack.repo.root, spec.name) else: # These versions need concretized specs. spec.concretize() - pkg = spack.db.get(spec) + pkg = spack.repo.get(spec) if args.stage_dir: print pkg.stage.path diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 2356170a9ac..9a507e69db3 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -128,7 +128,7 @@ def mirror_create(args): # If nothing is passed, use all packages. if not specs: - specs = [Spec(n) for n in spack.db.all_package_names()] + specs = [Spec(n) for n in spack.repo.all_package_names()] specs.sort(key=lambda s: s.format("$_$@").lower()) # Default name for directory is spack-mirror- diff --git a/lib/spack/spack/cmd/package-list.py b/lib/spack/spack/cmd/package-list.py index f0484828455..a14b06bf7f9 100644 --- a/lib/spack/spack/cmd/package-list.py +++ b/lib/spack/spack/cmd/package-list.py @@ -48,7 +48,7 @@ def rst_table(elts): def print_rst_package_list(): """Print out information on all packages in restructured text.""" - pkgs = sorted(spack.db.all_packages(), key=lambda s:s.name.lower()) + pkgs = sorted(spack.repo.all_packages(), key=lambda s:s.name.lower()) print ".. _package-list:" print diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index a6556c4828c..600cad87fe4 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -47,5 +47,5 @@ def patch(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: - package = spack.db.get(spec) + package = spack.repo.get(spec) package.do_patch() diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index 055b7c20623..5b701889412 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -85,7 +85,7 @@ def list_packages(rev): def pkg_add(args): for pkg_name in args.packages: - filename = spack.db.filename_for_package_name(pkg_name) + filename = spack.repo.filename_for_package_name(pkg_name) if not os.path.isfile(filename): tty.die("No such package: %s. Path does not exist:" % pkg_name, filename) diff --git a/lib/spack/spack/cmd/providers.py b/lib/spack/spack/cmd/providers.py index 2bcdc9fba2a..872afa98d59 100644 --- a/lib/spack/spack/cmd/providers.py +++ b/lib/spack/spack/cmd/providers.py @@ -39,4 +39,4 @@ def setup_parser(subparser): def providers(parser, args): for spec in spack.cmd.parse_specs(args.vpkg_spec): - colify(sorted(spack.db.providers_for(spec)), indent=4) + colify(sorted(spack.repo.providers_for(spec)), indent=4) diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 87f21b833d1..85cc83730c4 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -108,7 +108,7 @@ def repo_remove(args): def repo_list(args): """List package sources and their mnemoics""" - root_names = spack.db.repos + root_names = spack.repo.repos max_len = max(len(s[0]) for s in root_names) fmt = "%%-%ds%%s" % (max_len + 4) for root in root_names: diff --git a/lib/spack/spack/cmd/restage.py b/lib/spack/spack/cmd/restage.py index e735a12c327..a9ee26a539f 100644 --- a/lib/spack/spack/cmd/restage.py +++ b/lib/spack/spack/cmd/restage.py @@ -42,5 +42,5 @@ def restage(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: - package = spack.db.get(spec) + package = spack.repo.get(spec) package.do_restage() diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index f3dc97be171..f734b3dd3bc 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -49,5 +49,5 @@ def stage(parser, args): specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: - package = spack.db.get(spec) + package = spack.repo.get(spec) package.do_stage() diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 68b761d5dc8..7a8921d0580 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -115,7 +115,7 @@ def fetch_log(path): def failed_dependencies(spec): return set(childSpec for childSpec in spec.dependencies.itervalues() if not - spack.db.get(childSpec).installed) + spack.repo.get(childSpec).installed) def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): @@ -126,7 +126,7 @@ def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): continue failedDeps = failed_dependencies(spec) - package = spack.db.get(spec) + package = spack.repo.get(spec) if failedDeps: result = TestResult.SKIPPED dep = iter(failedDeps).next() @@ -171,7 +171,7 @@ def test_install(parser, args): newInstalls = set() for spec in topSpec.traverse(): - package = spack.db.get(spec) + package = spack.repo.get(spec) if not package.installed: newInstalls.add(spec) @@ -188,7 +188,7 @@ def test_install(parser, args): # Calling do_install for the top-level package would be sufficient but # this attempts to keep going if any package fails (other packages which # are not dependents may succeed) - package = spack.db.get(spec) + package = spack.repo.get(spec) if (not failed_dependencies(spec)) and (not package.installed): try: package.do_install( diff --git a/lib/spack/spack/cmd/urls.py b/lib/spack/spack/cmd/urls.py index 417ce3ab68b..14ce4c369f3 100644 --- a/lib/spack/spack/cmd/urls.py +++ b/lib/spack/spack/cmd/urls.py @@ -41,7 +41,7 @@ def setup_parser(subparser): def urls(parser, args): urls = set() - for pkg in spack.db.all_packages(): + for pkg in spack.repo.all_packages(): url = getattr(pkg.__class__, 'url', None) if url: urls.add(url) diff --git a/lib/spack/spack/cmd/versions.py b/lib/spack/spack/cmd/versions.py index ed16728261b..a5ea5e8f83a 100644 --- a/lib/spack/spack/cmd/versions.py +++ b/lib/spack/spack/cmd/versions.py @@ -34,7 +34,7 @@ def setup_parser(subparser): def versions(parser, args): - pkg = spack.db.get(args.package) + pkg = spack.repo.get(args.package) safe_versions = pkg.versions fetched_versions = pkg.fetch_remote_versions() diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index e0c14a04553..8e380083f3c 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -549,7 +549,7 @@ def query(self, query_spec=any, known=any, installed=True): for key, rec in self._data.items(): if installed is not any and rec.installed != installed: continue - if known is not any and spack.db.exists(rec.spec.name) != known: + if known is not any and spack.repo.exists(rec.spec.name) != known: continue if query_spec is any or rec.spec.satisfies(query_spec): results.append(rec.spec) diff --git a/lib/spack/spack/graph.py b/lib/spack/spack/graph.py index 5fb6a9cd234..0600016a519 100644 --- a/lib/spack/spack/graph.py +++ b/lib/spack/spack/graph.py @@ -523,7 +523,7 @@ def quote(string): return '"%s"' % string if not specs: - specs = [p.name for p in spack.db.all_packages()] + specs = [p.name for p in spack.repo.all_packages()] else: roots = specs specs = set() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 39d71bb4b9e..95b74bc961a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -372,7 +372,7 @@ def __init__(self, spec): self._total_time = 0.0 if self.is_extension: - spack.db.get(self.extendee_spec)._check_extendable() + spack.repo.get(self.extendee_spec)._check_extendable() @property @@ -548,7 +548,7 @@ def preorder_traversal(self, visited=None, **kwargs): yield spec continue - for pkg in spack.db.get(name).preorder_traversal(visited, **kwargs): + for pkg in spack.repo.get(name).preorder_traversal(visited, **kwargs): yield pkg diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index fe6e0a65a38..fef42fb5ee6 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -54,7 +54,7 @@ def __init__(self, pkg, path_or_url, level): if '://' in path_or_url: self.url = path_or_url else: - pkg_dir = spack.db.dirname_for_package_name(self.pkg_name) + pkg_dir = spack.repo.dirname_for_package_name(self.pkg_name) self.path = join_path(pkg_dir, path_or_url) if not os.path.isfile(self.path): raise NoSuchPatchFileError(pkg_name, self.path) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 5e59f240a48..bb0f194c139 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -487,7 +487,7 @@ def root(self): @property def package(self): - return spack.db.get(self) + return spack.repo.get(self) @property @@ -505,7 +505,7 @@ def virtual(self): @staticmethod def is_virtual(name): """Test if a name is virtual without requiring a Spec.""" - return not spack.db.exists(name) + return not spack.repo.exists(name) @property @@ -798,7 +798,7 @@ def _expand_virtual_packages(self): return changed for spec in virtuals: - providers = spack.db.providers_for(spec) + providers = spack.repo.providers_for(spec) concrete = spack.concretizer.choose_provider(spec, providers) concrete = concrete.copy() spec._replace_with(concrete) @@ -909,7 +909,7 @@ def _evaluate_dependency_conditions(self, name): the dependency. If no conditions are True (and we don't depend on it), return None. """ - pkg = spack.db.get(self.name) + pkg = spack.repo.get(self.name) conditions = pkg.dependencies[name] # evaluate when specs to figure out constraints on the dependency. @@ -1037,7 +1037,7 @@ def _normalize_helper(self, visited, spec_deps, provider_index): any_change = False changed = True - pkg = spack.db.get(self.name) + pkg = spack.repo.get(self.name) while changed: changed = False for dep_name in pkg.dependencies: @@ -1115,7 +1115,7 @@ def validate_names(self): for spec in self.traverse(): # Don't get a package for a virtual name. if not spec.virtual: - spack.db.get(spec.name) + spack.repo.get(spec.name) # validate compiler in addition to the package name. if spec.compiler: @@ -1247,7 +1247,7 @@ def satisfies(self, other, deps=True, strict=False): # A concrete provider can satisfy a virtual dependency. if not self.virtual and other.virtual: - pkg = spack.db.get(self.name) + pkg = spack.repo.get(self.name) if pkg.provides(other.name): for provided, when_spec in pkg.provided.items(): if self.satisfies(when_spec, deps=False, strict=strict): diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index b3a77d076a1..8842efe5a0f 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -125,22 +125,22 @@ def test_concretize_with_provides_when(self): we ask for some advanced version. """ self.assertTrue(not any(spec.satisfies('mpich2@:1.0') - for spec in spack.db.providers_for('mpi@2.1'))) + for spec in spack.repo.providers_for('mpi@2.1'))) self.assertTrue(not any(spec.satisfies('mpich2@:1.1') - for spec in spack.db.providers_for('mpi@2.2'))) + for spec in spack.repo.providers_for('mpi@2.2'))) self.assertTrue(not any(spec.satisfies('mpich2@:1.1') - for spec in spack.db.providers_for('mpi@2.2'))) + for spec in spack.repo.providers_for('mpi@2.2'))) self.assertTrue(not any(spec.satisfies('mpich@:1') - for spec in spack.db.providers_for('mpi@2'))) + for spec in spack.repo.providers_for('mpi@2'))) self.assertTrue(not any(spec.satisfies('mpich@:1') - for spec in spack.db.providers_for('mpi@3'))) + for spec in spack.repo.providers_for('mpi@3'))) self.assertTrue(not any(spec.satisfies('mpich2') - for spec in spack.db.providers_for('mpi@3'))) + for spec in spack.repo.providers_for('mpi@3'))) def test_virtual_is_fully_expanded_for_callpath(self): diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 8416143f2d2..5ce010ae8ff 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -79,7 +79,7 @@ class DatabaseTest(MockPackagesTest): def _mock_install(self, spec): s = Spec(spec) - pkg = spack.db.get(s.concretized()) + pkg = spack.repo.get(s.concretized()) pkg.do_install(fake=True) diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 580620e0e86..ded978b8187 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -62,7 +62,7 @@ def test_read_and_write_spec(self): finally that the directory can be removed by the directory layout. """ - packages = list(spack.db.all_packages())[:max_packages] + packages = list(spack.repo.all_packages())[:max_packages] for pkg in packages: spec = pkg.spec @@ -126,14 +126,14 @@ def test_handle_unknown_package(self): mock_db = RepoPath(spack.mock_packages_path) not_in_mock = set.difference( - set(spack.db.all_package_names()), + set(spack.repo.all_package_names()), set(mock_db.all_package_names())) packages = list(not_in_mock)[:max_packages] # Create all the packages that are not in mock. installed_specs = {} for pkg_name in packages: - spec = spack.db.get(pkg_name).spec + spec = spack.repo.get(pkg_name).spec # If a spec fails to concretize, just skip it. If it is a # real error, it will be caught by concretization tests. @@ -145,7 +145,7 @@ def test_handle_unknown_package(self): self.layout.create_install_directory(spec) installed_specs[spec] = self.layout.path_for_spec(spec) - spack.db.swap(mock_db) + spack.repo.swap(mock_db) # Now check that even without the package files, we know # enough to read a spec from the spec file. @@ -160,12 +160,12 @@ def test_handle_unknown_package(self): self.assertTrue(spec.eq_dag(spec_from_file)) self.assertEqual(spec.dag_hash(), spec_from_file.dag_hash()) - spack.db.swap(mock_db) + spack.repo.swap(mock_db) def test_find(self): """Test that finding specs within an install layout works.""" - packages = list(spack.db.all_packages())[:max_packages] + packages = list(spack.repo.all_packages())[:max_packages] # Create install prefixes for all packages in the list installed_specs = {} diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 9700bd75338..bf31bd2a5b3 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -50,7 +50,7 @@ def setUp(self): spec = Spec('git-test') spec.concretize() - self.pkg = spack.db.get(spec, new=True) + self.pkg = spack.repo.get(spec, new=True) def tearDown(self): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 531dfabaa18..559fc2959a7 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -47,7 +47,7 @@ def setUp(self): spec = Spec('hg-test') spec.concretize() - self.pkg = spack.db.get(spec, new=True) + self.pkg = spack.repo.get(spec, new=True) def tearDown(self): diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 5659e97a4d4..b280576c77e 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -78,7 +78,7 @@ def test_install_and_uninstall(self): self.assertTrue(spec.concrete) # Get the package - pkg = spack.db.get(spec) + pkg = spack.repo.get(spec) # Fake the URL for the package so it downloads from a file. pkg.fetcher = URLFetchStrategy(self.repo.url) diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 89ab14359ef..edd075a9a9e 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -55,7 +55,7 @@ def set_up_package(self, name, MockRepoClass, url_attr): spec.concretize() # Get the package and fix its fetch args to point to a mock repo - pkg = spack.db.get(spec) + pkg = spack.repo.get(spec) repo = MockRepoClass() self.repos[name] = repo diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 8c6273def00..2150b408760 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -37,7 +37,7 @@ def initmock(self): # us to set up contrived packages that don't interfere with # real ones. self.db = RepoPath(spack.mock_packages_path) - spack.db.swap(self.db) + spack.repo.swap(self.db) spack.config.clear_config_caches() self.real_scopes = spack.config.config_scopes @@ -59,7 +59,7 @@ def set_pkg_dep(self, pkg_name, spec): spec = Spec(spec) # Save original dependencies before making any changes. - pkg = spack.db.get(pkg_name) + pkg = spack.repo.get(pkg_name) if pkg_name not in self.saved_deps: self.saved_deps[pkg_name] = (pkg, pkg.dependencies.copy()) @@ -69,7 +69,7 @@ def set_pkg_dep(self, pkg_name, spec): def cleanmock(self): """Restore the real packages path after any test.""" - spack.db.swap(self.db) + spack.repo.swap(self.db) spack.config.config_scopes = self.real_scopes spack.config.clear_config_caches() diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index cd5d9e625e3..56e8b3f343e 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -38,92 +38,92 @@ class MultiMethodTest(MockPackagesTest): def test_no_version_match(self): - pkg = spack.db.get('multimethod@2.0') + pkg = spack.repo.get('multimethod@2.0') self.assertRaises(NoSuchMethodError, pkg.no_version_2) def test_one_version_match(self): - pkg = spack.db.get('multimethod@1.0') + pkg = spack.repo.get('multimethod@1.0') self.assertEqual(pkg.no_version_2(), 1) - pkg = spack.db.get('multimethod@3.0') + pkg = spack.repo.get('multimethod@3.0') self.assertEqual(pkg.no_version_2(), 3) - pkg = spack.db.get('multimethod@4.0') + pkg = spack.repo.get('multimethod@4.0') self.assertEqual(pkg.no_version_2(), 4) def test_version_overlap(self): - pkg = spack.db.get('multimethod@2.0') + pkg = spack.repo.get('multimethod@2.0') self.assertEqual(pkg.version_overlap(), 1) - pkg = spack.db.get('multimethod@5.0') + pkg = spack.repo.get('multimethod@5.0') self.assertEqual(pkg.version_overlap(), 2) def test_mpi_version(self): - pkg = spack.db.get('multimethod^mpich@3.0.4') + pkg = spack.repo.get('multimethod^mpich@3.0.4') self.assertEqual(pkg.mpi_version(), 3) - pkg = spack.db.get('multimethod^mpich2@1.2') + pkg = spack.repo.get('multimethod^mpich2@1.2') self.assertEqual(pkg.mpi_version(), 2) - pkg = spack.db.get('multimethod^mpich@1.0') + pkg = spack.repo.get('multimethod^mpich@1.0') self.assertEqual(pkg.mpi_version(), 1) def test_undefined_mpi_version(self): - pkg = spack.db.get('multimethod^mpich@0.4') + pkg = spack.repo.get('multimethod^mpich@0.4') self.assertEqual(pkg.mpi_version(), 1) - pkg = spack.db.get('multimethod^mpich@1.4') + pkg = spack.repo.get('multimethod^mpich@1.4') self.assertEqual(pkg.mpi_version(), 1) def test_default_works(self): - pkg = spack.db.get('multimethod%gcc') + pkg = spack.repo.get('multimethod%gcc') self.assertEqual(pkg.has_a_default(), 'gcc') - pkg = spack.db.get('multimethod%intel') + pkg = spack.repo.get('multimethod%intel') self.assertEqual(pkg.has_a_default(), 'intel') - pkg = spack.db.get('multimethod%pgi') + pkg = spack.repo.get('multimethod%pgi') self.assertEqual(pkg.has_a_default(), 'default') def test_architecture_match(self): - pkg = spack.db.get('multimethod=x86_64') + pkg = spack.repo.get('multimethod=x86_64') self.assertEqual(pkg.different_by_architecture(), 'x86_64') - pkg = spack.db.get('multimethod=ppc64') + pkg = spack.repo.get('multimethod=ppc64') self.assertEqual(pkg.different_by_architecture(), 'ppc64') - pkg = spack.db.get('multimethod=ppc32') + pkg = spack.repo.get('multimethod=ppc32') self.assertEqual(pkg.different_by_architecture(), 'ppc32') - pkg = spack.db.get('multimethod=arm64') + pkg = spack.repo.get('multimethod=arm64') self.assertEqual(pkg.different_by_architecture(), 'arm64') - pkg = spack.db.get('multimethod=macos') + pkg = spack.repo.get('multimethod=macos') self.assertRaises(NoSuchMethodError, pkg.different_by_architecture) def test_dependency_match(self): - pkg = spack.db.get('multimethod^zmpi') + pkg = spack.repo.get('multimethod^zmpi') self.assertEqual(pkg.different_by_dep(), 'zmpi') - pkg = spack.db.get('multimethod^mpich') + pkg = spack.repo.get('multimethod^mpich') self.assertEqual(pkg.different_by_dep(), 'mpich') # If we try to switch on some entirely different dep, it's ambiguous, # but should take the first option - pkg = spack.db.get('multimethod^foobar') + pkg = spack.repo.get('multimethod^foobar') self.assertEqual(pkg.different_by_dep(), 'mpich') def test_virtual_dep_match(self): - pkg = spack.db.get('multimethod^mpich2') + pkg = spack.repo.get('multimethod^mpich2') self.assertEqual(pkg.different_by_virtual_dep(), 2) - pkg = spack.db.get('multimethod^mpich@1.0') + pkg = spack.repo.get('multimethod^mpich@1.0') self.assertEqual(pkg.different_by_virtual_dep(), 1) diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index 0a132fd7018..b2d3da91b1f 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -35,8 +35,8 @@ class PackageSanityTest(unittest.TestCase): def check_db(self): """Get all packages in a DB to make sure they work.""" - for name in spack.db.all_package_names(): - spack.db.get(name) + for name in spack.repo.all_package_names(): + spack.repo.get(name) def test_get_all_packages(self): @@ -47,14 +47,14 @@ def test_get_all_packages(self): def ztest_get_all_mock_packages(self): """Get the mock packages once each too.""" db = RepoPath(spack.mock_packages_path) - spack.db.swap(db) + spack.repo.swap(db) self.check_db() - spack.db.swap(db) + spack.repo.swap(db) def ztest_url_versions(self): """Check URLs for regular packages, if they are explicitly defined.""" - for pkg in spack.db.all_packages(): + for pkg in spack.repo.all_packages(): for v, vattrs in pkg.versions.items(): if 'url' in vattrs: # If there is a url for the version check it. diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 8a786e364e4..e39def2ff2e 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -35,11 +35,11 @@ class PackagesTest(MockPackagesTest): def test_load_package(self): - pkg = spack.db.get('mpich') + pkg = spack.repo.get('mpich') def test_package_name(self): - pkg = spack.db.get('mpich') + pkg = spack.repo.get('mpich') self.assertEqual(pkg.name, 'mpich') @@ -50,7 +50,7 @@ def test_package_filename(self): def test_package_name(self): - pkg = spack.db.get('mpich') + pkg = spack.repo.get('mpich') self.assertEqual(pkg.name, 'mpich') diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index 5779d31ed2f..d2f7ea45b23 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -54,8 +54,8 @@ def pyfiles(self, *search_paths): def package_py_files(self): - for name in spack.db.all_package_names(): - yield spack.db.filename_for_package_name(name) + for name in spack.repo.all_package_names(): + yield spack.repo.filename_for_package_name(name) def check_python_versions(self, *files): diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 184fe8faa10..8abd089847e 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -49,7 +49,7 @@ def setUp(self): spec = Spec('svn-test') spec.concretize() - self.pkg = spack.db.get(spec, new=True) + self.pkg = spack.repo.get(spec, new=True) def tearDown(self): diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index 71682729979..477ca0436d9 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -79,15 +79,15 @@ def setUp(self): pkgX.installed = False pkgY.installed = False - self.saved_db = spack.db + self.saved_db = spack.repo pkgDb = MockPackageDb({specX:pkgX, specY:pkgY}) - spack.db = pkgDb + spack.repo = pkgDb def tearDown(self): super(UnitInstallTest, self).tearDown() - spack.db = self.saved_db + spack.repo = self.saved_db def test_installing_both(self): mo = MockOutput() From 3ce99cf4e18b599a42041a486b82498a0f43a488 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 25 Nov 2015 11:02:51 -0800 Subject: [PATCH 095/910] fix for the shell arg handling issues The setup-env.sh script now saves off the argument array before shifting off all the arguments so that the final command invocation doesn't arbitrarily requote the arguments. All external quoting is preserved now. This has been thoroughly tested with zsh, but should probably be tested with bash etc. before acceptance. --- share/spack/setup-env.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index b90846d28fc..8ee42bb10f8 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -56,6 +56,12 @@ # spack dotfiles. ######################################################################## function spack { + # save raw arguments into an array before butchering them + args=() + for a in "$@"; do + # yup, this is awful, blame bash2 compat + args=("${args[@]}" "$a") + done # accumulate initial flags for main spack command _sp_flags="" while [[ "$1" =~ ^- ]]; do @@ -116,7 +122,7 @@ function spack { esac ;; *) - command spack $_sp_flags $_sp_subcommand $_sp_spec + command spack "${args[@]}" ;; esac } From 60f7756626271b46ee7d32a80ba1e78b337d6f91 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 25 Nov 2015 12:28:43 -0800 Subject: [PATCH 096/910] Executable.__call__ original semantics were to avoid redirecting process output to the output stream if return_output was set. This branch undid that and this commit restores those semantics. --- lib/spack/spack/util/executable.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index 6848a7e2a54..6088d4514e6 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -100,7 +100,8 @@ def streamify(arg, mode): out, err = proc.communicate() self.returncode = proc.returncode - output.write(out) + if not return_output: + output.write(out) error.write(err) rc = proc.returncode From f8ffb005c8780c786c7b0adc3ff5e314d410bbee Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 25 Nov 2015 15:03:48 -0800 Subject: [PATCH 097/910] make cram an extension of python --- var/spack/packages/cram/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/cram/package.py b/var/spack/packages/cram/package.py index 4b8ec56f25f..1f1722993f9 100644 --- a/var/spack/packages/cram/package.py +++ b/var/spack/packages/cram/package.py @@ -7,6 +7,7 @@ class Cram(Package): version('1.0.1', 'c73711e945cf5dc603e44395f6647f5e') + extends('python') depends_on("mpi") def install(self, spec, prefix): From d50a18d9ebe38137690dfd0a6972616c0e07271c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 25 Nov 2015 17:10:45 -0800 Subject: [PATCH 098/910] XML output for unit tests is now enabled with an option (disabled by default) --- lib/spack/spack/cmd/test.py | 21 ++++++++++++++------- lib/spack/spack/test/__init__.py | 12 +++++++----- lib/spack/spack/test/tally_plugin.py | 1 + 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index 001ebfaef0b..f93d1268517 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -39,9 +39,12 @@ def setup_parser(subparser): 'names', nargs='*', help="Names of tests to run.") subparser.add_argument( '-l', '--list', action='store_true', dest='list', help="Show available tests") - # TODO: make XML output optional subparser.add_argument( - '-d', '--outputDir', dest='outputDir', help="Nose creates XML files in this directory") + '--createXmlOutput', action='store_true', dest='createXmlOutput', + help="Create JUnit XML from test results") + subparser.add_argument( + '--xmlOutputDir', dest='xmlOutputDir', + help="Nose creates XML files in this directory") subparser.add_argument( '-v', '--verbose', action='store_true', dest='verbose', help="verbose output") @@ -53,10 +56,14 @@ def test(parser, args): colify(spack.test.list_tests(), indent=2) else: - if not args.outputDir: - outputDir = join_path(os.getcwd(), "test-output") + if not args.createXmlOutput: + outputDir = None else: - outputDir = args.outputDir - if not os.path.exists(outputDir): - mkdirp(outputDir) + if not args.xmlOutputDir: + outputDir = join_path(os.getcwd(), "test-output") + else: + outputDir = os.path.abspath(args.xmlOutputDir) + + if not os.path.exists(outputDir): + mkdirp(outputDir) spack.test.run(args.names, outputDir, args.verbose) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 158e8bdfcb3..cb75f15ca19 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -92,11 +92,13 @@ def run(names, outputDir, verbose=False): tty.msg("Running test: %s" % test) - xmlOutputFname = "unittests-{0}.xml".format(test) - xmlOutputPath = join_path(outputDir, xmlOutputFname) - runOpts = ["--with-%s" % spack.test.tally_plugin.Tally.name, - "--with-xunit", - "--xunit-file={0}".format(xmlOutputPath)] + runOpts = ["--with-%s" % spack.test.tally_plugin.Tally.name] + + if outputDir: + xmlOutputFname = "unittests-{0}.xml".format(test) + xmlOutputPath = join_path(outputDir, xmlOutputFname) + runOpts += ["--with-xunit", + "--xunit-file={0}".format(xmlOutputPath)] argv = [""] + runOpts + [module] result = nose.run(argv=argv, addplugins=[tally]) diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index e314fae6cef..c167d585294 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -35,6 +35,7 @@ def __init__(self): self.failCount = 0 self.errorCount = 0 + # TODO: this doesn't account for the possibility of skipped tests @property def numberOfTests(self): return self.errorCount + self.failCount + self.successCount From fc96f62794a5c718afb5904d93bb9370ed4a261d Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 25 Nov 2015 18:08:17 -0800 Subject: [PATCH 099/910] Updated documentation/naming in tally_plugin to reflect that the count of the total number of tests run excludes skipped tests --- lib/spack/spack/test/__init__.py | 2 +- lib/spack/spack/test/tally_plugin.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index cb75f15ca19..7da3895cecc 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -104,7 +104,7 @@ def run(names, outputDir, verbose=False): succeeded = not tally.failCount and not tally.errorCount tty.msg("Tests Complete.", - "%5d tests run" % tally.numberOfTests, + "%5d tests run" % tally.numberOfTestsRun, "%5d failures" % tally.failCount, "%5d errors" % tally.errorCount) diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index c167d585294..9ca898c47ce 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -35,9 +35,9 @@ def __init__(self): self.failCount = 0 self.errorCount = 0 - # TODO: this doesn't account for the possibility of skipped tests @property - def numberOfTests(self): + def numberOfTestsRun(self): + """Excludes skipped tests""" return self.errorCount + self.failCount + self.successCount def options(self, parser, env=os.environ): From c5ed7e2a15fe1b0a07cd50bbeee599e09f00a140 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 25 Nov 2015 18:33:52 -0800 Subject: [PATCH 100/910] On systems which have already installed nose (perhaps an older version) importing nose (which is also in Spack's external/ directory) outputs a warning (since there is more than one nose package). This avoids printing the warning to the user. --- bin/spack | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bin/spack b/bin/spack index e92d7cc273f..b4f715ab60f 100755 --- a/bin/spack +++ b/bin/spack @@ -41,6 +41,14 @@ sys.path.insert(0, SPACK_LIB_PATH) SPACK_EXTERNAL_LIBS = os.path.join(SPACK_LIB_PATH, "external") sys.path.insert(0, SPACK_EXTERNAL_LIBS) +import warnings +# Avoid warnings when nose is installed with the python exe being used to run +# spack. Note this must be done after Spack's external libs directory is added +# to sys.path. +with warnings.catch_warnings(): + warnings.simplefilter("ignore") + import nose + # Quick and dirty check to clean orphaned .pyc files left over from # previous revisions. These files were present in earlier versions of # Spack, were removed, but shadow system modules that Spack still From 1fe626ec7cd8b8a2bceb9e73dd9597d9f99813cf Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 26 Nov 2015 17:53:33 +0100 Subject: [PATCH 101/910] resource directive : sketch of implementation + clang / llvm use case --- lib/spack/llnl/util/lang.py | 2 +- lib/spack/spack/directives.py | 36 +++++++++++++++++--- lib/spack/spack/package.py | 51 ++++++++++++++++++++++++----- lib/spack/spack/resource.py | 37 +++++++++++++++++++++ var/spack/packages/clang/package.py | 7 ++++ var/spack/packages/llvm/package.py | 15 +++++++-- 6 files changed, 132 insertions(+), 16 deletions(-) create mode 100644 lib/spack/spack/resource.py diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 156ee34c9ef..e7263687946 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -112,7 +112,7 @@ def partition_list(elements, predicate): def caller_locals(): """This will return the locals of the *parent* of the caller. - This allows a fucntion to insert variables into its caller's + This allows a function to insert variables into its caller's scope. Yes, this is some black magic, and yes it's useful for implementing things like depends_on and provides. """ diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 78039ac6f95..4d8333fd7db 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -44,11 +44,12 @@ class OpenMpi(Package): * ``variant`` """ -__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version', - 'variant' ] +__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version', + 'variant', 'resource'] import re import inspect +import functools from llnl.util.lang import * @@ -60,7 +61,8 @@ class OpenMpi(Package): from spack.patch import Patch from spack.variant import Variant from spack.spec import Spec, parse_anonymous_spec - +from spack.resource import Resource +from spack.fetch_strategy import URLFetchStrategy # # This is a list of all directives, built up as they are defined in @@ -79,8 +81,8 @@ class directive(object): """Decorator for Spack directives. Spack directives allow you to modify a package while it is being - defined, e.g. to add version or depenency information. Directives - are one of the key pieces of Spack's package "langauge", which is + defined, e.g. to add version or dependency information. Directives + are one of the key pieces of Spack's package "language", which is embedded in python. Here's an example directive: @@ -141,6 +143,7 @@ def ensure_dicts(self, pkg): def __call__(self, directive_function): directives[directive_function.__name__] = self + @functools.wraps(directive_function) def wrapped(*args, **kwargs): pkg = DictWrapper(caller_locals()) self.ensure_dicts(pkg) @@ -259,6 +262,29 @@ def variant(pkg, name, default=False, description=""): pkg.variants[name] = Variant(default, description) +@directive('resources') +def resource(pkg, **kwargs): + """ + Define an external resource to be fetched and staged when building the package. Based on the keywords present in the + dictionary the appropriate FetchStrategy will be used for the resource. + + List of recognized keywords: + + * 'when' : represents the condition upon which the resource is needed (optional) + * 'destination' : path where to extract / checkout the resource (optional) + + """ + when = kwargs.get('when', pkg.name) + # FIXME : currently I assume destination to be a relative path (rooted at pkg.stage.source_path) + destination = kwargs.get('destination', "") + when_spec = parse_anonymous_spec(when, pkg.name) + resources = pkg.resources.setdefault(when_spec, []) + # FIXME : change URLFetchStrategy with a factory that selects based on kwargs + fetcher = URLFetchStrategy(**kwargs) + # FIXME : should we infer the name somehow if not passed ? + name = kwargs.get('name') + resources.append(Resource(name, fetcher, destination)) + class DirectiveError(spack.error.SpackError): """This is raised when something is wrong with a package directive.""" def __init__(self, directive, message): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c631a35bf30..9fee962df3b 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -639,26 +639,53 @@ def do_fetch(self): "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg) self.stage.fetch() + + ########## + # Fetch resources + resources = self._get_resources() + # FIXME : choose the unique name appropriately. Is there a function somewhere for the base name ? + pieces = [self.name, str(self.version), self.spec.dag_hash()] + for resource in resources: + resource_stage_folder = '-'.join(pieces + [resource.name]) + stage = Stage(resource.fetcher, name=resource_stage_folder) + resource.fetcher.set_stage(stage) + resource.fetcher.fetch() + ########## + self._fetch_time = time.time() - start_time if spack.do_checksum and self.version in self.versions: self.stage.check() - def do_stage(self): """Unpacks the fetched tarball, then changes into the expanded tarball directory.""" if not self.spec.concrete: raise ValueError("Can only stage concrete packages.") - self.do_fetch() + def _expand_archive(stage, name=self.name): + archive_dir = stage.source_path + if not archive_dir: + stage.expand_archive() + tty.msg("Created stage in %s." % stage.path) + else: + tty.msg("Already staged %s in %s." % (name, stage.path)) - archive_dir = self.stage.source_path - if not archive_dir: - self.stage.expand_archive() - tty.msg("Created stage in %s." % self.stage.path) - else: - tty.msg("Already staged %s in %s." % (self.name, self.stage.path)) + + self.do_fetch() + _expand_archive(self.stage) + + ########## + # Stage resources in appropriate path + resources = self._get_resources() + for resource in resources: + stage = resource.fetcher.stage + _expand_archive(stage, resource.name) + link_path = join_path(self.stage.source_path, resource.destination, os.path.basename(stage.source_path)) + if not os.path.exists(link_path): + # Create a symlink + os.symlink(stage.source_path, link_path) + ########## self.stage.chdir_to_source() @@ -730,6 +757,14 @@ def do_fake_install(self): mkdirp(self.prefix.man1) + def _get_resources(self): + resources = [] + # Select the resources that are needed for this build + for when_spec, resource_list in self.resources.items(): + if when_spec in self.spec: + resources.extend(resource_list) + return resources + def _build_logger(self, log_path): """Create a context manager to log build output.""" diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py new file mode 100644 index 00000000000..00ee2d49e4f --- /dev/null +++ b/lib/spack/spack/resource.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +""" +Describes an optional resource needed for a build. Typically a bunch of sources that can be built in-tree within another +package to enable optional features. +""" + +class Resource(object): + """ + Represents an optional resource. Aggregates a name, a fetcher and a destination. + """ + def __init__(self, name, fetcher, destination): + self.name = name + self.fetcher = fetcher + self.destination = destination diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py index 4f977bf9a44..d0869362f55 100644 --- a/var/spack/packages/clang/package.py +++ b/var/spack/packages/clang/package.py @@ -39,6 +39,13 @@ class Clang(Package): version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz') version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz') + ########## + # @3.7.0 + resource(name='clang-tools-extra', + url='http://llvm.org/releases/3.7.0/clang-tools-extra-3.7.0.src.tar.xz', + md5='d5a87dacb65d981a427a536f6964642e', destination='tools', when='@3.7.0') + ########## + def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index a6759c3033e..b68aa82affe 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -41,13 +41,24 @@ class Llvm(Package): depends_on('python@2.7:') + ########## + # @3.7.0 + # TODO : Add support for libc++ <- libc++ABI <- libunwind with variant? + resource(name='compiler-rt', + url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5', + destination='projects', when='@3.7.0') + resource(name='openmp', + url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f', + destination='projects', when='@3.7.0') + ########## + def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag with working_dir('spack-build', create=True): cmake('..', - '-DLLVM_REQUIRES_RTTI=1', - '-DPYTHON_EXECUTABLE=%s/bin/python' % spec['python'].prefix, + '-DLLVM_REQUIRES_RTTI:BOOL=ON', + '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix, *std_cmake_args) make() make("install") From 89d5127900dda96b2a583c4c1a9bdac8e51c1c15 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 26 Nov 2015 14:19:27 -0800 Subject: [PATCH 102/910] New, cleaner package repository structure. Package repositories now look like this: top-level-dir/ repo.yaml packages/ libelf/ package.py mpich/ package.py ... This leaves room at the top level for additional metadata, source, per-repo configs, indexes, etc., and it makes it easy to see that something is a spack repo (just look for repo.yaml and packages). --- lib/spack/spack/__init__.py | 15 +- lib/spack/spack/cmd/repo.py | 2 +- lib/spack/spack/repository.py | 176 ++++++++++++------ var/spack/mock_packages/_repo.yaml | 2 - var/spack/packages/_repo.yaml | 2 - .../builtin.mock/packages}/a/package.py | 0 .../builtin.mock/packages}/b/package.py | 0 .../builtin.mock/packages}/c/package.py | 0 .../packages}/callpath/package.py | 0 .../packages}/direct_mpich/package.py | 0 .../builtin.mock/packages}/dyninst/package.py | 0 .../builtin.mock/packages}/e/package.py | 0 .../builtin.mock/packages}/fake/package.py | 0 .../packages}/git-test/package.py | 0 .../builtin.mock/packages}/hg-test/package.py | 0 .../packages}/indirect_mpich/package.py | 0 .../packages}/libdwarf/package.py | 0 .../builtin.mock/packages}/libelf/package.py | 0 .../builtin.mock/packages}/mpich/package.py | 0 .../builtin.mock/packages}/mpich2/package.py | 0 .../packages}/mpileaks/package.py | 0 .../packages}/multimethod/package.py | 0 .../packages}/optional-dep-test-2/package.py | 0 .../packages}/optional-dep-test-3/package.py | 0 .../packages}/optional-dep-test/package.py | 0 .../packages}/svn-test/package.py | 0 .../trivial_install_test_package/package.py | 0 .../builtin.mock/packages}/zmpi/package.py | 0 var/spack/repos/builtin.mock/repo.yaml | 2 + .../builtin}/packages/ImageMagick/package.py | 0 .../builtin}/packages/Mitos/package.py | 0 .../{ => repos/builtin}/packages/R/package.py | 0 .../packages/SAMRAI/no-tool-build.patch | 0 .../builtin}/packages/SAMRAI/package.py | 0 .../packages/activeharmony/package.py | 0 .../builtin}/packages/adept-utils/package.py | 0 .../builtin}/packages/apex/package.py | 0 .../builtin}/packages/arpack/package.py | 0 .../builtin}/packages/asciidoc/package.py | 0 .../builtin}/packages/atk/package.py | 0 .../builtin}/packages/atlas/package.py | 0 .../builtin}/packages/autoconf/package.py | 0 .../builtin}/packages/automaded/package.py | 0 .../builtin}/packages/automake/package.py | 0 .../builtin}/packages/bear/package.py | 0 .../builtin}/packages/bib2xhtml/package.py | 0 .../builtin}/packages/binutils/package.py | 0 .../builtin}/packages/bison/package.py | 0 .../builtin}/packages/boost/package.py | 0 .../packages/bowtie2/bowtie2-2.5.patch | 0 .../builtin}/packages/bowtie2/package.py | 0 .../builtin}/packages/boxlib/package.py | 0 .../builtin}/packages/bzip2/package.py | 0 .../builtin}/packages/cairo/package.py | 0 .../builtin}/packages/callpath/package.py | 0 .../builtin}/packages/cblas/package.py | 0 .../builtin}/packages/cgm/package.py | 0 .../builtin}/packages/clang/package.py | 0 .../builtin}/packages/cloog/package.py | 0 .../builtin}/packages/cmake/package.py | 0 .../builtin}/packages/coreutils/package.py | 0 .../builtin}/packages/cppcheck/package.py | 0 .../builtin}/packages/cram/package.py | 0 .../builtin}/packages/cscope/package.py | 0 .../builtin}/packages/cube/package.py | 0 .../builtin}/packages/czmq/package.py | 0 .../builtin}/packages/dbus/package.py | 0 .../builtin}/packages/docbook-xml/package.py | 0 .../builtin}/packages/doxygen/package.py | 0 .../builtin}/packages/dri2proto/package.py | 0 .../builtin}/packages/dtcmp/package.py | 0 .../builtin}/packages/dyninst/package.py | 0 .../builtin}/packages/elfutils/package.py | 0 .../builtin}/packages/extrae/package.py | 0 .../packages/exuberant-ctags/package.py | 0 .../builtin}/packages/fish/package.py | 0 .../builtin}/packages/flex/package.py | 0 .../builtin}/packages/flux/package.py | 0 .../builtin}/packages/fontconfig/package.py | 0 .../builtin}/packages/freetype/package.py | 0 .../builtin}/packages/gasnet/package.py | 0 .../builtin}/packages/gcc/package.py | 0 .../builtin}/packages/gdk-pixbuf/package.py | 0 .../builtin}/packages/geos/package.py | 0 .../builtin}/packages/gflags/package.py | 0 .../builtin}/packages/ghostscript/package.py | 0 .../builtin}/packages/git/package.py | 0 .../builtin}/packages/glib/package.py | 0 .../builtin}/packages/glm/package.py | 0 .../builtin}/packages/global/package.py | 0 .../builtin}/packages/glog/package.py | 0 .../builtin}/packages/gmp/package.py | 0 .../builtin}/packages/gnutls/package.py | 0 .../builtin}/packages/gperf/package.py | 0 .../builtin}/packages/gperftools/package.py | 0 .../builtin}/packages/graphlib/package.py | 0 .../builtin}/packages/graphviz/package.py | 0 .../builtin}/packages/gtkplus/package.py | 0 .../builtin}/packages/harfbuzz/package.py | 0 .../builtin}/packages/hdf5/package.py | 0 .../builtin}/packages/hwloc/package.py | 0 .../builtin}/packages/hypre/package.py | 0 .../builtin}/packages/icu/package.py | 0 .../builtin}/packages/icu4c/package.py | 0 .../builtin}/packages/isl/package.py | 0 .../builtin}/packages/jdk/package.py | 0 .../builtin}/packages/jpeg/package.py | 0 .../builtin}/packages/launchmon/package.py | 0 .../packages/launchmon/patch.lmon_install_dir | 0 .../builtin}/packages/lcms/package.py | 0 .../builtin}/packages/leveldb/package.py | 0 .../builtin}/packages/libNBC/package.py | 0 .../builtin}/packages/libarchive/package.py | 0 .../builtin}/packages/libcircle/package.py | 0 .../builtin}/packages/libdrm/package.py | 0 .../builtin}/packages/libdwarf/package.py | 0 .../builtin}/packages/libelf/package.py | 0 .../builtin}/packages/libevent/package.py | 0 .../builtin}/packages/libffi/package.py | 0 .../builtin}/packages/libgcrypt/package.py | 0 .../builtin}/packages/libgpg-error/package.py | 0 .../packages/libjpeg-turbo/package.py | 0 .../builtin}/packages/libjson-c/package.py | 0 .../builtin}/packages/libmng/package.py | 0 .../builtin}/packages/libmonitor/package.py | 0 .../builtin}/packages/libpciaccess/package.py | 0 .../builtin}/packages/libpng/package.py | 0 .../builtin}/packages/libsodium/package.py | 0 .../builtin}/packages/libtiff/package.py | 0 .../builtin}/packages/libtool/package.py | 0 .../builtin}/packages/libunwind/package.py | 0 .../builtin}/packages/libuuid/package.py | 0 .../builtin}/packages/libxcb/package.py | 0 .../builtin}/packages/libxml2/package.py | 0 .../builtin}/packages/libxshmfence/package.py | 0 .../builtin}/packages/libxslt/package.py | 0 .../builtin}/packages/llvm-lld/package.py | 0 .../builtin}/packages/llvm/package.py | 0 .../builtin}/packages/lmdb/package.py | 0 .../builtin}/packages/lua/package.py | 0 .../builtin}/packages/lwgrp/package.py | 0 .../builtin}/packages/lwm2/package.py | 0 .../builtin}/packages/matio/package.py | 0 .../builtin}/packages/memaxes/package.py | 0 .../builtin}/packages/mesa/package.py | 0 .../builtin}/packages/metis/package.py | 0 .../builtin}/packages/mpc/package.py | 0 .../builtin}/packages/mpe2/mpe2.patch | 0 .../builtin}/packages/mpe2/package.py | 0 .../builtin}/packages/mpfr/package.py | 0 .../packages/mpibash/mpibash-4.3.patch | 0 .../builtin}/packages/mpibash/package.py | 0 .../builtin}/packages/mpich/package.py | 0 .../builtin}/packages/mpileaks/package.py | 0 .../builtin}/packages/mrnet/package.py | 0 .../builtin}/packages/munge/package.py | 0 .../builtin}/packages/muster/package.py | 0 .../ad_lustre_rwcontig_open_source.patch | 0 .../builtin}/packages/mvapich2/package.py | 0 .../builtin}/packages/nasm/package.py | 0 .../builtin}/packages/ncdu/package.py | 0 .../builtin}/packages/ncurses/package.py | 0 .../packages/netcdf/netcdf-4.3.3-mpi.patch | 0 .../builtin}/packages/netcdf/package.py | 0 .../builtin}/packages/netgauge/package.py | 0 .../builtin}/packages/netlib-blas/package.py | 0 .../packages/netlib-lapack/package.py | 0 .../builtin}/packages/nettle/package.py | 0 .../builtin}/packages/ompss/package.py | 0 .../builtin}/packages/ompt-openmp/package.py | 0 .../builtin}/packages/opari2/package.py | 0 .../ad_lustre_rwcontig_open_source.patch | 0 .../packages/openmpi/llnl-platforms.patch | 0 .../builtin}/packages/openmpi/package.py | 0 .../builtin}/packages/openssl/package.py | 0 .../builtin}/packages/otf/package.py | 0 .../builtin}/packages/otf2/package.py | 0 .../builtin}/packages/pango/package.py | 0 .../builtin}/packages/papi/package.py | 0 .../builtin}/packages/paraver/package.py | 0 .../builtin}/packages/paraview/package.py | 0 .../builtin}/packages/parmetis/package.py | 0 .../builtin}/packages/parpack/package.py | 0 .../builtin}/packages/pcre/package.py | 0 .../builtin}/packages/petsc/package.py | 0 .../builtin}/packages/pidx/package.py | 0 .../builtin}/packages/pixman/package.py | 0 .../builtin}/packages/pkg-config/package.py | 0 .../packages/pmgr_collective/package.py | 0 .../builtin}/packages/postgresql/package.py | 0 .../builtin}/packages/ppl/package.py | 0 .../builtin}/packages/protobuf/package.py | 0 .../builtin}/packages/py-basemap/package.py | 0 .../builtin}/packages/py-biopython/package.py | 0 .../builtin}/packages/py-cffi/package.py | 0 .../builtin}/packages/py-cython/package.py | 0 .../builtin}/packages/py-dateutil/package.py | 0 .../builtin}/packages/py-epydoc/package.py | 0 .../builtin}/packages/py-genders/package.py | 0 .../builtin}/packages/py-gnuplot/package.py | 0 .../builtin}/packages/py-h5py/package.py | 0 .../builtin}/packages/py-ipython/package.py | 0 .../builtin}/packages/py-libxml2/package.py | 0 .../builtin}/packages/py-lockfile/package.py | 0 .../builtin}/packages/py-mako/package.py | 0 .../packages/py-matplotlib/package.py | 0 .../builtin}/packages/py-mock/package.py | 0 .../builtin}/packages/py-mpi4py/package.py | 0 .../builtin}/packages/py-mx/package.py | 0 .../builtin}/packages/py-nose/package.py | 0 .../builtin}/packages/py-numpy/package.py | 0 .../builtin}/packages/py-pandas/package.py | 0 .../builtin}/packages/py-pexpect/package.py | 0 .../builtin}/packages/py-pil/package.py | 0 .../builtin}/packages/py-pmw/package.py | 0 .../builtin}/packages/py-pychecker/package.py | 0 .../builtin}/packages/py-pycparser/package.py | 0 .../packages/py-pyelftools/package.py | 0 .../builtin}/packages/py-pygments/package.py | 0 .../builtin}/packages/py-pylint/package.py | 0 .../builtin}/packages/py-pypar/package.py | 0 .../builtin}/packages/py-pyparsing/package.py | 0 .../builtin}/packages/py-pyqt/package.py | 0 .../builtin}/packages/py-pyside/package.py | 0 .../packages/py-python-daemon/package.py | 0 .../builtin}/packages/py-pytz/package.py | 0 .../builtin}/packages/py-rpy2/package.py | 0 .../packages/py-scientificpython/package.py | 0 .../packages/py-scikit-learn/package.py | 0 .../builtin}/packages/py-scipy/package.py | 0 .../packages/py-setuptools/package.py | 0 .../builtin}/packages/py-shiboken/package.py | 0 .../builtin}/packages/py-sip/package.py | 0 .../builtin}/packages/py-six/package.py | 0 .../builtin}/packages/py-sphinx/package.py | 0 .../builtin}/packages/py-sympy/package.py | 0 .../packages/py-virtualenv/package.py | 0 .../builtin}/packages/py-yapf/package.py | 0 .../builtin}/packages/python/package.py | 0 .../builtin}/packages/qhull/package.py | 0 .../builtin}/packages/qt/package.py | 0 .../builtin}/packages/qthreads/package.py | 0 .../builtin}/packages/ravel/package.py | 0 .../builtin}/packages/readline/package.py | 0 .../rose/add_spack_compiler_recognition.patch | 0 .../builtin}/packages/rose/package.py | 0 .../builtin}/packages/ruby/package.py | 0 .../builtin}/packages/samtools/package.py | 0 .../packages/samtools/samtools1.2.patch | 0 .../builtin}/packages/scalasca/package.py | 0 .../builtin}/packages/scorep/package.py | 0 .../builtin}/packages/scotch/package.py | 0 .../builtin}/packages/scr/package.py | 0 .../builtin}/packages/silo/package.py | 0 .../builtin}/packages/snappy/package.py | 0 .../builtin}/packages/spindle/package.py | 0 .../builtin}/packages/sqlite/package.py | 0 .../packages/stat/configure_mpicxx.patch | 0 .../builtin}/packages/stat/package.py | 0 .../builtin}/packages/sundials/package.py | 0 .../builtin}/packages/swig/package.py | 0 .../builtin}/packages/task/package.py | 0 .../builtin}/packages/taskd/package.py | 0 .../builtin}/packages/tau/package.py | 0 .../builtin}/packages/tcl/package.py | 0 .../packages/the_silver_searcher/package.py | 0 .../builtin}/packages/thrift/package.py | 0 .../builtin}/packages/tk/package.py | 0 .../builtin}/packages/tmux/package.py | 0 .../builtin}/packages/tmuxinator/package.py | 0 .../builtin}/packages/trilinos/package.py | 0 .../builtin}/packages/uncrustify/package.py | 0 .../builtin}/packages/util-linux/package.py | 0 .../builtin}/packages/vim/package.py | 0 .../builtin}/packages/vtk/package.py | 0 .../builtin}/packages/wget/package.py | 0 .../builtin}/packages/wx/package.py | 0 .../builtin}/packages/wxpropgrid/package.py | 0 .../builtin}/packages/xcb-proto/package.py | 0 .../builtin}/packages/xz/package.py | 0 .../builtin}/packages/yasm/package.py | 0 .../builtin}/packages/zeromq/package.py | 0 .../builtin}/packages/zlib/package.py | 0 .../builtin}/packages/zsh/package.py | 0 var/spack/repos/builtin/repo.yaml | 2 + 285 files changed, 137 insertions(+), 64 deletions(-) delete mode 100644 var/spack/mock_packages/_repo.yaml delete mode 100644 var/spack/packages/_repo.yaml rename var/spack/{mock_packages => repos/builtin.mock/packages}/a/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/b/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/c/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/callpath/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/direct_mpich/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/dyninst/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/e/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/fake/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/git-test/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/hg-test/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/indirect_mpich/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/libdwarf/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/libelf/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/mpich/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/mpich2/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/mpileaks/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/multimethod/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/optional-dep-test-2/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/optional-dep-test-3/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/optional-dep-test/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/svn-test/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/trivial_install_test_package/package.py (100%) rename var/spack/{mock_packages => repos/builtin.mock/packages}/zmpi/package.py (100%) create mode 100644 var/spack/repos/builtin.mock/repo.yaml rename var/spack/{ => repos/builtin}/packages/ImageMagick/package.py (100%) rename var/spack/{ => repos/builtin}/packages/Mitos/package.py (100%) rename var/spack/{ => repos/builtin}/packages/R/package.py (100%) rename var/spack/{ => repos/builtin}/packages/SAMRAI/no-tool-build.patch (100%) rename var/spack/{ => repos/builtin}/packages/SAMRAI/package.py (100%) rename var/spack/{ => repos/builtin}/packages/activeharmony/package.py (100%) rename var/spack/{ => repos/builtin}/packages/adept-utils/package.py (100%) rename var/spack/{ => repos/builtin}/packages/apex/package.py (100%) rename var/spack/{ => repos/builtin}/packages/arpack/package.py (100%) rename var/spack/{ => repos/builtin}/packages/asciidoc/package.py (100%) rename var/spack/{ => repos/builtin}/packages/atk/package.py (100%) rename var/spack/{ => repos/builtin}/packages/atlas/package.py (100%) rename var/spack/{ => repos/builtin}/packages/autoconf/package.py (100%) rename var/spack/{ => repos/builtin}/packages/automaded/package.py (100%) rename var/spack/{ => repos/builtin}/packages/automake/package.py (100%) rename var/spack/{ => repos/builtin}/packages/bear/package.py (100%) rename var/spack/{ => repos/builtin}/packages/bib2xhtml/package.py (100%) rename var/spack/{ => repos/builtin}/packages/binutils/package.py (100%) rename var/spack/{ => repos/builtin}/packages/bison/package.py (100%) rename var/spack/{ => repos/builtin}/packages/boost/package.py (100%) rename var/spack/{ => repos/builtin}/packages/bowtie2/bowtie2-2.5.patch (100%) rename var/spack/{ => repos/builtin}/packages/bowtie2/package.py (100%) rename var/spack/{ => repos/builtin}/packages/boxlib/package.py (100%) rename var/spack/{ => repos/builtin}/packages/bzip2/package.py (100%) rename var/spack/{ => repos/builtin}/packages/cairo/package.py (100%) rename var/spack/{ => repos/builtin}/packages/callpath/package.py (100%) rename var/spack/{ => repos/builtin}/packages/cblas/package.py (100%) rename var/spack/{ => repos/builtin}/packages/cgm/package.py (100%) rename var/spack/{ => repos/builtin}/packages/clang/package.py (100%) rename var/spack/{ => repos/builtin}/packages/cloog/package.py (100%) rename var/spack/{ => repos/builtin}/packages/cmake/package.py (100%) rename var/spack/{ => repos/builtin}/packages/coreutils/package.py (100%) rename var/spack/{ => repos/builtin}/packages/cppcheck/package.py (100%) rename var/spack/{ => repos/builtin}/packages/cram/package.py (100%) rename var/spack/{ => repos/builtin}/packages/cscope/package.py (100%) rename var/spack/{ => repos/builtin}/packages/cube/package.py (100%) rename var/spack/{ => repos/builtin}/packages/czmq/package.py (100%) rename var/spack/{ => repos/builtin}/packages/dbus/package.py (100%) rename var/spack/{ => repos/builtin}/packages/docbook-xml/package.py (100%) rename var/spack/{ => repos/builtin}/packages/doxygen/package.py (100%) rename var/spack/{ => repos/builtin}/packages/dri2proto/package.py (100%) rename var/spack/{ => repos/builtin}/packages/dtcmp/package.py (100%) rename var/spack/{ => repos/builtin}/packages/dyninst/package.py (100%) rename var/spack/{ => repos/builtin}/packages/elfutils/package.py (100%) rename var/spack/{ => repos/builtin}/packages/extrae/package.py (100%) rename var/spack/{ => repos/builtin}/packages/exuberant-ctags/package.py (100%) rename var/spack/{ => repos/builtin}/packages/fish/package.py (100%) rename var/spack/{ => repos/builtin}/packages/flex/package.py (100%) rename var/spack/{ => repos/builtin}/packages/flux/package.py (100%) rename var/spack/{ => repos/builtin}/packages/fontconfig/package.py (100%) rename var/spack/{ => repos/builtin}/packages/freetype/package.py (100%) rename var/spack/{ => repos/builtin}/packages/gasnet/package.py (100%) rename var/spack/{ => repos/builtin}/packages/gcc/package.py (100%) rename var/spack/{ => repos/builtin}/packages/gdk-pixbuf/package.py (100%) rename var/spack/{ => repos/builtin}/packages/geos/package.py (100%) rename var/spack/{ => repos/builtin}/packages/gflags/package.py (100%) rename var/spack/{ => repos/builtin}/packages/ghostscript/package.py (100%) rename var/spack/{ => repos/builtin}/packages/git/package.py (100%) rename var/spack/{ => repos/builtin}/packages/glib/package.py (100%) rename var/spack/{ => repos/builtin}/packages/glm/package.py (100%) rename var/spack/{ => repos/builtin}/packages/global/package.py (100%) rename var/spack/{ => repos/builtin}/packages/glog/package.py (100%) rename var/spack/{ => repos/builtin}/packages/gmp/package.py (100%) rename var/spack/{ => repos/builtin}/packages/gnutls/package.py (100%) rename var/spack/{ => repos/builtin}/packages/gperf/package.py (100%) rename var/spack/{ => repos/builtin}/packages/gperftools/package.py (100%) rename var/spack/{ => repos/builtin}/packages/graphlib/package.py (100%) rename var/spack/{ => repos/builtin}/packages/graphviz/package.py (100%) rename var/spack/{ => repos/builtin}/packages/gtkplus/package.py (100%) rename var/spack/{ => repos/builtin}/packages/harfbuzz/package.py (100%) rename var/spack/{ => repos/builtin}/packages/hdf5/package.py (100%) rename var/spack/{ => repos/builtin}/packages/hwloc/package.py (100%) rename var/spack/{ => repos/builtin}/packages/hypre/package.py (100%) rename var/spack/{ => repos/builtin}/packages/icu/package.py (100%) rename var/spack/{ => repos/builtin}/packages/icu4c/package.py (100%) rename var/spack/{ => repos/builtin}/packages/isl/package.py (100%) rename var/spack/{ => repos/builtin}/packages/jdk/package.py (100%) rename var/spack/{ => repos/builtin}/packages/jpeg/package.py (100%) rename var/spack/{ => repos/builtin}/packages/launchmon/package.py (100%) rename var/spack/{ => repos/builtin}/packages/launchmon/patch.lmon_install_dir (100%) rename var/spack/{ => repos/builtin}/packages/lcms/package.py (100%) rename var/spack/{ => repos/builtin}/packages/leveldb/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libNBC/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libarchive/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libcircle/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libdrm/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libdwarf/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libelf/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libevent/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libffi/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libgcrypt/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libgpg-error/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libjpeg-turbo/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libjson-c/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libmng/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libmonitor/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libpciaccess/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libpng/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libsodium/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libtiff/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libtool/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libunwind/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libuuid/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libxcb/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libxml2/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libxshmfence/package.py (100%) rename var/spack/{ => repos/builtin}/packages/libxslt/package.py (100%) rename var/spack/{ => repos/builtin}/packages/llvm-lld/package.py (100%) rename var/spack/{ => repos/builtin}/packages/llvm/package.py (100%) rename var/spack/{ => repos/builtin}/packages/lmdb/package.py (100%) rename var/spack/{ => repos/builtin}/packages/lua/package.py (100%) rename var/spack/{ => repos/builtin}/packages/lwgrp/package.py (100%) rename var/spack/{ => repos/builtin}/packages/lwm2/package.py (100%) rename var/spack/{ => repos/builtin}/packages/matio/package.py (100%) rename var/spack/{ => repos/builtin}/packages/memaxes/package.py (100%) rename var/spack/{ => repos/builtin}/packages/mesa/package.py (100%) rename var/spack/{ => repos/builtin}/packages/metis/package.py (100%) rename var/spack/{ => repos/builtin}/packages/mpc/package.py (100%) rename var/spack/{ => repos/builtin}/packages/mpe2/mpe2.patch (100%) rename var/spack/{ => repos/builtin}/packages/mpe2/package.py (100%) rename var/spack/{ => repos/builtin}/packages/mpfr/package.py (100%) rename var/spack/{ => repos/builtin}/packages/mpibash/mpibash-4.3.patch (100%) rename var/spack/{ => repos/builtin}/packages/mpibash/package.py (100%) rename var/spack/{ => repos/builtin}/packages/mpich/package.py (100%) rename var/spack/{ => repos/builtin}/packages/mpileaks/package.py (100%) rename var/spack/{ => repos/builtin}/packages/mrnet/package.py (100%) rename var/spack/{ => repos/builtin}/packages/munge/package.py (100%) rename var/spack/{ => repos/builtin}/packages/muster/package.py (100%) rename var/spack/{ => repos/builtin}/packages/mvapich2/ad_lustre_rwcontig_open_source.patch (100%) rename var/spack/{ => repos/builtin}/packages/mvapich2/package.py (100%) rename var/spack/{ => repos/builtin}/packages/nasm/package.py (100%) rename var/spack/{ => repos/builtin}/packages/ncdu/package.py (100%) rename var/spack/{ => repos/builtin}/packages/ncurses/package.py (100%) rename var/spack/{ => repos/builtin}/packages/netcdf/netcdf-4.3.3-mpi.patch (100%) rename var/spack/{ => repos/builtin}/packages/netcdf/package.py (100%) rename var/spack/{ => repos/builtin}/packages/netgauge/package.py (100%) rename var/spack/{ => repos/builtin}/packages/netlib-blas/package.py (100%) rename var/spack/{ => repos/builtin}/packages/netlib-lapack/package.py (100%) rename var/spack/{ => repos/builtin}/packages/nettle/package.py (100%) rename var/spack/{ => repos/builtin}/packages/ompss/package.py (100%) rename var/spack/{ => repos/builtin}/packages/ompt-openmp/package.py (100%) rename var/spack/{ => repos/builtin}/packages/opari2/package.py (100%) rename var/spack/{ => repos/builtin}/packages/openmpi/ad_lustre_rwcontig_open_source.patch (100%) rename var/spack/{ => repos/builtin}/packages/openmpi/llnl-platforms.patch (100%) rename var/spack/{ => repos/builtin}/packages/openmpi/package.py (100%) rename var/spack/{ => repos/builtin}/packages/openssl/package.py (100%) rename var/spack/{ => repos/builtin}/packages/otf/package.py (100%) rename var/spack/{ => repos/builtin}/packages/otf2/package.py (100%) rename var/spack/{ => repos/builtin}/packages/pango/package.py (100%) rename var/spack/{ => repos/builtin}/packages/papi/package.py (100%) rename var/spack/{ => repos/builtin}/packages/paraver/package.py (100%) rename var/spack/{ => repos/builtin}/packages/paraview/package.py (100%) rename var/spack/{ => repos/builtin}/packages/parmetis/package.py (100%) rename var/spack/{ => repos/builtin}/packages/parpack/package.py (100%) rename var/spack/{ => repos/builtin}/packages/pcre/package.py (100%) rename var/spack/{ => repos/builtin}/packages/petsc/package.py (100%) rename var/spack/{ => repos/builtin}/packages/pidx/package.py (100%) rename var/spack/{ => repos/builtin}/packages/pixman/package.py (100%) rename var/spack/{ => repos/builtin}/packages/pkg-config/package.py (100%) rename var/spack/{ => repos/builtin}/packages/pmgr_collective/package.py (100%) rename var/spack/{ => repos/builtin}/packages/postgresql/package.py (100%) rename var/spack/{ => repos/builtin}/packages/ppl/package.py (100%) rename var/spack/{ => repos/builtin}/packages/protobuf/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-basemap/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-biopython/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-cffi/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-cython/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-dateutil/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-epydoc/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-genders/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-gnuplot/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-h5py/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-ipython/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-libxml2/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-lockfile/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-mako/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-matplotlib/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-mock/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-mpi4py/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-mx/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-nose/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-numpy/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-pandas/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-pexpect/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-pil/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-pmw/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-pychecker/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-pycparser/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-pyelftools/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-pygments/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-pylint/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-pypar/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-pyparsing/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-pyqt/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-pyside/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-python-daemon/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-pytz/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-rpy2/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-scientificpython/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-scikit-learn/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-scipy/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-setuptools/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-shiboken/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-sip/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-six/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-sphinx/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-sympy/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-virtualenv/package.py (100%) rename var/spack/{ => repos/builtin}/packages/py-yapf/package.py (100%) rename var/spack/{ => repos/builtin}/packages/python/package.py (100%) rename var/spack/{ => repos/builtin}/packages/qhull/package.py (100%) rename var/spack/{ => repos/builtin}/packages/qt/package.py (100%) rename var/spack/{ => repos/builtin}/packages/qthreads/package.py (100%) rename var/spack/{ => repos/builtin}/packages/ravel/package.py (100%) rename var/spack/{ => repos/builtin}/packages/readline/package.py (100%) rename var/spack/{ => repos/builtin}/packages/rose/add_spack_compiler_recognition.patch (100%) rename var/spack/{ => repos/builtin}/packages/rose/package.py (100%) rename var/spack/{ => repos/builtin}/packages/ruby/package.py (100%) rename var/spack/{ => repos/builtin}/packages/samtools/package.py (100%) rename var/spack/{ => repos/builtin}/packages/samtools/samtools1.2.patch (100%) rename var/spack/{ => repos/builtin}/packages/scalasca/package.py (100%) rename var/spack/{ => repos/builtin}/packages/scorep/package.py (100%) rename var/spack/{ => repos/builtin}/packages/scotch/package.py (100%) rename var/spack/{ => repos/builtin}/packages/scr/package.py (100%) rename var/spack/{ => repos/builtin}/packages/silo/package.py (100%) rename var/spack/{ => repos/builtin}/packages/snappy/package.py (100%) rename var/spack/{ => repos/builtin}/packages/spindle/package.py (100%) rename var/spack/{ => repos/builtin}/packages/sqlite/package.py (100%) rename var/spack/{ => repos/builtin}/packages/stat/configure_mpicxx.patch (100%) rename var/spack/{ => repos/builtin}/packages/stat/package.py (100%) rename var/spack/{ => repos/builtin}/packages/sundials/package.py (100%) rename var/spack/{ => repos/builtin}/packages/swig/package.py (100%) rename var/spack/{ => repos/builtin}/packages/task/package.py (100%) rename var/spack/{ => repos/builtin}/packages/taskd/package.py (100%) rename var/spack/{ => repos/builtin}/packages/tau/package.py (100%) rename var/spack/{ => repos/builtin}/packages/tcl/package.py (100%) rename var/spack/{ => repos/builtin}/packages/the_silver_searcher/package.py (100%) rename var/spack/{ => repos/builtin}/packages/thrift/package.py (100%) rename var/spack/{ => repos/builtin}/packages/tk/package.py (100%) rename var/spack/{ => repos/builtin}/packages/tmux/package.py (100%) rename var/spack/{ => repos/builtin}/packages/tmuxinator/package.py (100%) rename var/spack/{ => repos/builtin}/packages/trilinos/package.py (100%) rename var/spack/{ => repos/builtin}/packages/uncrustify/package.py (100%) rename var/spack/{ => repos/builtin}/packages/util-linux/package.py (100%) rename var/spack/{ => repos/builtin}/packages/vim/package.py (100%) rename var/spack/{ => repos/builtin}/packages/vtk/package.py (100%) rename var/spack/{ => repos/builtin}/packages/wget/package.py (100%) rename var/spack/{ => repos/builtin}/packages/wx/package.py (100%) rename var/spack/{ => repos/builtin}/packages/wxpropgrid/package.py (100%) rename var/spack/{ => repos/builtin}/packages/xcb-proto/package.py (100%) rename var/spack/{ => repos/builtin}/packages/xz/package.py (100%) rename var/spack/{ => repos/builtin}/packages/yasm/package.py (100%) rename var/spack/{ => repos/builtin}/packages/zeromq/package.py (100%) rename var/spack/{ => repos/builtin}/packages/zlib/package.py (100%) rename var/spack/{ => repos/builtin}/packages/zsh/package.py (100%) create mode 100644 var/spack/repos/builtin/repo.yaml diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 4f481ce9379..aab20cb2605 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -43,7 +43,7 @@ hooks_path = join_path(module_path, "hooks") var_path = join_path(spack_root, "var", "spack") stage_path = join_path(var_path, "stage") -packages_path = join_path(var_path, "packages") +repos_path = join_path(var_path, "repos") share_path = join_path(spack_root, "share", "spack") prefix = spack_root @@ -58,8 +58,12 @@ _repo_paths = spack.config.get_repos_config() if not _repo_paths: tty.die("Spack configuration contains no package repositories.") -repo = spack.repository.RepoPath(*_repo_paths) -sys.meta_path.append(repo) + +try: + repo = spack.repository.RepoPath(*_repo_paths) + sys.meta_path.append(repo) +except spack.repository.BadRepoError, e: + tty.die('Bad repository. %s' % e.message) # # Set up the installed packages database @@ -68,9 +72,10 @@ installed_db = Database(install_path) # -# Paths to mock files for testing. +# Paths to built-in Spack repositories. # -mock_packages_path = join_path(var_path, "mock_packages") +packages_path = join_path(repos_path, "builtin") +mock_packages_path = join_path(repos_path, "builtin.mock") mock_config_path = join_path(var_path, "mock_configs") mock_site_config = join_path(mock_config_path, "site_spackconfig") diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 85cc83730c4..395aa90bedf 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -32,7 +32,7 @@ import spack.spec import spack.config from spack.util.environment import get_path -from spack.repository import repo_config_filename +from spack.repository import repo_config_name import os import exceptions diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index c1545b36547..a2c0bbe1478 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -26,28 +26,32 @@ import exceptions import sys import inspect -import glob import imp import re -import itertools import traceback from bisect import bisect_left from external import yaml import llnl.util.tty as tty from llnl.util.filesystem import join_path -from llnl.util.lang import * import spack.error import spack.spec from spack.virtual import ProviderIndex from spack.util.naming import * -# Filename for package repo names -repo_config_filename = '_repo.yaml' +# +# Super-namespace for all packages. +# Package modules are imported as spack.pkg... +# +repo_namespace = 'spack.pkg' -# Filename for packages in repos. -package_file_name = 'package.py' +# +# These names describe how repos should be laid out in the filesystem. +# +repo_config_name = 'repo.yaml' # Top-level filename for repo config. +packages_dir_name = 'packages' # Top-level repo directory containing pkgs. +package_file_name = 'package.py' # Filename for packages in a repository. def _autospec(function): """Decorator that automatically converts the argument of a single-arg @@ -74,7 +78,10 @@ class RepoPath(object): combined results of the Repos in its list instead of on a single package repository. """ - def __init__(self, *repo_dirs): + def __init__(self, *repo_dirs, **kwargs): + # super-namespace for all packages in the RepoPath + self.super_namespace = kwargs.get('namespace', repo_namespace) + self.repos = [] self.by_namespace = NamespaceTrie() self.by_path = {} @@ -82,11 +89,9 @@ def __init__(self, *repo_dirs): self._all_package_names = [] self._provider_index = None + # Add each repo to this path. for root in repo_dirs: - # Try to make it a repo if it's not one. - if not isinstance(root, Repo): - repo = Repo(root) - # Add the repo to the path. + repo = Repo(root, self.super_namespace) self.put_last(repo) @@ -120,11 +125,11 @@ def _add(self, repo): repo, self.by_path[repo.root]) if repo.namespace in self.by_namespace: - raise DuplicateRepoError("Package repos cannot have the same name", + raise DuplicateRepoError("Package repos cannot provide the same namespace", repo, self.by_namespace[repo.namespace]) # Add repo to the pkg indexes - self.by_namespace[repo.namespace] = repo + self.by_namespace[repo.full_namespace] = repo self.by_path[repo.root] = repo # add names to the cached name list @@ -185,10 +190,10 @@ def find_module(self, fullname, path=None): # If it's a module in some repo, or if it is the repo's # namespace, let the repo handle it. for repo in self.repos: - if namespace == repo.namespace: + if namespace == repo.full_namespace: if repo.real_name(module_name): return repo - elif fullname == repo.namespace: + elif fullname == repo.full_namespace: return repo # No repo provides the namespace, but it is a valid prefix of @@ -200,13 +205,14 @@ def find_module(self, fullname, path=None): def load_module(self, fullname): - """Loads containing namespaces when necessary. + """Handles loading container namespaces when necessary. See ``Repo`` for how actual package modules are loaded. """ if fullname in sys.modules: return sys.modules[fullname] + # partition fullname into prefix and module name. namespace, dot, module_name = fullname.rpartition('.') @@ -252,41 +258,67 @@ class Repo(object): """Class representing a package repository in the filesystem. Each package repository must have a top-level configuration file - called `_repo.yaml`. + called `repo.yaml`. - Currently, `_repo.yaml` this must define: + Currently, `repo.yaml` this must define: `namespace`: A Python namespace where the repository's packages should live. """ - def __init__(self, root): - """Instantiate a package repository from a filesystem path.""" + def __init__(self, root, namespace=repo_namespace): + """Instantiate a package repository from a filesystem path. + + Arguments: + root The root directory of the repository. + + namespace A super-namespace that will contain the repo-defined + namespace (this is generally jsut `spack.pkg`). The + super-namespace is Spack's way of separating repositories + from other python namespaces. + + """ # Root directory, containing _repo.yaml and package dirs self.root = root - # Config file in /_repo.yaml - self.config_file = os.path.join(self.root, repo_config_filename) + # super-namespace for all packages in the Repo + self.super_namespace = namespace - # Read configuration from _repo.yaml + # check and raise BadRepoError on fail. + def check(condition, msg): + if not condition: raise BadRepoError(msg) + + # Validate repository layout. + self.config_file = join_path(self.root, repo_config_name) + check(os.path.isfile(self.config_file), + "No %s found in '%s'" % (repo_config_name, root)) + self.packages_path = join_path(self.root, packages_dir_name) + check(os.path.isdir(self.packages_path), + "No directory '%s' found in '%s'" % (repo_config_name, root)) + + # Read configuration and validate namespace config = self._read_config() - if not 'namespace' in config: - tty.die('Package repo in %s must define a namespace in %s.' - % (self.root, repo_config_filename)) + check('namespace' in config, '%s must define a namespace.' + % join_path(self.root, repo_config_name)) - # Check namespace in the repository configuration. self.namespace = config['namespace'] - if not re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', self.namespace): - tty.die(("Invalid namespace '%s' in '%s'. Namespaces must be " - "valid python identifiers separated by '.'") - % (self.namespace, self.root)) - self._names = self.namespace.split('.') + check(re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', self.namespace), + ("Invalid namespace '%s' in repo '%s'. " % (self.namespace, self.root)) + + "Namespaces must be valid python identifiers separated by '.'") + + # Set up 'full_namespace' to include the super-namespace + if self.super_namespace: + self.full_namespace = "%s.%s" % (self.super_namespace, self.namespace) + else: + self.full_namespace = self.namespace + + # Keep name components around for checking prefixes. + self._names = self.full_namespace.split('.') # These are internal cache variables. self._modules = {} self._classes = {} self._instances = {} - self._provider_index = None self._all_package_names = None @@ -301,11 +333,27 @@ def _create_namespace(self): we don't get runtime warnings from Python's module system. """ + parent = None for l in range(1, len(self._names)+1): ns = '.'.join(self._names[:l]) if not ns in sys.modules: - sys.modules[ns] = _make_namespace_module(ns) - sys.modules[ns].__loader__ = self + module = _make_namespace_module(ns) + module.__loader__ = self + sys.modules[ns] = module + + # Ensure the namespace is an atrribute of its parent, + # if it has not been set by something else already. + # + # This ensures that we can do things like: + # import spack.pkg.builtin.mpich as mpich + if parent: + modname = self._names[l-1] + if not hasattr(parent, modname): + setattr(parent, modname, module) + else: + # no need to set up a module, but keep track of the parent. + module = sys.modules[ns] + parent = module def real_name(self, import_name): @@ -349,7 +397,7 @@ def find_module(self, fullname, path=None): return self namespace, dot, module_name = fullname.rpartition('.') - if namespace == self.namespace: + if namespace == self.full_namespace: if self.real_name(module_name): return self @@ -369,14 +417,14 @@ def load_module(self, fullname): if self.is_prefix(fullname): module = _make_namespace_module(fullname) - elif namespace == self.namespace: + elif namespace == self.full_namespace: real_name = self.real_name(module_name) if not real_name: - raise ImportError("No module %s in repo %s" % (module_name, namespace)) + raise ImportError("No module %s in %s" % (module_name, self)) module = self._get_pkg_module(real_name) else: - raise ImportError("No module %s in repo %s" % (fullname, self.namespace)) + raise ImportError("No module %s in %s" % (fullname, self)) module.__loader__ = self sys.modules[fullname] = module @@ -392,7 +440,7 @@ def _read_config(self): if (not yaml_data or 'repo' not in yaml_data or not isinstance(yaml_data['repo'], dict)): tty.die("Invalid %s in repository %s" - % (repo_config_filename, self.root)) + % (repo_config_name, self.root)) return yaml_data['repo'] @@ -446,7 +494,7 @@ def extensions_for(self, extendee_spec): def dirname_for_package_name(self, pkg_name): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" - return join_path(self.root, pkg_name) + return join_path(self.packages_path, pkg_name) def filename_for_package_name(self, pkg_name): @@ -460,7 +508,6 @@ def filename_for_package_name(self, pkg_name): """ validate_module_name(pkg_name) pkg_dir = self.dirname_for_package_name(pkg_name) - return join_path(pkg_dir, package_file_name) @@ -469,12 +516,25 @@ def all_package_names(self): if self._all_package_names is None: self._all_package_names = [] - for pkg_name in os.listdir(self.root): - pkg_dir = join_path(self.root, pkg_name) - pkg_file = join_path(pkg_dir, package_file_name) - if os.path.isfile(pkg_file): - self._all_package_names.append(pkg_name) + for pkg_name in os.listdir(self.packages_path): + # Skip non-directories in the package root. + pkg_dir = join_path(self.packages_path, pkg_name) + if not os.path.isdir(pkg_dir): + continue + # Skip directories without a package.py in them. + pkg_file = join_path(self.packages_path, pkg_name, package_file_name) + if not os.path.isfile(pkg_file): + continue + + # Warn about invalid names that look like packages. + if not valid_module_name(pkg_name): + tty.warn("Skipping package at %s. '%s' is not a valid Spack module name." + % (pkg_dir, pkg_name)) + continue + + # All checks passed. Add it to the list. + self._all_package_names.append(pkg_name) self._all_package_names.sort() return self._all_package_names @@ -489,7 +549,8 @@ def exists(self, pkg_name): """Whether a package with the supplied name exists.""" # This does a binary search in the sorted list. idx = bisect_left(self.all_package_names(), pkg_name) - return self._all_package_names[idx] == pkg_name + return (idx < len(self._all_package_names) and + self._all_package_names[idx] == pkg_name) def _get_pkg_module(self, pkg_name): @@ -505,7 +566,7 @@ def _get_pkg_module(self, pkg_name): file_path = self.filename_for_package_name(pkg_name) if not os.path.exists(file_path): - raise UnknownPackageError(pkg_name, self.namespace) + raise UnknownPackageError(pkg_name, self) if not os.path.isfile(file_path): tty.die("Something's wrong. '%s' is not a file!" % file_path) @@ -513,10 +574,11 @@ def _get_pkg_module(self, pkg_name): if not os.access(file_path, os.R_OK): tty.die("Cannot read '%s'!" % file_path) - fullname = "%s.%s" % (self.namespace, pkg_name) + # e.g., spack.pkg.builtin.mpich + fullname = "%s.%s" % (self.full_namespace, pkg_name) module = imp.load_source(fullname, file_path) - module.__package__ = self.namespace + module.__package__ = self.full_namespace module.__loader__ = self self._modules[pkg_name] = module @@ -541,7 +603,7 @@ def _get_pkg_class(self, pkg_name): def __str__(self): - return "" % (self.namespace, self.root) + return "[Repo '%s' at '%s']" % (self.namespace, self.root) def __repr__(self): @@ -597,12 +659,18 @@ def installed_known_package_specs(self): yield spec +class BadRepoError(spack.error.SpackError): + """Raised when repo layout is invalid.""" + def __init__(self, msg): + super(BadRepoError, self).__init__(msg) + + class UnknownPackageError(spack.error.SpackError): """Raised when we encounter a package spack doesn't have.""" def __init__(self, name, repo=None): msg = None if repo: - msg = "Package %s not found in packagerepo %s." % (name, repo) + msg = "Package %s not found in repository %s." % (name, repo) else: msg = "Package %s not found." % name super(UnknownPackageError, self).__init__(msg) diff --git a/var/spack/mock_packages/_repo.yaml b/var/spack/mock_packages/_repo.yaml deleted file mode 100644 index b97b978de3a..00000000000 --- a/var/spack/mock_packages/_repo.yaml +++ /dev/null @@ -1,2 +0,0 @@ -repo: - namespace: gov.llnl.spack.mock diff --git a/var/spack/packages/_repo.yaml b/var/spack/packages/_repo.yaml deleted file mode 100644 index 4a371e1cad4..00000000000 --- a/var/spack/packages/_repo.yaml +++ /dev/null @@ -1,2 +0,0 @@ -repo: - namespace: gov.llnl.spack diff --git a/var/spack/mock_packages/a/package.py b/var/spack/repos/builtin.mock/packages/a/package.py similarity index 100% rename from var/spack/mock_packages/a/package.py rename to var/spack/repos/builtin.mock/packages/a/package.py diff --git a/var/spack/mock_packages/b/package.py b/var/spack/repos/builtin.mock/packages/b/package.py similarity index 100% rename from var/spack/mock_packages/b/package.py rename to var/spack/repos/builtin.mock/packages/b/package.py diff --git a/var/spack/mock_packages/c/package.py b/var/spack/repos/builtin.mock/packages/c/package.py similarity index 100% rename from var/spack/mock_packages/c/package.py rename to var/spack/repos/builtin.mock/packages/c/package.py diff --git a/var/spack/mock_packages/callpath/package.py b/var/spack/repos/builtin.mock/packages/callpath/package.py similarity index 100% rename from var/spack/mock_packages/callpath/package.py rename to var/spack/repos/builtin.mock/packages/callpath/package.py diff --git a/var/spack/mock_packages/direct_mpich/package.py b/var/spack/repos/builtin.mock/packages/direct_mpich/package.py similarity index 100% rename from var/spack/mock_packages/direct_mpich/package.py rename to var/spack/repos/builtin.mock/packages/direct_mpich/package.py diff --git a/var/spack/mock_packages/dyninst/package.py b/var/spack/repos/builtin.mock/packages/dyninst/package.py similarity index 100% rename from var/spack/mock_packages/dyninst/package.py rename to var/spack/repos/builtin.mock/packages/dyninst/package.py diff --git a/var/spack/mock_packages/e/package.py b/var/spack/repos/builtin.mock/packages/e/package.py similarity index 100% rename from var/spack/mock_packages/e/package.py rename to var/spack/repos/builtin.mock/packages/e/package.py diff --git a/var/spack/mock_packages/fake/package.py b/var/spack/repos/builtin.mock/packages/fake/package.py similarity index 100% rename from var/spack/mock_packages/fake/package.py rename to var/spack/repos/builtin.mock/packages/fake/package.py diff --git a/var/spack/mock_packages/git-test/package.py b/var/spack/repos/builtin.mock/packages/git-test/package.py similarity index 100% rename from var/spack/mock_packages/git-test/package.py rename to var/spack/repos/builtin.mock/packages/git-test/package.py diff --git a/var/spack/mock_packages/hg-test/package.py b/var/spack/repos/builtin.mock/packages/hg-test/package.py similarity index 100% rename from var/spack/mock_packages/hg-test/package.py rename to var/spack/repos/builtin.mock/packages/hg-test/package.py diff --git a/var/spack/mock_packages/indirect_mpich/package.py b/var/spack/repos/builtin.mock/packages/indirect_mpich/package.py similarity index 100% rename from var/spack/mock_packages/indirect_mpich/package.py rename to var/spack/repos/builtin.mock/packages/indirect_mpich/package.py diff --git a/var/spack/mock_packages/libdwarf/package.py b/var/spack/repos/builtin.mock/packages/libdwarf/package.py similarity index 100% rename from var/spack/mock_packages/libdwarf/package.py rename to var/spack/repos/builtin.mock/packages/libdwarf/package.py diff --git a/var/spack/mock_packages/libelf/package.py b/var/spack/repos/builtin.mock/packages/libelf/package.py similarity index 100% rename from var/spack/mock_packages/libelf/package.py rename to var/spack/repos/builtin.mock/packages/libelf/package.py diff --git a/var/spack/mock_packages/mpich/package.py b/var/spack/repos/builtin.mock/packages/mpich/package.py similarity index 100% rename from var/spack/mock_packages/mpich/package.py rename to var/spack/repos/builtin.mock/packages/mpich/package.py diff --git a/var/spack/mock_packages/mpich2/package.py b/var/spack/repos/builtin.mock/packages/mpich2/package.py similarity index 100% rename from var/spack/mock_packages/mpich2/package.py rename to var/spack/repos/builtin.mock/packages/mpich2/package.py diff --git a/var/spack/mock_packages/mpileaks/package.py b/var/spack/repos/builtin.mock/packages/mpileaks/package.py similarity index 100% rename from var/spack/mock_packages/mpileaks/package.py rename to var/spack/repos/builtin.mock/packages/mpileaks/package.py diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/repos/builtin.mock/packages/multimethod/package.py similarity index 100% rename from var/spack/mock_packages/multimethod/package.py rename to var/spack/repos/builtin.mock/packages/multimethod/package.py diff --git a/var/spack/mock_packages/optional-dep-test-2/package.py b/var/spack/repos/builtin.mock/packages/optional-dep-test-2/package.py similarity index 100% rename from var/spack/mock_packages/optional-dep-test-2/package.py rename to var/spack/repos/builtin.mock/packages/optional-dep-test-2/package.py diff --git a/var/spack/mock_packages/optional-dep-test-3/package.py b/var/spack/repos/builtin.mock/packages/optional-dep-test-3/package.py similarity index 100% rename from var/spack/mock_packages/optional-dep-test-3/package.py rename to var/spack/repos/builtin.mock/packages/optional-dep-test-3/package.py diff --git a/var/spack/mock_packages/optional-dep-test/package.py b/var/spack/repos/builtin.mock/packages/optional-dep-test/package.py similarity index 100% rename from var/spack/mock_packages/optional-dep-test/package.py rename to var/spack/repos/builtin.mock/packages/optional-dep-test/package.py diff --git a/var/spack/mock_packages/svn-test/package.py b/var/spack/repos/builtin.mock/packages/svn-test/package.py similarity index 100% rename from var/spack/mock_packages/svn-test/package.py rename to var/spack/repos/builtin.mock/packages/svn-test/package.py diff --git a/var/spack/mock_packages/trivial_install_test_package/package.py b/var/spack/repos/builtin.mock/packages/trivial_install_test_package/package.py similarity index 100% rename from var/spack/mock_packages/trivial_install_test_package/package.py rename to var/spack/repos/builtin.mock/packages/trivial_install_test_package/package.py diff --git a/var/spack/mock_packages/zmpi/package.py b/var/spack/repos/builtin.mock/packages/zmpi/package.py similarity index 100% rename from var/spack/mock_packages/zmpi/package.py rename to var/spack/repos/builtin.mock/packages/zmpi/package.py diff --git a/var/spack/repos/builtin.mock/repo.yaml b/var/spack/repos/builtin.mock/repo.yaml new file mode 100644 index 00000000000..30b068da13d --- /dev/null +++ b/var/spack/repos/builtin.mock/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: builtin.mock diff --git a/var/spack/packages/ImageMagick/package.py b/var/spack/repos/builtin/packages/ImageMagick/package.py similarity index 100% rename from var/spack/packages/ImageMagick/package.py rename to var/spack/repos/builtin/packages/ImageMagick/package.py diff --git a/var/spack/packages/Mitos/package.py b/var/spack/repos/builtin/packages/Mitos/package.py similarity index 100% rename from var/spack/packages/Mitos/package.py rename to var/spack/repos/builtin/packages/Mitos/package.py diff --git a/var/spack/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py similarity index 100% rename from var/spack/packages/R/package.py rename to var/spack/repos/builtin/packages/R/package.py diff --git a/var/spack/packages/SAMRAI/no-tool-build.patch b/var/spack/repos/builtin/packages/SAMRAI/no-tool-build.patch similarity index 100% rename from var/spack/packages/SAMRAI/no-tool-build.patch rename to var/spack/repos/builtin/packages/SAMRAI/no-tool-build.patch diff --git a/var/spack/packages/SAMRAI/package.py b/var/spack/repos/builtin/packages/SAMRAI/package.py similarity index 100% rename from var/spack/packages/SAMRAI/package.py rename to var/spack/repos/builtin/packages/SAMRAI/package.py diff --git a/var/spack/packages/activeharmony/package.py b/var/spack/repos/builtin/packages/activeharmony/package.py similarity index 100% rename from var/spack/packages/activeharmony/package.py rename to var/spack/repos/builtin/packages/activeharmony/package.py diff --git a/var/spack/packages/adept-utils/package.py b/var/spack/repos/builtin/packages/adept-utils/package.py similarity index 100% rename from var/spack/packages/adept-utils/package.py rename to var/spack/repos/builtin/packages/adept-utils/package.py diff --git a/var/spack/packages/apex/package.py b/var/spack/repos/builtin/packages/apex/package.py similarity index 100% rename from var/spack/packages/apex/package.py rename to var/spack/repos/builtin/packages/apex/package.py diff --git a/var/spack/packages/arpack/package.py b/var/spack/repos/builtin/packages/arpack/package.py similarity index 100% rename from var/spack/packages/arpack/package.py rename to var/spack/repos/builtin/packages/arpack/package.py diff --git a/var/spack/packages/asciidoc/package.py b/var/spack/repos/builtin/packages/asciidoc/package.py similarity index 100% rename from var/spack/packages/asciidoc/package.py rename to var/spack/repos/builtin/packages/asciidoc/package.py diff --git a/var/spack/packages/atk/package.py b/var/spack/repos/builtin/packages/atk/package.py similarity index 100% rename from var/spack/packages/atk/package.py rename to var/spack/repos/builtin/packages/atk/package.py diff --git a/var/spack/packages/atlas/package.py b/var/spack/repos/builtin/packages/atlas/package.py similarity index 100% rename from var/spack/packages/atlas/package.py rename to var/spack/repos/builtin/packages/atlas/package.py diff --git a/var/spack/packages/autoconf/package.py b/var/spack/repos/builtin/packages/autoconf/package.py similarity index 100% rename from var/spack/packages/autoconf/package.py rename to var/spack/repos/builtin/packages/autoconf/package.py diff --git a/var/spack/packages/automaded/package.py b/var/spack/repos/builtin/packages/automaded/package.py similarity index 100% rename from var/spack/packages/automaded/package.py rename to var/spack/repos/builtin/packages/automaded/package.py diff --git a/var/spack/packages/automake/package.py b/var/spack/repos/builtin/packages/automake/package.py similarity index 100% rename from var/spack/packages/automake/package.py rename to var/spack/repos/builtin/packages/automake/package.py diff --git a/var/spack/packages/bear/package.py b/var/spack/repos/builtin/packages/bear/package.py similarity index 100% rename from var/spack/packages/bear/package.py rename to var/spack/repos/builtin/packages/bear/package.py diff --git a/var/spack/packages/bib2xhtml/package.py b/var/spack/repos/builtin/packages/bib2xhtml/package.py similarity index 100% rename from var/spack/packages/bib2xhtml/package.py rename to var/spack/repos/builtin/packages/bib2xhtml/package.py diff --git a/var/spack/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py similarity index 100% rename from var/spack/packages/binutils/package.py rename to var/spack/repos/builtin/packages/binutils/package.py diff --git a/var/spack/packages/bison/package.py b/var/spack/repos/builtin/packages/bison/package.py similarity index 100% rename from var/spack/packages/bison/package.py rename to var/spack/repos/builtin/packages/bison/package.py diff --git a/var/spack/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py similarity index 100% rename from var/spack/packages/boost/package.py rename to var/spack/repos/builtin/packages/boost/package.py diff --git a/var/spack/packages/bowtie2/bowtie2-2.5.patch b/var/spack/repos/builtin/packages/bowtie2/bowtie2-2.5.patch similarity index 100% rename from var/spack/packages/bowtie2/bowtie2-2.5.patch rename to var/spack/repos/builtin/packages/bowtie2/bowtie2-2.5.patch diff --git a/var/spack/packages/bowtie2/package.py b/var/spack/repos/builtin/packages/bowtie2/package.py similarity index 100% rename from var/spack/packages/bowtie2/package.py rename to var/spack/repos/builtin/packages/bowtie2/package.py diff --git a/var/spack/packages/boxlib/package.py b/var/spack/repos/builtin/packages/boxlib/package.py similarity index 100% rename from var/spack/packages/boxlib/package.py rename to var/spack/repos/builtin/packages/boxlib/package.py diff --git a/var/spack/packages/bzip2/package.py b/var/spack/repos/builtin/packages/bzip2/package.py similarity index 100% rename from var/spack/packages/bzip2/package.py rename to var/spack/repos/builtin/packages/bzip2/package.py diff --git a/var/spack/packages/cairo/package.py b/var/spack/repos/builtin/packages/cairo/package.py similarity index 100% rename from var/spack/packages/cairo/package.py rename to var/spack/repos/builtin/packages/cairo/package.py diff --git a/var/spack/packages/callpath/package.py b/var/spack/repos/builtin/packages/callpath/package.py similarity index 100% rename from var/spack/packages/callpath/package.py rename to var/spack/repos/builtin/packages/callpath/package.py diff --git a/var/spack/packages/cblas/package.py b/var/spack/repos/builtin/packages/cblas/package.py similarity index 100% rename from var/spack/packages/cblas/package.py rename to var/spack/repos/builtin/packages/cblas/package.py diff --git a/var/spack/packages/cgm/package.py b/var/spack/repos/builtin/packages/cgm/package.py similarity index 100% rename from var/spack/packages/cgm/package.py rename to var/spack/repos/builtin/packages/cgm/package.py diff --git a/var/spack/packages/clang/package.py b/var/spack/repos/builtin/packages/clang/package.py similarity index 100% rename from var/spack/packages/clang/package.py rename to var/spack/repos/builtin/packages/clang/package.py diff --git a/var/spack/packages/cloog/package.py b/var/spack/repos/builtin/packages/cloog/package.py similarity index 100% rename from var/spack/packages/cloog/package.py rename to var/spack/repos/builtin/packages/cloog/package.py diff --git a/var/spack/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py similarity index 100% rename from var/spack/packages/cmake/package.py rename to var/spack/repos/builtin/packages/cmake/package.py diff --git a/var/spack/packages/coreutils/package.py b/var/spack/repos/builtin/packages/coreutils/package.py similarity index 100% rename from var/spack/packages/coreutils/package.py rename to var/spack/repos/builtin/packages/coreutils/package.py diff --git a/var/spack/packages/cppcheck/package.py b/var/spack/repos/builtin/packages/cppcheck/package.py similarity index 100% rename from var/spack/packages/cppcheck/package.py rename to var/spack/repos/builtin/packages/cppcheck/package.py diff --git a/var/spack/packages/cram/package.py b/var/spack/repos/builtin/packages/cram/package.py similarity index 100% rename from var/spack/packages/cram/package.py rename to var/spack/repos/builtin/packages/cram/package.py diff --git a/var/spack/packages/cscope/package.py b/var/spack/repos/builtin/packages/cscope/package.py similarity index 100% rename from var/spack/packages/cscope/package.py rename to var/spack/repos/builtin/packages/cscope/package.py diff --git a/var/spack/packages/cube/package.py b/var/spack/repos/builtin/packages/cube/package.py similarity index 100% rename from var/spack/packages/cube/package.py rename to var/spack/repos/builtin/packages/cube/package.py diff --git a/var/spack/packages/czmq/package.py b/var/spack/repos/builtin/packages/czmq/package.py similarity index 100% rename from var/spack/packages/czmq/package.py rename to var/spack/repos/builtin/packages/czmq/package.py diff --git a/var/spack/packages/dbus/package.py b/var/spack/repos/builtin/packages/dbus/package.py similarity index 100% rename from var/spack/packages/dbus/package.py rename to var/spack/repos/builtin/packages/dbus/package.py diff --git a/var/spack/packages/docbook-xml/package.py b/var/spack/repos/builtin/packages/docbook-xml/package.py similarity index 100% rename from var/spack/packages/docbook-xml/package.py rename to var/spack/repos/builtin/packages/docbook-xml/package.py diff --git a/var/spack/packages/doxygen/package.py b/var/spack/repos/builtin/packages/doxygen/package.py similarity index 100% rename from var/spack/packages/doxygen/package.py rename to var/spack/repos/builtin/packages/doxygen/package.py diff --git a/var/spack/packages/dri2proto/package.py b/var/spack/repos/builtin/packages/dri2proto/package.py similarity index 100% rename from var/spack/packages/dri2proto/package.py rename to var/spack/repos/builtin/packages/dri2proto/package.py diff --git a/var/spack/packages/dtcmp/package.py b/var/spack/repos/builtin/packages/dtcmp/package.py similarity index 100% rename from var/spack/packages/dtcmp/package.py rename to var/spack/repos/builtin/packages/dtcmp/package.py diff --git a/var/spack/packages/dyninst/package.py b/var/spack/repos/builtin/packages/dyninst/package.py similarity index 100% rename from var/spack/packages/dyninst/package.py rename to var/spack/repos/builtin/packages/dyninst/package.py diff --git a/var/spack/packages/elfutils/package.py b/var/spack/repos/builtin/packages/elfutils/package.py similarity index 100% rename from var/spack/packages/elfutils/package.py rename to var/spack/repos/builtin/packages/elfutils/package.py diff --git a/var/spack/packages/extrae/package.py b/var/spack/repos/builtin/packages/extrae/package.py similarity index 100% rename from var/spack/packages/extrae/package.py rename to var/spack/repos/builtin/packages/extrae/package.py diff --git a/var/spack/packages/exuberant-ctags/package.py b/var/spack/repos/builtin/packages/exuberant-ctags/package.py similarity index 100% rename from var/spack/packages/exuberant-ctags/package.py rename to var/spack/repos/builtin/packages/exuberant-ctags/package.py diff --git a/var/spack/packages/fish/package.py b/var/spack/repos/builtin/packages/fish/package.py similarity index 100% rename from var/spack/packages/fish/package.py rename to var/spack/repos/builtin/packages/fish/package.py diff --git a/var/spack/packages/flex/package.py b/var/spack/repos/builtin/packages/flex/package.py similarity index 100% rename from var/spack/packages/flex/package.py rename to var/spack/repos/builtin/packages/flex/package.py diff --git a/var/spack/packages/flux/package.py b/var/spack/repos/builtin/packages/flux/package.py similarity index 100% rename from var/spack/packages/flux/package.py rename to var/spack/repos/builtin/packages/flux/package.py diff --git a/var/spack/packages/fontconfig/package.py b/var/spack/repos/builtin/packages/fontconfig/package.py similarity index 100% rename from var/spack/packages/fontconfig/package.py rename to var/spack/repos/builtin/packages/fontconfig/package.py diff --git a/var/spack/packages/freetype/package.py b/var/spack/repos/builtin/packages/freetype/package.py similarity index 100% rename from var/spack/packages/freetype/package.py rename to var/spack/repos/builtin/packages/freetype/package.py diff --git a/var/spack/packages/gasnet/package.py b/var/spack/repos/builtin/packages/gasnet/package.py similarity index 100% rename from var/spack/packages/gasnet/package.py rename to var/spack/repos/builtin/packages/gasnet/package.py diff --git a/var/spack/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py similarity index 100% rename from var/spack/packages/gcc/package.py rename to var/spack/repos/builtin/packages/gcc/package.py diff --git a/var/spack/packages/gdk-pixbuf/package.py b/var/spack/repos/builtin/packages/gdk-pixbuf/package.py similarity index 100% rename from var/spack/packages/gdk-pixbuf/package.py rename to var/spack/repos/builtin/packages/gdk-pixbuf/package.py diff --git a/var/spack/packages/geos/package.py b/var/spack/repos/builtin/packages/geos/package.py similarity index 100% rename from var/spack/packages/geos/package.py rename to var/spack/repos/builtin/packages/geos/package.py diff --git a/var/spack/packages/gflags/package.py b/var/spack/repos/builtin/packages/gflags/package.py similarity index 100% rename from var/spack/packages/gflags/package.py rename to var/spack/repos/builtin/packages/gflags/package.py diff --git a/var/spack/packages/ghostscript/package.py b/var/spack/repos/builtin/packages/ghostscript/package.py similarity index 100% rename from var/spack/packages/ghostscript/package.py rename to var/spack/repos/builtin/packages/ghostscript/package.py diff --git a/var/spack/packages/git/package.py b/var/spack/repos/builtin/packages/git/package.py similarity index 100% rename from var/spack/packages/git/package.py rename to var/spack/repos/builtin/packages/git/package.py diff --git a/var/spack/packages/glib/package.py b/var/spack/repos/builtin/packages/glib/package.py similarity index 100% rename from var/spack/packages/glib/package.py rename to var/spack/repos/builtin/packages/glib/package.py diff --git a/var/spack/packages/glm/package.py b/var/spack/repos/builtin/packages/glm/package.py similarity index 100% rename from var/spack/packages/glm/package.py rename to var/spack/repos/builtin/packages/glm/package.py diff --git a/var/spack/packages/global/package.py b/var/spack/repos/builtin/packages/global/package.py similarity index 100% rename from var/spack/packages/global/package.py rename to var/spack/repos/builtin/packages/global/package.py diff --git a/var/spack/packages/glog/package.py b/var/spack/repos/builtin/packages/glog/package.py similarity index 100% rename from var/spack/packages/glog/package.py rename to var/spack/repos/builtin/packages/glog/package.py diff --git a/var/spack/packages/gmp/package.py b/var/spack/repos/builtin/packages/gmp/package.py similarity index 100% rename from var/spack/packages/gmp/package.py rename to var/spack/repos/builtin/packages/gmp/package.py diff --git a/var/spack/packages/gnutls/package.py b/var/spack/repos/builtin/packages/gnutls/package.py similarity index 100% rename from var/spack/packages/gnutls/package.py rename to var/spack/repos/builtin/packages/gnutls/package.py diff --git a/var/spack/packages/gperf/package.py b/var/spack/repos/builtin/packages/gperf/package.py similarity index 100% rename from var/spack/packages/gperf/package.py rename to var/spack/repos/builtin/packages/gperf/package.py diff --git a/var/spack/packages/gperftools/package.py b/var/spack/repos/builtin/packages/gperftools/package.py similarity index 100% rename from var/spack/packages/gperftools/package.py rename to var/spack/repos/builtin/packages/gperftools/package.py diff --git a/var/spack/packages/graphlib/package.py b/var/spack/repos/builtin/packages/graphlib/package.py similarity index 100% rename from var/spack/packages/graphlib/package.py rename to var/spack/repos/builtin/packages/graphlib/package.py diff --git a/var/spack/packages/graphviz/package.py b/var/spack/repos/builtin/packages/graphviz/package.py similarity index 100% rename from var/spack/packages/graphviz/package.py rename to var/spack/repos/builtin/packages/graphviz/package.py diff --git a/var/spack/packages/gtkplus/package.py b/var/spack/repos/builtin/packages/gtkplus/package.py similarity index 100% rename from var/spack/packages/gtkplus/package.py rename to var/spack/repos/builtin/packages/gtkplus/package.py diff --git a/var/spack/packages/harfbuzz/package.py b/var/spack/repos/builtin/packages/harfbuzz/package.py similarity index 100% rename from var/spack/packages/harfbuzz/package.py rename to var/spack/repos/builtin/packages/harfbuzz/package.py diff --git a/var/spack/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py similarity index 100% rename from var/spack/packages/hdf5/package.py rename to var/spack/repos/builtin/packages/hdf5/package.py diff --git a/var/spack/packages/hwloc/package.py b/var/spack/repos/builtin/packages/hwloc/package.py similarity index 100% rename from var/spack/packages/hwloc/package.py rename to var/spack/repos/builtin/packages/hwloc/package.py diff --git a/var/spack/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py similarity index 100% rename from var/spack/packages/hypre/package.py rename to var/spack/repos/builtin/packages/hypre/package.py diff --git a/var/spack/packages/icu/package.py b/var/spack/repos/builtin/packages/icu/package.py similarity index 100% rename from var/spack/packages/icu/package.py rename to var/spack/repos/builtin/packages/icu/package.py diff --git a/var/spack/packages/icu4c/package.py b/var/spack/repos/builtin/packages/icu4c/package.py similarity index 100% rename from var/spack/packages/icu4c/package.py rename to var/spack/repos/builtin/packages/icu4c/package.py diff --git a/var/spack/packages/isl/package.py b/var/spack/repos/builtin/packages/isl/package.py similarity index 100% rename from var/spack/packages/isl/package.py rename to var/spack/repos/builtin/packages/isl/package.py diff --git a/var/spack/packages/jdk/package.py b/var/spack/repos/builtin/packages/jdk/package.py similarity index 100% rename from var/spack/packages/jdk/package.py rename to var/spack/repos/builtin/packages/jdk/package.py diff --git a/var/spack/packages/jpeg/package.py b/var/spack/repos/builtin/packages/jpeg/package.py similarity index 100% rename from var/spack/packages/jpeg/package.py rename to var/spack/repos/builtin/packages/jpeg/package.py diff --git a/var/spack/packages/launchmon/package.py b/var/spack/repos/builtin/packages/launchmon/package.py similarity index 100% rename from var/spack/packages/launchmon/package.py rename to var/spack/repos/builtin/packages/launchmon/package.py diff --git a/var/spack/packages/launchmon/patch.lmon_install_dir b/var/spack/repos/builtin/packages/launchmon/patch.lmon_install_dir similarity index 100% rename from var/spack/packages/launchmon/patch.lmon_install_dir rename to var/spack/repos/builtin/packages/launchmon/patch.lmon_install_dir diff --git a/var/spack/packages/lcms/package.py b/var/spack/repos/builtin/packages/lcms/package.py similarity index 100% rename from var/spack/packages/lcms/package.py rename to var/spack/repos/builtin/packages/lcms/package.py diff --git a/var/spack/packages/leveldb/package.py b/var/spack/repos/builtin/packages/leveldb/package.py similarity index 100% rename from var/spack/packages/leveldb/package.py rename to var/spack/repos/builtin/packages/leveldb/package.py diff --git a/var/spack/packages/libNBC/package.py b/var/spack/repos/builtin/packages/libNBC/package.py similarity index 100% rename from var/spack/packages/libNBC/package.py rename to var/spack/repos/builtin/packages/libNBC/package.py diff --git a/var/spack/packages/libarchive/package.py b/var/spack/repos/builtin/packages/libarchive/package.py similarity index 100% rename from var/spack/packages/libarchive/package.py rename to var/spack/repos/builtin/packages/libarchive/package.py diff --git a/var/spack/packages/libcircle/package.py b/var/spack/repos/builtin/packages/libcircle/package.py similarity index 100% rename from var/spack/packages/libcircle/package.py rename to var/spack/repos/builtin/packages/libcircle/package.py diff --git a/var/spack/packages/libdrm/package.py b/var/spack/repos/builtin/packages/libdrm/package.py similarity index 100% rename from var/spack/packages/libdrm/package.py rename to var/spack/repos/builtin/packages/libdrm/package.py diff --git a/var/spack/packages/libdwarf/package.py b/var/spack/repos/builtin/packages/libdwarf/package.py similarity index 100% rename from var/spack/packages/libdwarf/package.py rename to var/spack/repos/builtin/packages/libdwarf/package.py diff --git a/var/spack/packages/libelf/package.py b/var/spack/repos/builtin/packages/libelf/package.py similarity index 100% rename from var/spack/packages/libelf/package.py rename to var/spack/repos/builtin/packages/libelf/package.py diff --git a/var/spack/packages/libevent/package.py b/var/spack/repos/builtin/packages/libevent/package.py similarity index 100% rename from var/spack/packages/libevent/package.py rename to var/spack/repos/builtin/packages/libevent/package.py diff --git a/var/spack/packages/libffi/package.py b/var/spack/repos/builtin/packages/libffi/package.py similarity index 100% rename from var/spack/packages/libffi/package.py rename to var/spack/repos/builtin/packages/libffi/package.py diff --git a/var/spack/packages/libgcrypt/package.py b/var/spack/repos/builtin/packages/libgcrypt/package.py similarity index 100% rename from var/spack/packages/libgcrypt/package.py rename to var/spack/repos/builtin/packages/libgcrypt/package.py diff --git a/var/spack/packages/libgpg-error/package.py b/var/spack/repos/builtin/packages/libgpg-error/package.py similarity index 100% rename from var/spack/packages/libgpg-error/package.py rename to var/spack/repos/builtin/packages/libgpg-error/package.py diff --git a/var/spack/packages/libjpeg-turbo/package.py b/var/spack/repos/builtin/packages/libjpeg-turbo/package.py similarity index 100% rename from var/spack/packages/libjpeg-turbo/package.py rename to var/spack/repos/builtin/packages/libjpeg-turbo/package.py diff --git a/var/spack/packages/libjson-c/package.py b/var/spack/repos/builtin/packages/libjson-c/package.py similarity index 100% rename from var/spack/packages/libjson-c/package.py rename to var/spack/repos/builtin/packages/libjson-c/package.py diff --git a/var/spack/packages/libmng/package.py b/var/spack/repos/builtin/packages/libmng/package.py similarity index 100% rename from var/spack/packages/libmng/package.py rename to var/spack/repos/builtin/packages/libmng/package.py diff --git a/var/spack/packages/libmonitor/package.py b/var/spack/repos/builtin/packages/libmonitor/package.py similarity index 100% rename from var/spack/packages/libmonitor/package.py rename to var/spack/repos/builtin/packages/libmonitor/package.py diff --git a/var/spack/packages/libpciaccess/package.py b/var/spack/repos/builtin/packages/libpciaccess/package.py similarity index 100% rename from var/spack/packages/libpciaccess/package.py rename to var/spack/repos/builtin/packages/libpciaccess/package.py diff --git a/var/spack/packages/libpng/package.py b/var/spack/repos/builtin/packages/libpng/package.py similarity index 100% rename from var/spack/packages/libpng/package.py rename to var/spack/repos/builtin/packages/libpng/package.py diff --git a/var/spack/packages/libsodium/package.py b/var/spack/repos/builtin/packages/libsodium/package.py similarity index 100% rename from var/spack/packages/libsodium/package.py rename to var/spack/repos/builtin/packages/libsodium/package.py diff --git a/var/spack/packages/libtiff/package.py b/var/spack/repos/builtin/packages/libtiff/package.py similarity index 100% rename from var/spack/packages/libtiff/package.py rename to var/spack/repos/builtin/packages/libtiff/package.py diff --git a/var/spack/packages/libtool/package.py b/var/spack/repos/builtin/packages/libtool/package.py similarity index 100% rename from var/spack/packages/libtool/package.py rename to var/spack/repos/builtin/packages/libtool/package.py diff --git a/var/spack/packages/libunwind/package.py b/var/spack/repos/builtin/packages/libunwind/package.py similarity index 100% rename from var/spack/packages/libunwind/package.py rename to var/spack/repos/builtin/packages/libunwind/package.py diff --git a/var/spack/packages/libuuid/package.py b/var/spack/repos/builtin/packages/libuuid/package.py similarity index 100% rename from var/spack/packages/libuuid/package.py rename to var/spack/repos/builtin/packages/libuuid/package.py diff --git a/var/spack/packages/libxcb/package.py b/var/spack/repos/builtin/packages/libxcb/package.py similarity index 100% rename from var/spack/packages/libxcb/package.py rename to var/spack/repos/builtin/packages/libxcb/package.py diff --git a/var/spack/packages/libxml2/package.py b/var/spack/repos/builtin/packages/libxml2/package.py similarity index 100% rename from var/spack/packages/libxml2/package.py rename to var/spack/repos/builtin/packages/libxml2/package.py diff --git a/var/spack/packages/libxshmfence/package.py b/var/spack/repos/builtin/packages/libxshmfence/package.py similarity index 100% rename from var/spack/packages/libxshmfence/package.py rename to var/spack/repos/builtin/packages/libxshmfence/package.py diff --git a/var/spack/packages/libxslt/package.py b/var/spack/repos/builtin/packages/libxslt/package.py similarity index 100% rename from var/spack/packages/libxslt/package.py rename to var/spack/repos/builtin/packages/libxslt/package.py diff --git a/var/spack/packages/llvm-lld/package.py b/var/spack/repos/builtin/packages/llvm-lld/package.py similarity index 100% rename from var/spack/packages/llvm-lld/package.py rename to var/spack/repos/builtin/packages/llvm-lld/package.py diff --git a/var/spack/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py similarity index 100% rename from var/spack/packages/llvm/package.py rename to var/spack/repos/builtin/packages/llvm/package.py diff --git a/var/spack/packages/lmdb/package.py b/var/spack/repos/builtin/packages/lmdb/package.py similarity index 100% rename from var/spack/packages/lmdb/package.py rename to var/spack/repos/builtin/packages/lmdb/package.py diff --git a/var/spack/packages/lua/package.py b/var/spack/repos/builtin/packages/lua/package.py similarity index 100% rename from var/spack/packages/lua/package.py rename to var/spack/repos/builtin/packages/lua/package.py diff --git a/var/spack/packages/lwgrp/package.py b/var/spack/repos/builtin/packages/lwgrp/package.py similarity index 100% rename from var/spack/packages/lwgrp/package.py rename to var/spack/repos/builtin/packages/lwgrp/package.py diff --git a/var/spack/packages/lwm2/package.py b/var/spack/repos/builtin/packages/lwm2/package.py similarity index 100% rename from var/spack/packages/lwm2/package.py rename to var/spack/repos/builtin/packages/lwm2/package.py diff --git a/var/spack/packages/matio/package.py b/var/spack/repos/builtin/packages/matio/package.py similarity index 100% rename from var/spack/packages/matio/package.py rename to var/spack/repos/builtin/packages/matio/package.py diff --git a/var/spack/packages/memaxes/package.py b/var/spack/repos/builtin/packages/memaxes/package.py similarity index 100% rename from var/spack/packages/memaxes/package.py rename to var/spack/repos/builtin/packages/memaxes/package.py diff --git a/var/spack/packages/mesa/package.py b/var/spack/repos/builtin/packages/mesa/package.py similarity index 100% rename from var/spack/packages/mesa/package.py rename to var/spack/repos/builtin/packages/mesa/package.py diff --git a/var/spack/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py similarity index 100% rename from var/spack/packages/metis/package.py rename to var/spack/repos/builtin/packages/metis/package.py diff --git a/var/spack/packages/mpc/package.py b/var/spack/repos/builtin/packages/mpc/package.py similarity index 100% rename from var/spack/packages/mpc/package.py rename to var/spack/repos/builtin/packages/mpc/package.py diff --git a/var/spack/packages/mpe2/mpe2.patch b/var/spack/repos/builtin/packages/mpe2/mpe2.patch similarity index 100% rename from var/spack/packages/mpe2/mpe2.patch rename to var/spack/repos/builtin/packages/mpe2/mpe2.patch diff --git a/var/spack/packages/mpe2/package.py b/var/spack/repos/builtin/packages/mpe2/package.py similarity index 100% rename from var/spack/packages/mpe2/package.py rename to var/spack/repos/builtin/packages/mpe2/package.py diff --git a/var/spack/packages/mpfr/package.py b/var/spack/repos/builtin/packages/mpfr/package.py similarity index 100% rename from var/spack/packages/mpfr/package.py rename to var/spack/repos/builtin/packages/mpfr/package.py diff --git a/var/spack/packages/mpibash/mpibash-4.3.patch b/var/spack/repos/builtin/packages/mpibash/mpibash-4.3.patch similarity index 100% rename from var/spack/packages/mpibash/mpibash-4.3.patch rename to var/spack/repos/builtin/packages/mpibash/mpibash-4.3.patch diff --git a/var/spack/packages/mpibash/package.py b/var/spack/repos/builtin/packages/mpibash/package.py similarity index 100% rename from var/spack/packages/mpibash/package.py rename to var/spack/repos/builtin/packages/mpibash/package.py diff --git a/var/spack/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py similarity index 100% rename from var/spack/packages/mpich/package.py rename to var/spack/repos/builtin/packages/mpich/package.py diff --git a/var/spack/packages/mpileaks/package.py b/var/spack/repos/builtin/packages/mpileaks/package.py similarity index 100% rename from var/spack/packages/mpileaks/package.py rename to var/spack/repos/builtin/packages/mpileaks/package.py diff --git a/var/spack/packages/mrnet/package.py b/var/spack/repos/builtin/packages/mrnet/package.py similarity index 100% rename from var/spack/packages/mrnet/package.py rename to var/spack/repos/builtin/packages/mrnet/package.py diff --git a/var/spack/packages/munge/package.py b/var/spack/repos/builtin/packages/munge/package.py similarity index 100% rename from var/spack/packages/munge/package.py rename to var/spack/repos/builtin/packages/munge/package.py diff --git a/var/spack/packages/muster/package.py b/var/spack/repos/builtin/packages/muster/package.py similarity index 100% rename from var/spack/packages/muster/package.py rename to var/spack/repos/builtin/packages/muster/package.py diff --git a/var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch b/var/spack/repos/builtin/packages/mvapich2/ad_lustre_rwcontig_open_source.patch similarity index 100% rename from var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch rename to var/spack/repos/builtin/packages/mvapich2/ad_lustre_rwcontig_open_source.patch diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py similarity index 100% rename from var/spack/packages/mvapich2/package.py rename to var/spack/repos/builtin/packages/mvapich2/package.py diff --git a/var/spack/packages/nasm/package.py b/var/spack/repos/builtin/packages/nasm/package.py similarity index 100% rename from var/spack/packages/nasm/package.py rename to var/spack/repos/builtin/packages/nasm/package.py diff --git a/var/spack/packages/ncdu/package.py b/var/spack/repos/builtin/packages/ncdu/package.py similarity index 100% rename from var/spack/packages/ncdu/package.py rename to var/spack/repos/builtin/packages/ncdu/package.py diff --git a/var/spack/packages/ncurses/package.py b/var/spack/repos/builtin/packages/ncurses/package.py similarity index 100% rename from var/spack/packages/ncurses/package.py rename to var/spack/repos/builtin/packages/ncurses/package.py diff --git a/var/spack/packages/netcdf/netcdf-4.3.3-mpi.patch b/var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch similarity index 100% rename from var/spack/packages/netcdf/netcdf-4.3.3-mpi.patch rename to var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch diff --git a/var/spack/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py similarity index 100% rename from var/spack/packages/netcdf/package.py rename to var/spack/repos/builtin/packages/netcdf/package.py diff --git a/var/spack/packages/netgauge/package.py b/var/spack/repos/builtin/packages/netgauge/package.py similarity index 100% rename from var/spack/packages/netgauge/package.py rename to var/spack/repos/builtin/packages/netgauge/package.py diff --git a/var/spack/packages/netlib-blas/package.py b/var/spack/repos/builtin/packages/netlib-blas/package.py similarity index 100% rename from var/spack/packages/netlib-blas/package.py rename to var/spack/repos/builtin/packages/netlib-blas/package.py diff --git a/var/spack/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py similarity index 100% rename from var/spack/packages/netlib-lapack/package.py rename to var/spack/repos/builtin/packages/netlib-lapack/package.py diff --git a/var/spack/packages/nettle/package.py b/var/spack/repos/builtin/packages/nettle/package.py similarity index 100% rename from var/spack/packages/nettle/package.py rename to var/spack/repos/builtin/packages/nettle/package.py diff --git a/var/spack/packages/ompss/package.py b/var/spack/repos/builtin/packages/ompss/package.py similarity index 100% rename from var/spack/packages/ompss/package.py rename to var/spack/repos/builtin/packages/ompss/package.py diff --git a/var/spack/packages/ompt-openmp/package.py b/var/spack/repos/builtin/packages/ompt-openmp/package.py similarity index 100% rename from var/spack/packages/ompt-openmp/package.py rename to var/spack/repos/builtin/packages/ompt-openmp/package.py diff --git a/var/spack/packages/opari2/package.py b/var/spack/repos/builtin/packages/opari2/package.py similarity index 100% rename from var/spack/packages/opari2/package.py rename to var/spack/repos/builtin/packages/opari2/package.py diff --git a/var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch b/var/spack/repos/builtin/packages/openmpi/ad_lustre_rwcontig_open_source.patch similarity index 100% rename from var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch rename to var/spack/repos/builtin/packages/openmpi/ad_lustre_rwcontig_open_source.patch diff --git a/var/spack/packages/openmpi/llnl-platforms.patch b/var/spack/repos/builtin/packages/openmpi/llnl-platforms.patch similarity index 100% rename from var/spack/packages/openmpi/llnl-platforms.patch rename to var/spack/repos/builtin/packages/openmpi/llnl-platforms.patch diff --git a/var/spack/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py similarity index 100% rename from var/spack/packages/openmpi/package.py rename to var/spack/repos/builtin/packages/openmpi/package.py diff --git a/var/spack/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py similarity index 100% rename from var/spack/packages/openssl/package.py rename to var/spack/repos/builtin/packages/openssl/package.py diff --git a/var/spack/packages/otf/package.py b/var/spack/repos/builtin/packages/otf/package.py similarity index 100% rename from var/spack/packages/otf/package.py rename to var/spack/repos/builtin/packages/otf/package.py diff --git a/var/spack/packages/otf2/package.py b/var/spack/repos/builtin/packages/otf2/package.py similarity index 100% rename from var/spack/packages/otf2/package.py rename to var/spack/repos/builtin/packages/otf2/package.py diff --git a/var/spack/packages/pango/package.py b/var/spack/repos/builtin/packages/pango/package.py similarity index 100% rename from var/spack/packages/pango/package.py rename to var/spack/repos/builtin/packages/pango/package.py diff --git a/var/spack/packages/papi/package.py b/var/spack/repos/builtin/packages/papi/package.py similarity index 100% rename from var/spack/packages/papi/package.py rename to var/spack/repos/builtin/packages/papi/package.py diff --git a/var/spack/packages/paraver/package.py b/var/spack/repos/builtin/packages/paraver/package.py similarity index 100% rename from var/spack/packages/paraver/package.py rename to var/spack/repos/builtin/packages/paraver/package.py diff --git a/var/spack/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py similarity index 100% rename from var/spack/packages/paraview/package.py rename to var/spack/repos/builtin/packages/paraview/package.py diff --git a/var/spack/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py similarity index 100% rename from var/spack/packages/parmetis/package.py rename to var/spack/repos/builtin/packages/parmetis/package.py diff --git a/var/spack/packages/parpack/package.py b/var/spack/repos/builtin/packages/parpack/package.py similarity index 100% rename from var/spack/packages/parpack/package.py rename to var/spack/repos/builtin/packages/parpack/package.py diff --git a/var/spack/packages/pcre/package.py b/var/spack/repos/builtin/packages/pcre/package.py similarity index 100% rename from var/spack/packages/pcre/package.py rename to var/spack/repos/builtin/packages/pcre/package.py diff --git a/var/spack/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py similarity index 100% rename from var/spack/packages/petsc/package.py rename to var/spack/repos/builtin/packages/petsc/package.py diff --git a/var/spack/packages/pidx/package.py b/var/spack/repos/builtin/packages/pidx/package.py similarity index 100% rename from var/spack/packages/pidx/package.py rename to var/spack/repos/builtin/packages/pidx/package.py diff --git a/var/spack/packages/pixman/package.py b/var/spack/repos/builtin/packages/pixman/package.py similarity index 100% rename from var/spack/packages/pixman/package.py rename to var/spack/repos/builtin/packages/pixman/package.py diff --git a/var/spack/packages/pkg-config/package.py b/var/spack/repos/builtin/packages/pkg-config/package.py similarity index 100% rename from var/spack/packages/pkg-config/package.py rename to var/spack/repos/builtin/packages/pkg-config/package.py diff --git a/var/spack/packages/pmgr_collective/package.py b/var/spack/repos/builtin/packages/pmgr_collective/package.py similarity index 100% rename from var/spack/packages/pmgr_collective/package.py rename to var/spack/repos/builtin/packages/pmgr_collective/package.py diff --git a/var/spack/packages/postgresql/package.py b/var/spack/repos/builtin/packages/postgresql/package.py similarity index 100% rename from var/spack/packages/postgresql/package.py rename to var/spack/repos/builtin/packages/postgresql/package.py diff --git a/var/spack/packages/ppl/package.py b/var/spack/repos/builtin/packages/ppl/package.py similarity index 100% rename from var/spack/packages/ppl/package.py rename to var/spack/repos/builtin/packages/ppl/package.py diff --git a/var/spack/packages/protobuf/package.py b/var/spack/repos/builtin/packages/protobuf/package.py similarity index 100% rename from var/spack/packages/protobuf/package.py rename to var/spack/repos/builtin/packages/protobuf/package.py diff --git a/var/spack/packages/py-basemap/package.py b/var/spack/repos/builtin/packages/py-basemap/package.py similarity index 100% rename from var/spack/packages/py-basemap/package.py rename to var/spack/repos/builtin/packages/py-basemap/package.py diff --git a/var/spack/packages/py-biopython/package.py b/var/spack/repos/builtin/packages/py-biopython/package.py similarity index 100% rename from var/spack/packages/py-biopython/package.py rename to var/spack/repos/builtin/packages/py-biopython/package.py diff --git a/var/spack/packages/py-cffi/package.py b/var/spack/repos/builtin/packages/py-cffi/package.py similarity index 100% rename from var/spack/packages/py-cffi/package.py rename to var/spack/repos/builtin/packages/py-cffi/package.py diff --git a/var/spack/packages/py-cython/package.py b/var/spack/repos/builtin/packages/py-cython/package.py similarity index 100% rename from var/spack/packages/py-cython/package.py rename to var/spack/repos/builtin/packages/py-cython/package.py diff --git a/var/spack/packages/py-dateutil/package.py b/var/spack/repos/builtin/packages/py-dateutil/package.py similarity index 100% rename from var/spack/packages/py-dateutil/package.py rename to var/spack/repos/builtin/packages/py-dateutil/package.py diff --git a/var/spack/packages/py-epydoc/package.py b/var/spack/repos/builtin/packages/py-epydoc/package.py similarity index 100% rename from var/spack/packages/py-epydoc/package.py rename to var/spack/repos/builtin/packages/py-epydoc/package.py diff --git a/var/spack/packages/py-genders/package.py b/var/spack/repos/builtin/packages/py-genders/package.py similarity index 100% rename from var/spack/packages/py-genders/package.py rename to var/spack/repos/builtin/packages/py-genders/package.py diff --git a/var/spack/packages/py-gnuplot/package.py b/var/spack/repos/builtin/packages/py-gnuplot/package.py similarity index 100% rename from var/spack/packages/py-gnuplot/package.py rename to var/spack/repos/builtin/packages/py-gnuplot/package.py diff --git a/var/spack/packages/py-h5py/package.py b/var/spack/repos/builtin/packages/py-h5py/package.py similarity index 100% rename from var/spack/packages/py-h5py/package.py rename to var/spack/repos/builtin/packages/py-h5py/package.py diff --git a/var/spack/packages/py-ipython/package.py b/var/spack/repos/builtin/packages/py-ipython/package.py similarity index 100% rename from var/spack/packages/py-ipython/package.py rename to var/spack/repos/builtin/packages/py-ipython/package.py diff --git a/var/spack/packages/py-libxml2/package.py b/var/spack/repos/builtin/packages/py-libxml2/package.py similarity index 100% rename from var/spack/packages/py-libxml2/package.py rename to var/spack/repos/builtin/packages/py-libxml2/package.py diff --git a/var/spack/packages/py-lockfile/package.py b/var/spack/repos/builtin/packages/py-lockfile/package.py similarity index 100% rename from var/spack/packages/py-lockfile/package.py rename to var/spack/repos/builtin/packages/py-lockfile/package.py diff --git a/var/spack/packages/py-mako/package.py b/var/spack/repos/builtin/packages/py-mako/package.py similarity index 100% rename from var/spack/packages/py-mako/package.py rename to var/spack/repos/builtin/packages/py-mako/package.py diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/repos/builtin/packages/py-matplotlib/package.py similarity index 100% rename from var/spack/packages/py-matplotlib/package.py rename to var/spack/repos/builtin/packages/py-matplotlib/package.py diff --git a/var/spack/packages/py-mock/package.py b/var/spack/repos/builtin/packages/py-mock/package.py similarity index 100% rename from var/spack/packages/py-mock/package.py rename to var/spack/repos/builtin/packages/py-mock/package.py diff --git a/var/spack/packages/py-mpi4py/package.py b/var/spack/repos/builtin/packages/py-mpi4py/package.py similarity index 100% rename from var/spack/packages/py-mpi4py/package.py rename to var/spack/repos/builtin/packages/py-mpi4py/package.py diff --git a/var/spack/packages/py-mx/package.py b/var/spack/repos/builtin/packages/py-mx/package.py similarity index 100% rename from var/spack/packages/py-mx/package.py rename to var/spack/repos/builtin/packages/py-mx/package.py diff --git a/var/spack/packages/py-nose/package.py b/var/spack/repos/builtin/packages/py-nose/package.py similarity index 100% rename from var/spack/packages/py-nose/package.py rename to var/spack/repos/builtin/packages/py-nose/package.py diff --git a/var/spack/packages/py-numpy/package.py b/var/spack/repos/builtin/packages/py-numpy/package.py similarity index 100% rename from var/spack/packages/py-numpy/package.py rename to var/spack/repos/builtin/packages/py-numpy/package.py diff --git a/var/spack/packages/py-pandas/package.py b/var/spack/repos/builtin/packages/py-pandas/package.py similarity index 100% rename from var/spack/packages/py-pandas/package.py rename to var/spack/repos/builtin/packages/py-pandas/package.py diff --git a/var/spack/packages/py-pexpect/package.py b/var/spack/repos/builtin/packages/py-pexpect/package.py similarity index 100% rename from var/spack/packages/py-pexpect/package.py rename to var/spack/repos/builtin/packages/py-pexpect/package.py diff --git a/var/spack/packages/py-pil/package.py b/var/spack/repos/builtin/packages/py-pil/package.py similarity index 100% rename from var/spack/packages/py-pil/package.py rename to var/spack/repos/builtin/packages/py-pil/package.py diff --git a/var/spack/packages/py-pmw/package.py b/var/spack/repos/builtin/packages/py-pmw/package.py similarity index 100% rename from var/spack/packages/py-pmw/package.py rename to var/spack/repos/builtin/packages/py-pmw/package.py diff --git a/var/spack/packages/py-pychecker/package.py b/var/spack/repos/builtin/packages/py-pychecker/package.py similarity index 100% rename from var/spack/packages/py-pychecker/package.py rename to var/spack/repos/builtin/packages/py-pychecker/package.py diff --git a/var/spack/packages/py-pycparser/package.py b/var/spack/repos/builtin/packages/py-pycparser/package.py similarity index 100% rename from var/spack/packages/py-pycparser/package.py rename to var/spack/repos/builtin/packages/py-pycparser/package.py diff --git a/var/spack/packages/py-pyelftools/package.py b/var/spack/repos/builtin/packages/py-pyelftools/package.py similarity index 100% rename from var/spack/packages/py-pyelftools/package.py rename to var/spack/repos/builtin/packages/py-pyelftools/package.py diff --git a/var/spack/packages/py-pygments/package.py b/var/spack/repos/builtin/packages/py-pygments/package.py similarity index 100% rename from var/spack/packages/py-pygments/package.py rename to var/spack/repos/builtin/packages/py-pygments/package.py diff --git a/var/spack/packages/py-pylint/package.py b/var/spack/repos/builtin/packages/py-pylint/package.py similarity index 100% rename from var/spack/packages/py-pylint/package.py rename to var/spack/repos/builtin/packages/py-pylint/package.py diff --git a/var/spack/packages/py-pypar/package.py b/var/spack/repos/builtin/packages/py-pypar/package.py similarity index 100% rename from var/spack/packages/py-pypar/package.py rename to var/spack/repos/builtin/packages/py-pypar/package.py diff --git a/var/spack/packages/py-pyparsing/package.py b/var/spack/repos/builtin/packages/py-pyparsing/package.py similarity index 100% rename from var/spack/packages/py-pyparsing/package.py rename to var/spack/repos/builtin/packages/py-pyparsing/package.py diff --git a/var/spack/packages/py-pyqt/package.py b/var/spack/repos/builtin/packages/py-pyqt/package.py similarity index 100% rename from var/spack/packages/py-pyqt/package.py rename to var/spack/repos/builtin/packages/py-pyqt/package.py diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/repos/builtin/packages/py-pyside/package.py similarity index 100% rename from var/spack/packages/py-pyside/package.py rename to var/spack/repos/builtin/packages/py-pyside/package.py diff --git a/var/spack/packages/py-python-daemon/package.py b/var/spack/repos/builtin/packages/py-python-daemon/package.py similarity index 100% rename from var/spack/packages/py-python-daemon/package.py rename to var/spack/repos/builtin/packages/py-python-daemon/package.py diff --git a/var/spack/packages/py-pytz/package.py b/var/spack/repos/builtin/packages/py-pytz/package.py similarity index 100% rename from var/spack/packages/py-pytz/package.py rename to var/spack/repos/builtin/packages/py-pytz/package.py diff --git a/var/spack/packages/py-rpy2/package.py b/var/spack/repos/builtin/packages/py-rpy2/package.py similarity index 100% rename from var/spack/packages/py-rpy2/package.py rename to var/spack/repos/builtin/packages/py-rpy2/package.py diff --git a/var/spack/packages/py-scientificpython/package.py b/var/spack/repos/builtin/packages/py-scientificpython/package.py similarity index 100% rename from var/spack/packages/py-scientificpython/package.py rename to var/spack/repos/builtin/packages/py-scientificpython/package.py diff --git a/var/spack/packages/py-scikit-learn/package.py b/var/spack/repos/builtin/packages/py-scikit-learn/package.py similarity index 100% rename from var/spack/packages/py-scikit-learn/package.py rename to var/spack/repos/builtin/packages/py-scikit-learn/package.py diff --git a/var/spack/packages/py-scipy/package.py b/var/spack/repos/builtin/packages/py-scipy/package.py similarity index 100% rename from var/spack/packages/py-scipy/package.py rename to var/spack/repos/builtin/packages/py-scipy/package.py diff --git a/var/spack/packages/py-setuptools/package.py b/var/spack/repos/builtin/packages/py-setuptools/package.py similarity index 100% rename from var/spack/packages/py-setuptools/package.py rename to var/spack/repos/builtin/packages/py-setuptools/package.py diff --git a/var/spack/packages/py-shiboken/package.py b/var/spack/repos/builtin/packages/py-shiboken/package.py similarity index 100% rename from var/spack/packages/py-shiboken/package.py rename to var/spack/repos/builtin/packages/py-shiboken/package.py diff --git a/var/spack/packages/py-sip/package.py b/var/spack/repos/builtin/packages/py-sip/package.py similarity index 100% rename from var/spack/packages/py-sip/package.py rename to var/spack/repos/builtin/packages/py-sip/package.py diff --git a/var/spack/packages/py-six/package.py b/var/spack/repos/builtin/packages/py-six/package.py similarity index 100% rename from var/spack/packages/py-six/package.py rename to var/spack/repos/builtin/packages/py-six/package.py diff --git a/var/spack/packages/py-sphinx/package.py b/var/spack/repos/builtin/packages/py-sphinx/package.py similarity index 100% rename from var/spack/packages/py-sphinx/package.py rename to var/spack/repos/builtin/packages/py-sphinx/package.py diff --git a/var/spack/packages/py-sympy/package.py b/var/spack/repos/builtin/packages/py-sympy/package.py similarity index 100% rename from var/spack/packages/py-sympy/package.py rename to var/spack/repos/builtin/packages/py-sympy/package.py diff --git a/var/spack/packages/py-virtualenv/package.py b/var/spack/repos/builtin/packages/py-virtualenv/package.py similarity index 100% rename from var/spack/packages/py-virtualenv/package.py rename to var/spack/repos/builtin/packages/py-virtualenv/package.py diff --git a/var/spack/packages/py-yapf/package.py b/var/spack/repos/builtin/packages/py-yapf/package.py similarity index 100% rename from var/spack/packages/py-yapf/package.py rename to var/spack/repos/builtin/packages/py-yapf/package.py diff --git a/var/spack/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py similarity index 100% rename from var/spack/packages/python/package.py rename to var/spack/repos/builtin/packages/python/package.py diff --git a/var/spack/packages/qhull/package.py b/var/spack/repos/builtin/packages/qhull/package.py similarity index 100% rename from var/spack/packages/qhull/package.py rename to var/spack/repos/builtin/packages/qhull/package.py diff --git a/var/spack/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py similarity index 100% rename from var/spack/packages/qt/package.py rename to var/spack/repos/builtin/packages/qt/package.py diff --git a/var/spack/packages/qthreads/package.py b/var/spack/repos/builtin/packages/qthreads/package.py similarity index 100% rename from var/spack/packages/qthreads/package.py rename to var/spack/repos/builtin/packages/qthreads/package.py diff --git a/var/spack/packages/ravel/package.py b/var/spack/repos/builtin/packages/ravel/package.py similarity index 100% rename from var/spack/packages/ravel/package.py rename to var/spack/repos/builtin/packages/ravel/package.py diff --git a/var/spack/packages/readline/package.py b/var/spack/repos/builtin/packages/readline/package.py similarity index 100% rename from var/spack/packages/readline/package.py rename to var/spack/repos/builtin/packages/readline/package.py diff --git a/var/spack/packages/rose/add_spack_compiler_recognition.patch b/var/spack/repos/builtin/packages/rose/add_spack_compiler_recognition.patch similarity index 100% rename from var/spack/packages/rose/add_spack_compiler_recognition.patch rename to var/spack/repos/builtin/packages/rose/add_spack_compiler_recognition.patch diff --git a/var/spack/packages/rose/package.py b/var/spack/repos/builtin/packages/rose/package.py similarity index 100% rename from var/spack/packages/rose/package.py rename to var/spack/repos/builtin/packages/rose/package.py diff --git a/var/spack/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py similarity index 100% rename from var/spack/packages/ruby/package.py rename to var/spack/repos/builtin/packages/ruby/package.py diff --git a/var/spack/packages/samtools/package.py b/var/spack/repos/builtin/packages/samtools/package.py similarity index 100% rename from var/spack/packages/samtools/package.py rename to var/spack/repos/builtin/packages/samtools/package.py diff --git a/var/spack/packages/samtools/samtools1.2.patch b/var/spack/repos/builtin/packages/samtools/samtools1.2.patch similarity index 100% rename from var/spack/packages/samtools/samtools1.2.patch rename to var/spack/repos/builtin/packages/samtools/samtools1.2.patch diff --git a/var/spack/packages/scalasca/package.py b/var/spack/repos/builtin/packages/scalasca/package.py similarity index 100% rename from var/spack/packages/scalasca/package.py rename to var/spack/repos/builtin/packages/scalasca/package.py diff --git a/var/spack/packages/scorep/package.py b/var/spack/repos/builtin/packages/scorep/package.py similarity index 100% rename from var/spack/packages/scorep/package.py rename to var/spack/repos/builtin/packages/scorep/package.py diff --git a/var/spack/packages/scotch/package.py b/var/spack/repos/builtin/packages/scotch/package.py similarity index 100% rename from var/spack/packages/scotch/package.py rename to var/spack/repos/builtin/packages/scotch/package.py diff --git a/var/spack/packages/scr/package.py b/var/spack/repos/builtin/packages/scr/package.py similarity index 100% rename from var/spack/packages/scr/package.py rename to var/spack/repos/builtin/packages/scr/package.py diff --git a/var/spack/packages/silo/package.py b/var/spack/repos/builtin/packages/silo/package.py similarity index 100% rename from var/spack/packages/silo/package.py rename to var/spack/repos/builtin/packages/silo/package.py diff --git a/var/spack/packages/snappy/package.py b/var/spack/repos/builtin/packages/snappy/package.py similarity index 100% rename from var/spack/packages/snappy/package.py rename to var/spack/repos/builtin/packages/snappy/package.py diff --git a/var/spack/packages/spindle/package.py b/var/spack/repos/builtin/packages/spindle/package.py similarity index 100% rename from var/spack/packages/spindle/package.py rename to var/spack/repos/builtin/packages/spindle/package.py diff --git a/var/spack/packages/sqlite/package.py b/var/spack/repos/builtin/packages/sqlite/package.py similarity index 100% rename from var/spack/packages/sqlite/package.py rename to var/spack/repos/builtin/packages/sqlite/package.py diff --git a/var/spack/packages/stat/configure_mpicxx.patch b/var/spack/repos/builtin/packages/stat/configure_mpicxx.patch similarity index 100% rename from var/spack/packages/stat/configure_mpicxx.patch rename to var/spack/repos/builtin/packages/stat/configure_mpicxx.patch diff --git a/var/spack/packages/stat/package.py b/var/spack/repos/builtin/packages/stat/package.py similarity index 100% rename from var/spack/packages/stat/package.py rename to var/spack/repos/builtin/packages/stat/package.py diff --git a/var/spack/packages/sundials/package.py b/var/spack/repos/builtin/packages/sundials/package.py similarity index 100% rename from var/spack/packages/sundials/package.py rename to var/spack/repos/builtin/packages/sundials/package.py diff --git a/var/spack/packages/swig/package.py b/var/spack/repos/builtin/packages/swig/package.py similarity index 100% rename from var/spack/packages/swig/package.py rename to var/spack/repos/builtin/packages/swig/package.py diff --git a/var/spack/packages/task/package.py b/var/spack/repos/builtin/packages/task/package.py similarity index 100% rename from var/spack/packages/task/package.py rename to var/spack/repos/builtin/packages/task/package.py diff --git a/var/spack/packages/taskd/package.py b/var/spack/repos/builtin/packages/taskd/package.py similarity index 100% rename from var/spack/packages/taskd/package.py rename to var/spack/repos/builtin/packages/taskd/package.py diff --git a/var/spack/packages/tau/package.py b/var/spack/repos/builtin/packages/tau/package.py similarity index 100% rename from var/spack/packages/tau/package.py rename to var/spack/repos/builtin/packages/tau/package.py diff --git a/var/spack/packages/tcl/package.py b/var/spack/repos/builtin/packages/tcl/package.py similarity index 100% rename from var/spack/packages/tcl/package.py rename to var/spack/repos/builtin/packages/tcl/package.py diff --git a/var/spack/packages/the_silver_searcher/package.py b/var/spack/repos/builtin/packages/the_silver_searcher/package.py similarity index 100% rename from var/spack/packages/the_silver_searcher/package.py rename to var/spack/repos/builtin/packages/the_silver_searcher/package.py diff --git a/var/spack/packages/thrift/package.py b/var/spack/repos/builtin/packages/thrift/package.py similarity index 100% rename from var/spack/packages/thrift/package.py rename to var/spack/repos/builtin/packages/thrift/package.py diff --git a/var/spack/packages/tk/package.py b/var/spack/repos/builtin/packages/tk/package.py similarity index 100% rename from var/spack/packages/tk/package.py rename to var/spack/repos/builtin/packages/tk/package.py diff --git a/var/spack/packages/tmux/package.py b/var/spack/repos/builtin/packages/tmux/package.py similarity index 100% rename from var/spack/packages/tmux/package.py rename to var/spack/repos/builtin/packages/tmux/package.py diff --git a/var/spack/packages/tmuxinator/package.py b/var/spack/repos/builtin/packages/tmuxinator/package.py similarity index 100% rename from var/spack/packages/tmuxinator/package.py rename to var/spack/repos/builtin/packages/tmuxinator/package.py diff --git a/var/spack/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py similarity index 100% rename from var/spack/packages/trilinos/package.py rename to var/spack/repos/builtin/packages/trilinos/package.py diff --git a/var/spack/packages/uncrustify/package.py b/var/spack/repos/builtin/packages/uncrustify/package.py similarity index 100% rename from var/spack/packages/uncrustify/package.py rename to var/spack/repos/builtin/packages/uncrustify/package.py diff --git a/var/spack/packages/util-linux/package.py b/var/spack/repos/builtin/packages/util-linux/package.py similarity index 100% rename from var/spack/packages/util-linux/package.py rename to var/spack/repos/builtin/packages/util-linux/package.py diff --git a/var/spack/packages/vim/package.py b/var/spack/repos/builtin/packages/vim/package.py similarity index 100% rename from var/spack/packages/vim/package.py rename to var/spack/repos/builtin/packages/vim/package.py diff --git a/var/spack/packages/vtk/package.py b/var/spack/repos/builtin/packages/vtk/package.py similarity index 100% rename from var/spack/packages/vtk/package.py rename to var/spack/repos/builtin/packages/vtk/package.py diff --git a/var/spack/packages/wget/package.py b/var/spack/repos/builtin/packages/wget/package.py similarity index 100% rename from var/spack/packages/wget/package.py rename to var/spack/repos/builtin/packages/wget/package.py diff --git a/var/spack/packages/wx/package.py b/var/spack/repos/builtin/packages/wx/package.py similarity index 100% rename from var/spack/packages/wx/package.py rename to var/spack/repos/builtin/packages/wx/package.py diff --git a/var/spack/packages/wxpropgrid/package.py b/var/spack/repos/builtin/packages/wxpropgrid/package.py similarity index 100% rename from var/spack/packages/wxpropgrid/package.py rename to var/spack/repos/builtin/packages/wxpropgrid/package.py diff --git a/var/spack/packages/xcb-proto/package.py b/var/spack/repos/builtin/packages/xcb-proto/package.py similarity index 100% rename from var/spack/packages/xcb-proto/package.py rename to var/spack/repos/builtin/packages/xcb-proto/package.py diff --git a/var/spack/packages/xz/package.py b/var/spack/repos/builtin/packages/xz/package.py similarity index 100% rename from var/spack/packages/xz/package.py rename to var/spack/repos/builtin/packages/xz/package.py diff --git a/var/spack/packages/yasm/package.py b/var/spack/repos/builtin/packages/yasm/package.py similarity index 100% rename from var/spack/packages/yasm/package.py rename to var/spack/repos/builtin/packages/yasm/package.py diff --git a/var/spack/packages/zeromq/package.py b/var/spack/repos/builtin/packages/zeromq/package.py similarity index 100% rename from var/spack/packages/zeromq/package.py rename to var/spack/repos/builtin/packages/zeromq/package.py diff --git a/var/spack/packages/zlib/package.py b/var/spack/repos/builtin/packages/zlib/package.py similarity index 100% rename from var/spack/packages/zlib/package.py rename to var/spack/repos/builtin/packages/zlib/package.py diff --git a/var/spack/packages/zsh/package.py b/var/spack/repos/builtin/packages/zsh/package.py similarity index 100% rename from var/spack/packages/zsh/package.py rename to var/spack/repos/builtin/packages/zsh/package.py diff --git a/var/spack/repos/builtin/repo.yaml b/var/spack/repos/builtin/repo.yaml new file mode 100644 index 00000000000..54b282db6be --- /dev/null +++ b/var/spack/repos/builtin/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: builtin From 763d850a0e1766a0b915686504bc07f8f2a92058 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 26 Nov 2015 14:23:12 -0800 Subject: [PATCH 103/910] Add tests for importing packages directly. Added test for, e.g.: import spack.pkg.builtin.mock.mpich import spack.pkg.builtin.mock.mpich as mpich from spack.pkg.builtin.mock.mpich import Mpich Among others. These ensure that direct package imports work so that packages can be extended. --- lib/spack/spack/test/packages.py | 42 ++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index e39def2ff2e..f023516eba2 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -46,7 +46,8 @@ def test_package_name(self): def test_package_filename(self): repo = Repo(spack.mock_packages_path) filename = repo.filename_for_package_name('mpich') - self.assertEqual(filename, join_path(spack.mock_packages_path, 'mpich', 'package.py')) + self.assertEqual(filename, + join_path(spack.mock_packages_path, 'packages', 'mpich', 'package.py')) def test_package_name(self): @@ -57,7 +58,9 @@ def test_package_name(self): def test_nonexisting_package_filename(self): repo = Repo(spack.mock_packages_path) filename = repo.filename_for_package_name('some-nonexisting-package') - self.assertEqual(filename, join_path(spack.mock_packages_path, 'some-nonexisting-package', 'package.py')) + self.assertEqual( + filename, + join_path(spack.mock_packages_path, 'packages', 'some-nonexisting-package', 'package.py')) def test_package_class_names(self): @@ -66,3 +69,38 @@ def test_package_class_names(self): self.assertEqual('PmgrCollective', mod_to_class('pmgr-collective')) self.assertEqual('Pmgrcollective', mod_to_class('PmgrCollective')) self.assertEqual('_3db', mod_to_class('3db')) + + + # + # Below tests target direct imports of spack packages from the + # spack.pkg namespace + # + + def test_import_package(self): + import spack.pkg.builtin.mock.mpich + + + def test_import_package_as(self): + import spack.pkg.builtin.mock.mpich as mp + + + def test_import_class_from_package(self): + from spack.pkg.builtin.mock.mpich import Mpich + + + def test_import_module_from_package(self): + from spack.pkg.builtin.mock import mpich + + + def test_import_namespace_container_modules(self): + import spack.pkg + import spack.pkg as p + from spack import pkg + + import spack.pkg.builtin + import spack.pkg.builtin as b + from spack.pkg import builtin + + import spack.pkg.builtin.mock + import spack.pkg.builtin.mock as m + from spack.pkg.builtin import mock From 7383bd393eb36b67f308b643d1ce02b53ebb255d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 27 Nov 2015 23:06:18 -0800 Subject: [PATCH 104/910] Fixed bug #42: problem with satisfies() for virtual dependencies. - _cross_provider_maps() had suffered some bit rot (map returned was ill-formed but still worked for cases with one vdep) - ProviderIndex.satisfies() was only checking whether the result map was non-empty. It should check whether all common vdeps are *in* the result map, as that indicates there is *some* way to satisfy *all* of them. We were checking whether there was some way to satisfy *any one* of them, which is wrong. - Above would cause a problem when there is more than one vdep provider. - Added test that covers this case. - Added `constrained()` method to Spec. Analogous to `normalized()`: `constrain():constrained() :: normalize():normalized()` --- lib/spack/spack/spec.py | 7 ++++ lib/spack/spack/test/spec_semantics.py | 13 ++++++- lib/spack/spack/virtual.py | 9 +++-- .../mock_packages/netlib-blas/package.py | 36 ++++++++++++++++++ .../mock_packages/netlib-lapack/package.py | 37 +++++++++++++++++++ var/spack/mock_packages/openblas/package.py | 37 +++++++++++++++++++ 6 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 var/spack/mock_packages/netlib-blas/package.py create mode 100644 var/spack/mock_packages/netlib-lapack/package.py create mode 100644 var/spack/mock_packages/openblas/package.py diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 92880e9cbf8..a10edcc6fce 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1207,6 +1207,13 @@ def common_dependencies(self, other): return common + def constrained(self, other, deps=True): + """Return a constrained copy without modifying this spec.""" + clone = self.copy(deps=deps) + clone.constrain(other, deps) + return clone + + def dep_difference(self, other): """Returns dependencies in self that are not in other.""" mine = set(s.name for s in self.traverse(root=False)) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 6666dbbb52c..64220e58933 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -190,11 +190,23 @@ def test_unsatisfiable_variant_mismatch(self): def test_satisfies_virtual(self): + # Don't use check_satisfies: it checks constrain() too, and + # you can't constrain a non-virtual by a virtual. self.assertTrue(Spec('mpich').satisfies(Spec('mpi'))) self.assertTrue(Spec('mpich2').satisfies(Spec('mpi'))) self.assertTrue(Spec('zmpi').satisfies(Spec('mpi'))) + def test_satisfies_virtual_dep_with_virtual_constraint(self): + """Ensure we can satisfy virtual constraints when there are multiple + vdep providers in the specs.""" + self.assertTrue(Spec('netlib-lapack ^openblas').satisfies('netlib-lapack ^openblas')) + self.assertFalse(Spec('netlib-lapack ^netlib-blas').satisfies('netlib-lapack ^openblas')) + + self.assertFalse(Spec('netlib-lapack ^openblas').satisfies('netlib-lapack ^netlib-blas')) + self.assertTrue(Spec('netlib-lapack ^netlib-blas').satisfies('netlib-lapack ^netlib-blas')) + + # ================================================================================ # Indexing specs # ================================================================================ @@ -327,4 +339,3 @@ def test_constrain_dependency_not_changed(self): self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') self.check_constrain_not_changed('libelf^foo=bgqos_0', 'libelf^foo=bgqos_0') - diff --git a/lib/spack/spack/virtual.py b/lib/spack/spack/virtual.py index c77b259d611..f92cc4509c5 100644 --- a/lib/spack/spack/virtual.py +++ b/lib/spack/spack/virtual.py @@ -117,12 +117,13 @@ def providers_for(self, *vpkg_specs): return sorted(providers) - # TODO: this is pretty darned nasty, and inefficient. + # TODO: this is pretty darned nasty, and inefficient, but there + # are not that many vdeps in most specs. def _cross_provider_maps(self, lmap, rmap): result = {} for lspec, rspec in itertools.product(lmap, rmap): try: - constrained = lspec.copy().constrain(rspec) + constrained = lspec.constrained(rspec) except spack.spec.UnsatisfiableSpecError: continue @@ -130,7 +131,7 @@ def _cross_provider_maps(self, lmap, rmap): for lp_spec, rp_spec in itertools.product(lmap[lspec], rmap[rspec]): if lp_spec.name == rp_spec.name: try: - const = lp_spec.copy().constrain(rp_spec,deps=False) + const = lp_spec.constrained(rp_spec, deps=False) result.setdefault(constrained, set()).add(const) except spack.spec.UnsatisfiableSpecError: continue @@ -157,4 +158,4 @@ def satisfies(self, other): if crossed: result[name] = crossed - return bool(result) + return all(c in result for c in common) diff --git a/var/spack/mock_packages/netlib-blas/package.py b/var/spack/mock_packages/netlib-blas/package.py new file mode 100644 index 00000000000..199327812e1 --- /dev/null +++ b/var/spack/mock_packages/netlib-blas/package.py @@ -0,0 +1,36 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class NetlibBlas(Package): + homepage = "http://www.netlib.org/lapack/" + url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" + + version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') + + provides('blas') + + def install(self, spec, prefix): + pass diff --git a/var/spack/mock_packages/netlib-lapack/package.py b/var/spack/mock_packages/netlib-lapack/package.py new file mode 100644 index 00000000000..8f7f236f1b4 --- /dev/null +++ b/var/spack/mock_packages/netlib-lapack/package.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class NetlibLapack(Package): + homepage = "http://www.netlib.org/lapack/" + url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" + + version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') + + provides('lapack') + depends_on('blas') + + def install(self, spec, prefix): + pass diff --git a/var/spack/mock_packages/openblas/package.py b/var/spack/mock_packages/openblas/package.py new file mode 100644 index 00000000000..9c2fb305735 --- /dev/null +++ b/var/spack/mock_packages/openblas/package.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Openblas(Package): + """OpenBLAS: An optimized BLAS library""" + homepage = "http://www.openblas.net" + url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + + version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') + + provides('blas') + + def install(self, spec, prefix): + pass From a338e0efd5464a40d2e206ed00af99e5dcc53a79 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 28 Nov 2015 16:21:31 -0800 Subject: [PATCH 105/910] Fix spack.db -> spack.repo in bin/spack --- bin/spack | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/spack b/bin/spack index efa1ccc281b..18fd1f0f34b 100755 --- a/bin/spack +++ b/bin/spack @@ -113,8 +113,8 @@ def main(): spack.spack_working_dir = working_dir if args.mock: - from spack.packages import PackageFinder - spack.db.swap(PackageFinder(spack.mock_packages_path)) + from spack.repository import RepoPath + spack.repo.swap(RepoPath(spack.mock_packages_path)) # If the user asked for it, don't check ssl certs. if args.insecure: From 73ef06018e2a6542d90f0b65a42e4a0287525b7a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 28 Nov 2015 16:26:23 -0800 Subject: [PATCH 106/910] Integrate namespace attribute into spec, spec DAG, spec YAML. --- lib/spack/spack/config.py | 22 ++++++--- lib/spack/spack/database.py | 2 + lib/spack/spack/directory_layout.py | 9 ++-- lib/spack/spack/repository.py | 29 ++++++++---- lib/spack/spack/spec.py | 65 ++++++++++++++++++++------ lib/spack/spack/test/spec_semantics.py | 33 +++++++++++-- 6 files changed, 125 insertions(+), 35 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 66da91f6290..36bf8a7fc33 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -115,7 +115,7 @@ def __init__(self, name, filename, merge, strip): self.result_dict = {} _config_sections[name] = self -_ConfigCategory('config', 'config.yaml', True, False) +_ConfigCategory('repos', 'repos.yaml', True, True) _ConfigCategory('compilers', 'compilers.yaml', True, True) _ConfigCategory('mirrors', 'mirrors.yaml', True, True) _ConfigCategory('view', 'views.yaml', True, True) @@ -212,7 +212,7 @@ def substitute_spack_prefix(path): return path.replace('$spack', spack.prefix) -def get_config(category='config'): +def get_config(category): """Get the confguration tree for a category. Strips off the top-level category entry from the dict @@ -233,6 +233,10 @@ def get_config(category='config'): continue result = result[category.name] + # ignore empty sections for easy commenting of single-line configs. + if result is None: + continue + category.files_read_from.insert(0, path) if category.merge: category.result_dict = _merge_yaml(category.result_dict, result) @@ -266,12 +270,18 @@ def get_compilers_config(arch=None): def get_repos_config(): - config = get_config() - if 'repos' not in config: + repo_list = get_config('repos') + if repo_list is None: return [] - repo_list = config['repos'] - return [substitute_spack_prefix(repo) for repo in repo_list] + if not isinstance(repo_list, list): + tty.die("Bad repository configuration. 'repos' element does not contain a list.") + + def expand_repo_path(path): + path = substitute_spack_prefix(path) + path = os.path.expanduser(path) + return path + return [expand_repo_path(repo) for repo in repo_list] def get_mirror_config(): diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 8e380083f3c..5b3bd7502f1 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -211,6 +211,8 @@ def _read_spec_from_yaml(self, hash_key, installs, parent_key=None): child = self._read_spec_from_yaml(dep_hash, installs, hash_key) spec._add_dependency(child) + spec._normal = True + spec._concrete = True return spec diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index da8f4187cc5..30e2c939504 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -211,9 +211,12 @@ def read_spec(self, path): with open(path) as f: spec = Spec.from_yaml(f) - # Specs read from actual installations are always concrete - spec._normal = True - spec._concrete = True + # Specs read from actual installs are always concrete, so mark + # all parts of the spec. + for s in spec.traverse(): + s._normal = True + s._concrete = True + return spec diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index a2c0bbe1478..1b09d63cdda 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -238,7 +238,16 @@ def get(self, spec, new=False): Raises UnknownPackageError if not found. """ - return self.repo_for_pkg(spec.name).get(spec) + # if the spec has a fully qualified namespace, we grab it + # directly and ignore overlay precedence. + if spec.namespace: + fullspace = '%s.%s' % (self.super_namespace, spec.namespace) + if not fullspace in self.by_namespace: + raise UnknownPackageError( + "No configured repository contains package %s." % spec.fullname) + return self.by_namespace[fullspace].get(spec) + else: + return self.repo_for_pkg(spec.name).get(spec) def dirname_for_package_name(self, pkg_name): @@ -454,20 +463,24 @@ def get(self, spec, new=False): if spec.virtual: raise UnknownPackageError(spec.name) - if new and spec in self._instances: - del self._instances[spec] + if spec.namespace and spec.namespace != self.namespace: + raise UnknownPackageError("Repository %s does not contain package %s." + % (self.namespace, spec.fullname)) - if not spec in self._instances: + if new or spec not in self._instances: PackageClass = self._get_pkg_class(spec.name) try: - copy = spec.copy() - self._instances[copy] = PackageClass(copy) + package = PackageClass(spec.copy()) + self._instances[spec] = package + return package + except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) - raise FailedConstructorError(spec.name, *sys.exc_info()) + raise FailedConstructorError(spec.fullname, *sys.exc_info()) - return self._instances[spec] + else: + return self._instances[spec] def purge(self): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index bb0f194c139..303df6df381 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -465,6 +465,13 @@ def _add_dependency(self, spec): self.dependencies[spec.name] = spec spec.dependents[self.name] = self + # + # Public interface + # + @property + def fullname(self): + return '%s.%s' % (self.namespace, self.name) if self.namespace else self.name + @property def root(self): @@ -518,6 +525,7 @@ def concrete(self): return True self._concrete = bool(not self.virtual + and self.namespace is not None and self.versions.concrete and self.variants.concrete and self.architecture @@ -658,6 +666,12 @@ def to_node_dict(self): 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } + + # Older concrete specs do not have a namespace. Omit for + # consistent hashing. + if not self.concrete or self.namespace: + d['namespace'] = self.namespace + if self.compiler: d.update(self.compiler.to_dict()) else: @@ -682,6 +696,7 @@ def from_node_dict(node): node = node[name] spec = Spec(name) + spec.namespace = node.get('namespace', None) spec.versions = VersionList.from_dict(node) spec.architecture = node['arch'] @@ -834,7 +849,20 @@ def concretize(self): changed = any(changes) force=True - self._concrete = True + for s in self.traverse(): + # After concretizing, assign namespaces to anything left. + # Note that this doesn't count as a "change". The repository + # configuration is constant throughout a spack run, and + # normalize and concretize evaluate Packages using Repo.get(), + # which respects precedence. So, a namespace assignment isn't + # changing how a package name would have been interpreted and + # we can do it as late as possible to allow as much + # compatibility across repositories as possible. + if s.namespace is None: + s.namespace = spack.repo.repo_for_pkg(s.name).namespace + + # Mark everything in the spec as concrete, as well. + s._concrete = True def concretized(self): @@ -909,7 +937,7 @@ def _evaluate_dependency_conditions(self, name): the dependency. If no conditions are True (and we don't depend on it), return None. """ - pkg = spack.repo.get(self.name) + pkg = spack.repo.get(self.fullname) conditions = pkg.dependencies[name] # evaluate when specs to figure out constraints on the dependency. @@ -1037,7 +1065,7 @@ def _normalize_helper(self, visited, spec_deps, provider_index): any_change = False changed = True - pkg = spack.repo.get(self.name) + pkg = spack.repo.get(self.fullname) while changed: changed = False for dep_name in pkg.dependencies: @@ -1058,18 +1086,17 @@ def normalize(self, force=False): the root, and ONLY the ones that were explicitly provided are there. Normalization turns a partial flat spec into a DAG, where: - 1. ALL dependencies of the root package are in the DAG. - 2. Each node's dependencies dict only contains its direct deps. + 1. Known dependencies of the root package are in the DAG. + 2. Each node's dependencies dict only contains its known direct deps. 3. There is only ONE unique spec for each package in the DAG. * This includes virtual packages. If there a non-virtual package that provides a virtual package that is in the spec, then we replace the virtual package with the non-virtual one. - 4. The spec DAG matches package DAG, including default variant values. - TODO: normalize should probably implement some form of cycle detection, to ensure that the spec is actually a DAG. + """ if self._normal and not force: return False @@ -1115,7 +1142,7 @@ def validate_names(self): for spec in self.traverse(): # Don't get a package for a virtual name. if not spec.virtual: - spack.repo.get(spec.name) + spack.repo.get(spec.fullname) # validate compiler in addition to the package name. if spec.compiler: @@ -1138,6 +1165,10 @@ def constrain(self, other, deps=True): if not self.name == other.name: raise UnsatisfiableSpecNameError(self.name, other.name) + if other.namespace is not None: + if self.namespace is not None and other.namespace != self.namespace: + raise UnsatisfiableSpecNameError(self.fullname, other.fullname) + if not self.versions.overlaps(other.versions): raise UnsatisfiableVersionSpecError(self.versions, other.versions) @@ -1181,7 +1212,7 @@ def _constrain_dependencies(self, other): # TODO: might want more detail than this, e.g. specific deps # in violation. if this becomes a priority get rid of this - # check and be more specici about what's wrong. + # check and be more specific about what's wrong. if not other.satisfies_dependencies(self): raise UnsatisfiableDependencySpecError(other, self) @@ -1247,7 +1278,7 @@ def satisfies(self, other, deps=True, strict=False): # A concrete provider can satisfy a virtual dependency. if not self.virtual and other.virtual: - pkg = spack.repo.get(self.name) + pkg = spack.repo.get(self.fullname) if pkg.provides(other.name): for provided, when_spec in pkg.provided.items(): if self.satisfies(when_spec, deps=False, strict=strict): @@ -1259,6 +1290,11 @@ def satisfies(self, other, deps=True, strict=False): if self.name != other.name: return False + # namespaces either match, or other doesn't require one. + if other.namespace is not None: + if self.namespace is not None and self.namespace != other.namespace: + return False + if self.versions and other.versions: if not self.versions.satisfies(other.versions, strict=strict): return False @@ -1476,8 +1512,8 @@ def ne_dag(self, other): def _cmp_node(self): """Comparison key for just *this node* and not its deps.""" - return (self.name, self.versions, self.variants, - self.architecture, self.compiler) + return (self.name, self.namespace, self.versions, + self.variants, self.architecture, self.compiler) def eq_node(self, other): @@ -1507,7 +1543,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): in the format string. The format strings you can provide are:: $_ Package name - $. Long package name + $. Full package name (with namespace) $@ Version $% Compiler $%@ Compiler & compiler version @@ -1556,8 +1592,7 @@ def write(s, c): if c == '_': out.write(fmt % self.name) elif c == '.': - longname = '%s.%s.%s' % (self.namespace, self.name) if self.namespace else self.name - out.write(fmt % longname) + out.write(fmt % self.fullname) elif c == '@': if self.versions and self.versions != _any_version: write(fmt % (c + str(self.versions)), c) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 6666dbbb52c..87b7b628edc 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -35,7 +35,10 @@ class SpecSematicsTest(MockPackagesTest): # ================================================================================ def check_satisfies(self, spec, anon_spec, concrete=False): left = Spec(spec, concrete=concrete) - right = parse_anonymous_spec(anon_spec, left.name) + try: + right = Spec(anon_spec) # if it's not anonymous, allow it. + except: + right = parse_anonymous_spec(anon_spec, left.name) # Satisfies is one-directional. self.assertTrue(left.satisfies(right)) @@ -48,7 +51,10 @@ def check_satisfies(self, spec, anon_spec, concrete=False): def check_unsatisfiable(self, spec, anon_spec, concrete=False): left = Spec(spec, concrete=concrete) - right = parse_anonymous_spec(anon_spec, left.name) + try: + right = Spec(anon_spec) # if it's not anonymous, allow it. + except: + right = parse_anonymous_spec(anon_spec, left.name) self.assertFalse(left.satisfies(right)) self.assertFalse(left.satisfies(anon_spec)) @@ -88,6 +94,28 @@ def test_satisfies(self): self.check_satisfies('libdwarf^libelf@0.8.13', '^libelf@0:1') + def test_satisfies_namespace(self): + self.check_satisfies('builtin.mpich', 'mpich') + self.check_satisfies('builtin.mock.mpich', 'mpich') + + # TODO: only works for deps now, but shouldn't we allow this for root spec? + # self.check_satisfies('builtin.mock.mpich', 'mpi') + + self.check_satisfies('builtin.mock.mpich', 'builtin.mock.mpich') + + self.check_unsatisfiable('builtin.mock.mpich', 'builtin.mpich') + + + def test_satisfies_namespaced_dep(self): + """Ensure spec from same or unspecified namespace satisfies namespace constraint.""" + self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpich') + + self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpi') + self.check_satisfies('mpileaks ^builtin.mock.mpich', '^builtin.mock.mpich') + + self.check_unsatisfiable('mpileaks ^builtin.mock.mpich', '^builtin.mpich') + + def test_satisfies_compiler(self): self.check_satisfies('foo%gcc', '%gcc') self.check_satisfies('foo%intel', '%intel') @@ -327,4 +355,3 @@ def test_constrain_dependency_not_changed(self): self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') self.check_constrain_not_changed('libelf^foo=bgqos_0', 'libelf^foo=bgqos_0') - From dfcf567de0cc944f7cdd0bcdee3b841bd0fbb900 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 28 Nov 2015 19:00:24 -0800 Subject: [PATCH 107/910] Add a cleaned up repo command. --- lib/spack/spack/cmd/repo.py | 143 +++++++++++++++++++----------------- 1 file changed, 77 insertions(+), 66 deletions(-) diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 395aa90bedf..8a3ea5989e8 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -22,103 +22,114 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import os +import re +import shutil + from external import argparse import llnl.util.tty as tty -from llnl.util.tty.color import colorize -from llnl.util.tty.colify import colify -from llnl.util.lang import index_by from llnl.util.filesystem import join_path, mkdirp import spack.spec import spack.config from spack.util.environment import get_path -from spack.repository import repo_config_name +from spack.repository import packages_dir_name, repo_config_name, Repo -import os -import exceptions -from contextlib import closing - -description = "Manage package sources" +description = "Manage package source repositories." def setup_parser(subparser): - sp = subparser.add_subparsers( - metavar='SUBCOMMAND', dest='repo_command') - - add_parser = sp.add_parser('add', help=repo_add.__doc__) - add_parser.add_argument('directory', help="Directory containing the packages.") + sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='repo_command') + # Create create_parser = sp.add_parser('create', help=repo_create.__doc__) - create_parser.add_argument('directory', help="Directory containing the packages.") - create_parser.add_argument('name', help="Name of new package repository.") - - remove_parser = sp.add_parser('remove', help=repo_remove.__doc__) - remove_parser.add_argument('name') + create_parser.add_argument( + 'namespace', help="Namespace to identify packages in the repository.") + create_parser.add_argument( + 'directory', help="Directory to create the repo in. Defaults to same as namespace.", nargs='?') + # List list_parser = sp.add_parser('list', help=repo_list.__doc__) -def add_to_config(dir): - config = spack.config.get_config() - user_config = spack.config.get_config('user') - orig = None - if config.has_value('repo', '', 'directories'): - orig = config.get_value('repo', '', 'directories') - if orig and dir in orig.split(':'): - return False +def repo_create(args): + """Create a new package repo for a particular namespace.""" + namespace = args.namespace + if not re.match(r'\w[\.\w-]*', namespace): + tty.die("Invalid namespace: '%s'" % namespace) - newsetting = orig + ':' + dir if orig else dir - user_config.set_value('repo', '', 'directories', newsetting) - user_config.write() - return True + root = args.directory + if not root: + root = namespace + + existed = False + if os.path.exists(root): + if os.path.isfile(root): + tty.die('File %s already exists and is not a directory' % root) + elif os.path.isdir(root): + if not os.access(root, os.R_OK | os.W_OK): + tty.die('Cannot create new repo in %s: cannot access directory.' % root) + if os.listdir(root): + tty.die('Cannot create new repo in %s: directory is not empty.' % root) + existed = True + + full_path = os.path.realpath(root) + parent = os.path.dirname(full_path) + if not os.access(parent, os.R_OK | os.W_OK): + tty.die("Cannot create repository in %s: can't access parent!" % root) + + try: + config_path = os.path.join(root, repo_config_name) + packages_path = os.path.join(root, packages_dir_name) + + mkdirp(packages_path) + with open(config_path, 'w') as config: + config.write("repo:\n") + config.write(" namespace: '%s'\n" % namespace) + + except (IOError, OSError) as e: + tty.die('Failed to create new repository in %s.' % root, + "Caused by %s: %s" % (type(e), e)) + + # try to clean up. + if existed: + shutil.rmtree(config_path, ignore_errors=True) + shutil.rmtree(packages_path, ignore_errors=True) + else: + shutil.rmtree(root, ignore_errors=True) + + tty.msg("Created repo with namespace '%s'." % namespace) + tty.msg("To register it with Spack, add a line like this to ~/.spack/repos.yaml:", + 'repos:', + ' - ' + full_path) def repo_add(args): - """Add package sources to the Spack configuration.""" - if not add_to_config(args.directory): - tty.die('Repo directory %s already exists in the repo list' % dir) - - -def repo_create(args): - """Create a new package repo at a directory and name""" - dir = args.directory - name = args.name - - if os.path.exists(dir) and not os.path.isdir(dir): - tty.die('File %s already exists and is not a directory' % dir) - if not os.path.exists(dir): - try: - mkdirp(dir) - except exceptions.OSError, e: - tty.die('Failed to create new directory %s' % dir) - path = os.path.join(dir, repo_config_filename) - try: - with closing(open(path, 'w')) as repofile: - repofile.write(name + '\n') - except exceptions.IOError, e: - tty.die('Could not create new file %s' % path) - - if not add_to_config(args.directory): - tty.warn('Repo directory %s already exists in the repo list' % dir) + """Remove a package source from the Spack configuration""" + # FIXME: how to deal with this with the current config architecture? + # FIXME: Repos do not have mnemonics, which I assumed would be simpler... should they have them after all? def repo_remove(args): """Remove a package source from the Spack configuration""" - pass + # FIXME: see above. def repo_list(args): """List package sources and their mnemoics""" - root_names = spack.repo.repos - max_len = max(len(s[0]) for s in root_names) - fmt = "%%-%ds%%s" % (max_len + 4) - for root in root_names: - print fmt % (root[0], root[1]) + roots = spack.config.get_repos_config() + repos = [Repo(r) for r in roots] + msg = "%d package repositor" % len(repos) + msg += "y." if len(repos) == 1 else "ies." + tty.msg(msg) + + max_ns_len = max(len(r.namespace) for r in repos) + for repo in repos: + fmt = "%%-%ds%%s" % (max_ns_len + 4) + print fmt % (repo.namespace, repo.root) def repo(parser, args): - action = { 'add' : repo_add, - 'create' : repo_create, - 'remove' : repo_remove, + action = { 'create' : repo_create, 'list' : repo_list } action[args.repo_command](args) From cc349e9a32edcada79ada4b6c3cc71034cff7974 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 28 Nov 2015 19:11:51 -0800 Subject: [PATCH 108/910] Add default repos.yaml --- etc/spack/repos.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 etc/spack/repos.yaml diff --git a/etc/spack/repos.yaml b/etc/spack/repos.yaml new file mode 100644 index 00000000000..2d4ff54ce6d --- /dev/null +++ b/etc/spack/repos.yaml @@ -0,0 +1,8 @@ +# ------------------------------------------------------------------------- +# This is the default spack repository configuration. +# +# Changes to this file will affect all users of this spack install, +# although users can override these settings in their ~/.spack/repos.yaml. +# ------------------------------------------------------------------------- +repos: + - $spack/var/spack/repos/builtin From 05b791a621e49b9611aaccd28163a826582d1983 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 28 Nov 2015 19:25:53 -0800 Subject: [PATCH 109/910] Add citation information to README --- README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3a2c535d4e6..1df1db8bd69 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,14 @@ To install spack and install your first package: Documentation ---------------- -[Full documentation](http://scalability-llnl.github.io/spack) -for Spack is also available. +[**Full documentation**](http://scalability-llnl.github.io/spack) for Spack is +the first place to look. + +See also: + * [Technical paper](http://www.computer.org/csdl/proceedings/sc/2015/3723/00/2807623.pdf) and + [slides](https://tgamblin.github.io/files/Gamblin-Spack-SC15-Talk.pdf) on Spack's design and implementation. + * [Short presentation](https://tgamblin.github.io/files/Gamblin-Spack-Lightning-Talk-BOF-SC15.pdf) from the *Getting Scientific Software Installed* BOF session at Supercomputing 2015. + Get Involved! ------------------------ @@ -66,6 +72,15 @@ Many thanks go to Spack's [contributors](https://github.com/scalability-llnl/spa Spack was originally written by Todd Gamblin, tgamblin@llnl.gov. +### Citing Spack + +If you are referencing Spack in a publication, please cite the following paper: + + * Todd Gamblin, Matthew P. LeGendre, Michael R. Collette, Gregory L. Lee, + Adam Moody, Bronis R. de Supinski, and W. Scott Futral. + [**The Spack Package Manager: Bringing Order to HPC Software Chaos**](http://www.computer.org/csdl/proceedings/sc/2015/3723/00/2807623.pdf). + In *Supercomputing 2015 (SC’15)*, Austin, Texas, November 15-20 2015. LLNL-CONF-669890. + Release ---------------- Spack is released under an LGPL license. For more details see the From b253308eeedc9c4ea7a9db579e2daa2117a46356 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Sun, 29 Nov 2015 19:20:05 -0800 Subject: [PATCH 110/910] returning flux to a building state for web release Flux package reworked to include all new dependencies, fixed issues with a clean-build of hwloc, lua and czmq as well that prevented flux from building cold on a minimal system. --- var/spack/packages/czmq/package.py | 16 +++++++++++++++- var/spack/packages/flux/package.py | 7 +++++-- var/spack/packages/hwloc/package.py | 2 ++ var/spack/packages/lua/package.py | 4 ++-- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/czmq/package.py b/var/spack/packages/czmq/package.py index a2f19475541..06653321798 100644 --- a/var/spack/packages/czmq/package.py +++ b/var/spack/packages/czmq/package.py @@ -1,4 +1,5 @@ from spack import * +import os class Czmq(Package): """ A C interface to the ZMQ library """ @@ -7,11 +8,24 @@ class Czmq(Package): version('3.0.2', '23e9885f7ee3ce88d99d0425f52e9be1', url='https://github.com/zeromq/czmq/archive/v3.0.2.tar.gz') + depends_on('libtool') + depends_on('automake') + depends_on('autoconf') + depends_on('pkg-config') depends_on('zeromq') def install(self, spec, prefix): bash = which("bash") - bash("./autogen.sh") + # Work around autogen.sh oddities + # bash("./autogen.sh") + mkdirp("config") + autoreconf = which("autoreconf") + autoreconf("--install", "--verbose", "--force", + "-I", "config", + "-I", os.path.join(spec['pkg-config'].prefix, "share", "aclocal"), + "-I", os.path.join(spec['automake'].prefix, "share", "aclocal"), + "-I", os.path.join(spec['libtool'].prefix, "share", "aclocal"), + ) configure("--prefix=%s" % prefix) make() diff --git a/var/spack/packages/flux/package.py b/var/spack/packages/flux/package.py index c128f46be8b..dc4e0594c5e 100644 --- a/var/spack/packages/flux/package.py +++ b/var/spack/packages/flux/package.py @@ -12,20 +12,23 @@ class Flux(Package): # Also needs autotools, but should use the system version if available depends_on("zeromq@4.0.4:") depends_on("czmq@2.2:") + depends_on("hwloc") depends_on("lua@5.1:5.1.99") depends_on("munge") depends_on("libjson-c") depends_on("libxslt") + depends_on("python") + depends_on("py-cffi") + # TODO: This provides a catalog, hacked with environment below for now depends_on("docbook-xml") depends_on("asciidoc") - depends_on("python") - depends_on("py-cffi") def install(self, spec, prefix): # Bootstrap with autotools bash = which('bash') bash('./autogen.sh') + bash('./autogen.sh') #yes, twice, intentionally # Fix asciidoc dependency on xml style sheets and whatnot os.environ['XML_CATALOG_FILES'] = os.path.join(spec['docbook-xml'].prefix, diff --git a/var/spack/packages/hwloc/package.py b/var/spack/packages/hwloc/package.py index 2f46399afd2..452a7d7ce3b 100644 --- a/var/spack/packages/hwloc/package.py +++ b/var/spack/packages/hwloc/package.py @@ -19,6 +19,8 @@ class Hwloc(Package): url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.1.tar.bz2') version('1.9', '1f9f9155682fe8946a97c08896109508') + depends_on('libpciaccess') + def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/packages/lua/package.py b/var/spack/packages/lua/package.py index fa4b4f9edcb..6d8f7806d98 100644 --- a/var/spack/packages/lua/package.py +++ b/var/spack/packages/lua/package.py @@ -21,8 +21,8 @@ class Lua(Package): def install(self, spec, prefix): make('INSTALL_TOP=%s' % prefix, - 'MYLDFLAGS=-L%s/lib' % spec['ncurses'].prefix, + 'MYLDFLAGS="-L%s/lib -Wl,-rpath,%s"' % (spec['ncurses'].prefix,spec['ncurses'].prefix), 'linux') make('INSTALL_TOP=%s' % prefix, - 'MYLDFLAGS=-L%s/lib' % spec['ncurses'].prefix, + 'MYLDFLAGS="-L%s/lib -Wl,-rpath,%s"' % (spec['ncurses'].prefix,spec['ncurses'].prefix), 'install') From 45ffcdee7b86c483ba2e42e86feed5771cd3f8e5 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 29 Nov 2015 21:40:23 -0800 Subject: [PATCH 111/910] Add very basic Travis CI support. --- .travis.yml | 23 +++++++++++++++++++++++ README.md | 2 ++ 2 files changed, 25 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000000..c39dbf0c63b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,23 @@ +language: python +python: + - "2.6" + - "2.7" + +# No need to install any deps. +install: true + +before_install: + # Need this for the git tests to succeed. + - git config --global user.email "spack@example.com" + - git config --global user.name "Test User" + +script: + - . share/spack/setup-env.sh + - spack test + +notifications: + email: + recipients: + - tgamblin@llnl.gov + on_success: change + on_failure: always diff --git a/README.md b/README.md index 1df1db8bd69..9830e31356a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ ![image](share/spack/logo/spack-logo-text-64.png "Spack") ============ +[![Build Status](https://travis-ci.org/scalability-llnl/spack.png?branch=develop)](https://travis-ci.org/scalability-llnl/spack) + Spack is a package management tool designed to support multiple versions and configurations of software on a wide variety of platforms and environments. It was designed for large supercomputing centers, From 55237b094a20b39c30ae75fe40b0bf842cf4abae Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 29 Nov 2015 22:09:11 -0800 Subject: [PATCH 112/910] Use new travis insfrastructure (sudo:false) --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index c39dbf0c63b..ab379be4867 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,9 @@ python: - "2.6" - "2.7" +# Use new Travis infrastructure (Docker can't sudo yet) +sudo: false + # No need to install any deps. install: true From d5c7cd5e50bc0355eebb7d5ef9209a600712ed0e Mon Sep 17 00:00:00 2001 From: Kevin Huck Date: Mon, 30 Nov 2015 09:48:15 -0800 Subject: [PATCH 113/910] Had to force a patch to the OMPT support, so the hash for v0.1 has changed. --- var/spack/packages/ompt-openmp/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/ompt-openmp/package.py b/var/spack/packages/ompt-openmp/package.py index 9c1b4b95701..3edd9cbddd5 100644 --- a/var/spack/packages/ompt-openmp/package.py +++ b/var/spack/packages/ompt-openmp/package.py @@ -5,7 +5,7 @@ class OmptOpenmp(Package): homepage = "https://github.com/OpenMPToolsInterface/LLVM-openmp" url = "http://github.com/khuck/LLVM-openmp/archive/v0.1.tar.gz" - version('0.1', '3375b5ce67a48cae107371fcd811f639') + version('0.1', 'cc004c28b6c0d564340ed6174045b370') # depends_on("foo") From cd1207329b953c36822bd79cb6c7a965e21be80a Mon Sep 17 00:00:00 2001 From: Kevin Huck Date: Mon, 30 Nov 2015 10:32:07 -0800 Subject: [PATCH 114/910] Updating the OMPT hash again... --- var/spack/packages/ompt-openmp/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/ompt-openmp/package.py b/var/spack/packages/ompt-openmp/package.py index 3edd9cbddd5..e5bcfb51f03 100644 --- a/var/spack/packages/ompt-openmp/package.py +++ b/var/spack/packages/ompt-openmp/package.py @@ -5,7 +5,7 @@ class OmptOpenmp(Package): homepage = "https://github.com/OpenMPToolsInterface/LLVM-openmp" url = "http://github.com/khuck/LLVM-openmp/archive/v0.1.tar.gz" - version('0.1', 'cc004c28b6c0d564340ed6174045b370') + version('0.1', '2334e6a84b52da41b27afd9831ed5370') # depends_on("foo") From a173ab1e3185e322406d0b7379ea77872fbe34c2 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 1 Dec 2015 12:16:08 +0100 Subject: [PATCH 115/910] clang : default OpenMP is libomp --- var/spack/packages/clang/package.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py index d0869362f55..eac1863b975 100644 --- a/var/spack/packages/clang/package.py +++ b/var/spack/packages/clang/package.py @@ -50,9 +50,15 @@ def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag with working_dir('spack-build', create=True): + + options = [] + if '@3.7.0:' in spec: + options.append('-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp') + options.extend(std_cmake_args) + cmake('..', - '-DCLANG_PATH_TO_LLVM_BUILD=%s' % spec['llvm'].prefix, - '-DLLVM_MAIN_SRC_DIR=%s' % spec['llvm'].prefix, - *std_cmake_args) + '-DCLANG_PATH_TO_LLVM_BUILD:PATH=%s' % spec['llvm'].prefix, + '-DLLVM_MAIN_SRC_DIR:PATH=%s' % spec['llvm'].prefix, + *options) make() make("install") From a075d581eff5e07cd4b703cbcd1d1e5f490b1fb4 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 1 Dec 2015 12:56:46 +0100 Subject: [PATCH 116/910] resource : fetch strategy constructed from kwargs instead or hardcoded URLFetchStrategy --- lib/spack/spack/directives.py | 5 ++--- lib/spack/spack/fetch_strategy.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 4d8333fd7db..741df3a31b9 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -62,7 +62,7 @@ class OpenMpi(Package): from spack.variant import Variant from spack.spec import Spec, parse_anonymous_spec from spack.resource import Resource -from spack.fetch_strategy import URLFetchStrategy +from spack.fetch_strategy import from_kwargs # # This is a list of all directives, built up as they are defined in @@ -279,8 +279,7 @@ def resource(pkg, **kwargs): destination = kwargs.get('destination', "") when_spec = parse_anonymous_spec(when, pkg.name) resources = pkg.resources.setdefault(when_spec, []) - # FIXME : change URLFetchStrategy with a factory that selects based on kwargs - fetcher = URLFetchStrategy(**kwargs) + fetcher = from_kwargs(**kwargs) # FIXME : should we infer the name somehow if not passed ? name = kwargs.get('name') resources.append(Resource(name, fetcher, destination)) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 5e6850d14b6..1f5ef16caa2 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -634,6 +634,22 @@ def from_url(url): return URLFetchStrategy(url) +def from_kwargs(**kwargs): + """ + Construct the appropriate FetchStrategy from the given keyword arguments. + + :param kwargs: dictionary of keyword arguments + :return: fetcher or raise a FetchError exception + """ + for fetcher in all_strategies: + if fetcher.matches(kwargs): + return fetcher(**kwargs) + # Raise an error in case we can't instantiate any known strategy + message = "Cannot instantiate any FetchStrategy" + long_message = message + " from the given arguments : {arguments}".format(srguments=kwargs) + raise FetchError(message, long_message) + + def args_are_for(args, fetcher): fetcher.matches(args) From d3d9b5401b4d265f4de927cd34828a8c8b99e923 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 1 Dec 2015 13:26:44 +0100 Subject: [PATCH 117/910] resources : changed name of stage folder --- lib/spack/spack/package.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 9fee962df3b..c1dbbafc797 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -643,10 +643,9 @@ def do_fetch(self): ########## # Fetch resources resources = self._get_resources() - # FIXME : choose the unique name appropriately. Is there a function somewhere for the base name ? - pieces = [self.name, str(self.version), self.spec.dag_hash()] for resource in resources: - resource_stage_folder = '-'.join(pieces + [resource.name]) + pieces = ['resource', resource.name, self.spec.dag_hash()] + resource_stage_folder = '-'.join(pieces) stage = Stage(resource.fetcher, name=resource_stage_folder) resource.fetcher.set_stage(stage) resource.fetcher.fetch() From b85dccca927bdd2e9c5d30841544ef4328ce9d56 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 1 Dec 2015 15:12:26 +0100 Subject: [PATCH 118/910] resources : added error handling for destination keyword on resource directive --- lib/spack/spack/directives.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 741df3a31b9..48a7df74629 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -42,6 +42,7 @@ class OpenMpi(Package): * ``extends`` * ``patch`` * ``variant`` + * ``resource`` """ __all__ = ['depends_on', 'extends', 'provides', 'patch', 'version', @@ -49,9 +50,11 @@ class OpenMpi(Package): import re import inspect +import os.path import functools from llnl.util.lang import * +from llnl.util.filesystem import join_path import spack import spack.spec @@ -271,16 +274,27 @@ def resource(pkg, **kwargs): List of recognized keywords: * 'when' : represents the condition upon which the resource is needed (optional) - * 'destination' : path where to extract / checkout the resource (optional) + * 'destination' : path where to extract / checkout the resource (optional). This path must be a relative path, + and it must fall inside the stage area of the main package. """ when = kwargs.get('when', pkg.name) - # FIXME : currently I assume destination to be a relative path (rooted at pkg.stage.source_path) destination = kwargs.get('destination', "") + # Check if the path is relative + if os.path.isabs(destination): + message = "The destination keyword of a resource directive can't be an absolute path.\n" + message += "\tdestination : '{dest}\n'".format(dest=destination) + raise RuntimeError(message) + # Check if the path falls within the main package stage area + test_path = 'stage_folder_root/' + normalized_destination = os.path.normpath(join_path(test_path, destination)) # Normalized absolute path + if test_path not in normalized_destination: + message = "The destination folder of a resource must fall within the main package stage directory.\n" + message += "\tdestination : '{dest}'\n".format(dest=destination) + raise RuntimeError(message) when_spec = parse_anonymous_spec(when, pkg.name) resources = pkg.resources.setdefault(when_spec, []) fetcher = from_kwargs(**kwargs) - # FIXME : should we infer the name somehow if not passed ? name = kwargs.get('name') resources.append(Resource(name, fetcher, destination)) From 4b2168ab8ef5b5e69545a9bd82ef8804a4108d75 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 2 Dec 2015 10:13:39 +0100 Subject: [PATCH 119/910] resources : added missing check after fetch --- lib/spack/spack/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c1dbbafc797..c10283a6ee5 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -649,6 +649,7 @@ def do_fetch(self): stage = Stage(resource.fetcher, name=resource_stage_folder) resource.fetcher.set_stage(stage) resource.fetcher.fetch() + resource.fetcher.check() ########## self._fetch_time = time.time() - start_time From 39a3cfd4d9fc8d0981ba118de9a1e049fba4b144 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 2 Dec 2015 12:24:37 +0100 Subject: [PATCH 120/910] reource directive accepts 'basename' keyword llvm : libc++ variant --- lib/spack/spack/directives.py | 5 ++++- lib/spack/spack/package.py | 3 ++- lib/spack/spack/resource.py | 4 +++- var/spack/packages/llvm/package.py | 9 ++++++++- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 48a7df74629..07ef32294b0 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -276,10 +276,12 @@ def resource(pkg, **kwargs): * 'when' : represents the condition upon which the resource is needed (optional) * 'destination' : path where to extract / checkout the resource (optional). This path must be a relative path, and it must fall inside the stage area of the main package. + * 'basename' : basename of the resource source folder within destination (optional). """ when = kwargs.get('when', pkg.name) destination = kwargs.get('destination', "") + basename = kwargs.get('basename', None) # Check if the path is relative if os.path.isabs(destination): message = "The destination keyword of a resource directive can't be an absolute path.\n" @@ -296,7 +298,8 @@ def resource(pkg, **kwargs): resources = pkg.resources.setdefault(when_spec, []) fetcher = from_kwargs(**kwargs) name = kwargs.get('name') - resources.append(Resource(name, fetcher, destination)) + resources.append(Resource(name, fetcher, destination, basename)) + class DirectiveError(spack.error.SpackError): """This is raised when something is wrong with a package directive.""" diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c10283a6ee5..3f48231c754 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -681,7 +681,8 @@ def _expand_archive(stage, name=self.name): for resource in resources: stage = resource.fetcher.stage _expand_archive(stage, resource.name) - link_path = join_path(self.stage.source_path, resource.destination, os.path.basename(stage.source_path)) + basename = os.path.basename(stage.source_path) if resource.basename is None else resource.basename + link_path = join_path(self.stage.source_path, resource.destination, basename) if not os.path.exists(link_path): # Create a symlink os.symlink(stage.source_path, link_path) diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py index 00ee2d49e4f..b20612686d9 100644 --- a/lib/spack/spack/resource.py +++ b/lib/spack/spack/resource.py @@ -27,11 +27,13 @@ package to enable optional features. """ + class Resource(object): """ Represents an optional resource. Aggregates a name, a fetcher and a destination. """ - def __init__(self, name, fetcher, destination): + def __init__(self, name, fetcher, destination, basename): self.name = name self.fetcher = fetcher self.destination = destination + self.basename = basename diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index b68aa82affe..872a6c082b5 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -41,15 +41,22 @@ class Llvm(Package): depends_on('python@2.7:') + variant('libcxx', default=False, description="Builds the LLVM Standard C++ library targeting C++11") + ########## # @3.7.0 - # TODO : Add support for libc++ <- libc++ABI <- libunwind with variant? resource(name='compiler-rt', url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5', destination='projects', when='@3.7.0') resource(name='openmp', url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f', destination='projects', when='@3.7.0') + resource(name='libcxx', + url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd', + destination='projects', basename='libcxx', when='+libcxx@3.7.0') + resource(name='libcxxabi', + url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772', + destination='projects', basename='libcxxabi', when='+libcxx@3.7.0') ########## def install(self, spec, prefix): From 4bb6f23ae12dc557b00c48628be5cf8310391ccf Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Thu, 26 Nov 2015 14:20:27 +0100 Subject: [PATCH 121/910] Adding variant for mpi and python + compression for iostream --- var/spack/packages/boost/package.py | 79 +++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index 35824d53a26..5248d7f3bdb 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -43,7 +43,15 @@ class Boost(Package): version('1.34.1', '2d938467e8a448a2c9763e0a9f8ca7e5') version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') + variant('python', default=False, description='Activate the component Boost.Python') + variant('mpi', default=False, description='Activate the component Boost.MPI') + variant('compression', default=True, description='Activate the compression Boost.iostreams') + depends_on('mpi', when='+mpi') + depends_on('python', when='+python') + depends_on('zlib', when='+compression') + depends_on('bzip2', when='+compression') + def url_for_version(self, version): """Handle Boost's weird URLs, which write the version two different ways.""" parts = [str(p) for p in Version(version)] @@ -52,15 +60,78 @@ def url_for_version(self, version): return "http://downloads.sourceforge.net/project/boost/boost/%s/boost_%s.tar.bz2" % ( dots, underscores) + def determine_toolset(self): + toolsets = {'gcc': 'gcc', + 'icpc': 'intel', + 'clang++': 'clang'} + + for cc, toolset in toolsets.iteritems(): + if(cc in self.compiler.cxx_names): + return toolset + + # fallback to gcc if no toolset found + return 'gcc' + + def determine_bootstrap_options(self, spec, options): + options.append('--with-toolset=%s' % self.determine_toolset()) + + without_libs = [] + if '~mpi' in spec: + without_libs.append('mpi') + if '~python' in spec: + without_libs.append('python') + else: + options.append('--with-python=%s' % (spec['python'].prefix.bin + '/python')) + + if without_libs: + options.append('--without-libraries=%s' % ','.join(without_libs)) + + with open('user-config.jam', 'w') as f: + if '+mpi' in spec: + f.write('using mpi : %s ;\n' % (spec['mpi'].prefix.bin + '/mpicxx')) + if '+python' in spec: + f.write('using python : %s : %s ;\n' % (spec['python'].version, + (spec['python'].prefix.bin + '/python'))) + + def determine_b2_options(self, spec, options): + if '+debug' in spec: + options.append('variant=debug') + else: + options.append('variant=release') + + if '~compression' in spec: + options.extend(['-s NO_BZIP2=1', + '-s NO_ZLIB=1', + ]) + + if '+compression' in spec: + options.extend(['-s BZIP2_INCLUDE=%s' % spec['bzip2'].prefix.include, + '-s BZIP2_LIBPATH=%s' % spec['bzip2'].prefix.lib, + '-s ZLIB_INCLUDE=%s' % spec['zlib'].prefix.include, + '-s ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib]) + + options.extend(['toolset=%s' % self.determine_toolset(), + 'link=static,shared', + '--layout=tagged']) def install(self, spec, prefix): + # to make him find the user-config.jam + env['BOOST_BUILD_PATH'] = './' + bootstrap = Executable('./bootstrap.sh') - bootstrap() + + bootstrap_options = ['--prefix=%s' % prefix] + self.determine_bootstrap_options(spec, bootstrap_options) + + bootstrap(*bootstrap_options) # b2 used to be called bjam, before 1.47 (sigh) b2name = './b2' if spec.satisfies('@1.47:') else './bjam' b2 = Executable(b2name) - b2('install', - '-j %s' % make_jobs, - '--prefix=%s' % prefix) + b2_options = ['-j %s' % make_jobs] + + self.determine_b2_options(spec, b2_options) + + b2('install', 'threading=single', *b2_options) + b2('install', 'threading=multi', *b2_options) From aa137d8be0101776f0841e3d75642245454d2b32 Mon Sep 17 00:00:00 2001 From: Simon Thompson Date: Wed, 2 Dec 2015 14:20:11 +0000 Subject: [PATCH 122/910] fix build to work against depends for modules created by setup.py and add python3 support --- var/spack/packages/python/package.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 31fb8b3d5ab..12b282910b4 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -2,6 +2,7 @@ import re from contextlib import closing from llnl.util.lang import match_predicate +from spack.util.environment import * from spack import * import spack @@ -16,22 +17,33 @@ class Python(Package): version('2.7.8', 'd235bdfa75b8396942e360a70487ee00') version('2.7.10', 'c685ef0b8e9f27b5e3db5db12b268ac6') + version('3.5.0', 'd149d2812f10cbe04c042232e7964171') depends_on("openssl") depends_on("bzip2") depends_on("readline") depends_on("ncurses") depends_on("sqlite") + depends_on("zlib") def install(self, spec, prefix): # Need this to allow python build to find the Python installation. env['PYTHONHOME'] = prefix env['MACOSX_DEPLOYMENT_TARGET'] = '10.6' - # Rest of install is pretty standard. + # Rest of install is pretty standard except setup.py needs to be able to read the CPPFLAGS + # and LDFLAGS as it scans for the library and headers to build configure("--prefix=%s" % prefix, "--with-threads", - "--enable-shared") + "--enable-shared", + "CPPFLAGS=-I%s/include -I%s/include -I%s/include -I%s/include -I%s/include -I%s/include" % ( + spec['openssl'].prefix, spec['bzip2'].prefix, + spec['readline'].prefix, spec['ncurses'].prefix, + spec['sqlite'].prefix, spec['zlib'].prefix), + "LDFLAGS=-L%s/lib -L%s/lib -L%s/lib -L%s/lib -L%s/lib -L%s/lib" % ( + spec['openssl'].prefix, spec['bzip2'].prefix, + spec['readline'].prefix, spec['ncurses'].prefix, + spec['sqlite'].prefix, spec['zlib'].prefix)) make() make("install") @@ -63,7 +75,10 @@ def setup_dependent_environment(self, module, spec, ext_spec): python('setup.py', 'install', '--prefix=%s' % prefix) """ # Python extension builds can have a global python executable function - module.python = Executable(join_path(spec.prefix.bin, 'python')) + if self.version >= Version("3.0.0") and self.version < Version("4.0.0"): + module.python = Executable(join_path(spec.prefix.bin, 'python3')) + else: + module.python = Executable(join_path(spec.prefix.bin, 'python')) # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) From 1a132e4fdb9271ed7dec0990aa419e1a1ed1a9a5 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 2 Dec 2015 16:18:29 +0100 Subject: [PATCH 123/910] Missing declaration of the debug variant --- var/spack/packages/boost/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index 5248d7f3bdb..81dadbbf611 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -43,6 +43,7 @@ class Boost(Package): version('1.34.1', '2d938467e8a448a2c9763e0a9f8ca7e5') version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') + variant('debug', default=False, description='Switch to the debug version of Boost') variant('python', default=False, description='Activate the component Boost.Python') variant('mpi', default=False, description='Activate the component Boost.MPI') variant('compression', default=True, description='Activate the compression Boost.iostreams') From b98b6122044a40e195c4d868ae33890d59b2f0ea Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 2 Dec 2015 18:10:28 -0800 Subject: [PATCH 124/910] Try to be more careful to only suppress the warning about nose already being imported. Other warnings should be output. --- bin/spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/spack b/bin/spack index b4f715ab60f..45be6ee0ffd 100755 --- a/bin/spack +++ b/bin/spack @@ -46,7 +46,7 @@ import warnings # spack. Note this must be done after Spack's external libs directory is added # to sys.path. with warnings.catch_warnings(): - warnings.simplefilter("ignore") + warnings.filterwarnings("ignore", ".*nose was already imported") import nose # Quick and dirty check to clean orphaned .pyc files left over from From 50bd4d2e4ed3fed83afa30e7796eaa99989bc3ce Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 2 Dec 2015 16:18:25 +0100 Subject: [PATCH 125/910] mirror : deals correctly with variants that optionally enable resources (if they are archive URLs) --- lib/spack/spack/mirror.py | 81 +++++++++++++++++++++++++++----------- lib/spack/spack/package.py | 19 ++++++--- 2 files changed, 72 insertions(+), 28 deletions(-) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 306c8085aa5..1c38d791641 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -26,7 +26,7 @@ This file contains code for creating spack mirror directories. A mirror is an organized hierarchy containing specially named archive files. This enabled spack to know where to find files in a mirror if -the main server for a particualr package is down. Or, if the computer +the main server for a particular package is down. Or, if the computer where spack is run is not connected to the internet, it allows spack to download packages directly from a mirror (e.g., on an intranet). """ @@ -42,7 +42,7 @@ from spack.spec import Spec from spack.stage import Stage from spack.version import * -from spack.util.compression import extension +from spack.util.compression import extension, allowed_archive def mirror_archive_filename(spec): @@ -87,11 +87,26 @@ def get_matching_versions(specs, **kwargs): if v.satisfies(spec.versions): s = Spec(pkg.name) s.versions = VersionList([v]) + s.variants = spec.variants.copy() matching.append(s) return matching +def suggest_archive_basename(resource): + """ + Return a tentative basename for an archive. Raise an exception if the name is among the allowed archive types. + + :param fetcher: + :return: + """ + basename = os.path.basename(resource.fetcher.url) + if not allowed_archive(basename): + raise RuntimeError("%s is not an allowed archive tye" % basename) + return basename + + + def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with package archives. @@ -108,7 +123,7 @@ def create(path, specs, **kwargs): Return Value: Returns a tuple of lists: (present, mirrored, error) - * present: Package specs that were already prsent. + * present: Package specs that were already present. * mirrored: Package specs that were successfully mirrored. * error: Package specs that failed to mirror due to some error. @@ -140,6 +155,7 @@ def create(path, specs, **kwargs): error = [] # Iterate through packages and download all the safe tarballs for each of them + everything_already_exists = True for spec in version_specs: pkg = spec.package @@ -152,26 +168,47 @@ def create(path, specs, **kwargs): if os.path.exists(archive_path): tty.msg("Already added %s" % spec.format("$_$@")) + else: + everything_already_exists = False + # Set up a stage and a fetcher for the download + unique_fetch_name = spec.format("$_$@") + fetcher = fs.for_package_version(pkg, pkg.version) + stage = Stage(fetcher, name=unique_fetch_name) + fetcher.set_stage(stage) + + # Do the fetch and checksum if necessary + fetcher.fetch() + if not kwargs.get('no_checksum', False): + fetcher.check() + tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version)) + + # Fetchers have to know how to archive their files. Use + # that to move/copy/create an archive in the mirror. + fetcher.archive(archive_path) + tty.msg("Added %s." % spec.format("$_$@")) + + # Fetch resources if they are associated with the spec + resources = pkg._get_resources() + for resource in resources: + resource_archive_path = join_path(subdir, suggest_archive_basename(resource)) + if os.path.exists(resource_archive_path): + tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version)) + continue + everything_already_exists = False + resource_stage_folder = pkg._resource_stage(resource) + resource_stage = Stage(resource.fetcher, name=resource_stage_folder) + resource.fetcher.set_stage(resource_stage) + resource.fetcher.fetch() + if not kwargs.get('no_checksum', False): + resource.fetcher.check() + tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version)) + resource.fetcher.archive(resource_archive_path) + tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version)) + + if everything_already_exists: present.append(spec) - continue - - # Set up a stage and a fetcher for the download - unique_fetch_name = spec.format("$_$@") - fetcher = fs.for_package_version(pkg, pkg.version) - stage = Stage(fetcher, name=unique_fetch_name) - fetcher.set_stage(stage) - - # Do the fetch and checksum if necessary - fetcher.fetch() - if not kwargs.get('no_checksum', False): - fetcher.check() - tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version)) - - # Fetchers have to know how to archive their files. Use - # that to move/copy/create an archive in the mirror. - fetcher.archive(archive_path) - tty.msg("Added %s." % spec.format("$_$@")) - mirrored.append(spec) + else: + mirrored.append(spec) except Exception, e: if spack.debug: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 3f48231c754..dcb514af007 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -644,12 +644,14 @@ def do_fetch(self): # Fetch resources resources = self._get_resources() for resource in resources: - pieces = ['resource', resource.name, self.spec.dag_hash()] - resource_stage_folder = '-'.join(pieces) - stage = Stage(resource.fetcher, name=resource_stage_folder) - resource.fetcher.set_stage(stage) - resource.fetcher.fetch() - resource.fetcher.check() + resource_stage_folder = self._resource_stage(resource) + # FIXME : works only for URLFetchStrategy + resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url)) + resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror) + resource.fetcher.set_stage(resource_stage) + # Delegate to stage object to trigger mirror logic + resource_stage.fetch() + resource_stage.check() ########## self._fetch_time = time.time() - start_time @@ -766,6 +768,11 @@ def _get_resources(self): resources.extend(resource_list) return resources + def _resource_stage(self, resource): + pieces = ['resource', resource.name, self.spec.dag_hash()] + resource_stage_folder = '-'.join(pieces) + return resource_stage_folder + def _build_logger(self, log_path): """Create a context manager to log build output.""" From 0f4efc018d9029cdc9b34b30930f8ee9a4abe542 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Thu, 3 Dec 2015 15:01:44 -0800 Subject: [PATCH 126/910] updated package versions --- var/spack/packages/llvm/package.py | 1 + var/spack/packages/openssl/package.py | 1 + var/spack/packages/pcre/package.py | 1 + 3 files changed, 3 insertions(+) diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index a6759c3033e..80c44e4d5fa 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -38,6 +38,7 @@ class Llvm(Package): version('3.7.0', 'b98b9495e5655a672d6cb83e1a180f8e', url='http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz') version('3.6.2', '0c1ee3597d75280dee603bae9cbf5cc2', url='http://llvm.org/releases/3.6.2/llvm-3.6.2.src.tar.xz') version('3.5.1', '2d3d8004f38852aa679e5945b8ce0b14', url='http://llvm.org/releases/3.5.1/llvm-3.5.1.src.tar.xz') + version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package depends_on('python@2.7:') diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index 4bf1f832d45..40648fca492 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -11,6 +11,7 @@ class Openssl(Package): version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') + version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') depends_on("zlib") parallel = False diff --git a/var/spack/packages/pcre/package.py b/var/spack/packages/pcre/package.py index 3424048a6cb..e38d337e3d3 100644 --- a/var/spack/packages/pcre/package.py +++ b/var/spack/packages/pcre/package.py @@ -8,6 +8,7 @@ class Pcre(Package): url = "ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.36.tar.bz2" version('8.36', 'b767bc9af0c20bc9c1fe403b0d41ad97') + version('8.38', '00aabbfe56d5a48b270f999b508c5ad2') def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 5891847ae49b7f281e643b07ec84e624f070b1f6 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Thu, 3 Dec 2015 15:35:28 -0800 Subject: [PATCH 127/910] added py-pillow and updated py-basemap to depend on it --- var/spack/packages/py-basemap/package.py | 4 ++-- var/spack/packages/py-pillow/package.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 var/spack/packages/py-pillow/package.py diff --git a/var/spack/packages/py-basemap/package.py b/var/spack/packages/py-basemap/package.py index 45f1085ba1a..8dfc99b28d6 100644 --- a/var/spack/packages/py-basemap/package.py +++ b/var/spack/packages/py-basemap/package.py @@ -11,8 +11,8 @@ class PyBasemap(Package): extends('python') depends_on('py-setuptools') depends_on('py-numpy') - depends_on('py-matplotlib') - depends_on('py-pil') + depends_on('py-matplotlib+gui') + depends_on('py-pillow') depends_on("geos") def install(self, spec, prefix): diff --git a/var/spack/packages/py-pillow/package.py b/var/spack/packages/py-pillow/package.py new file mode 100644 index 00000000000..adc8507bd57 --- /dev/null +++ b/var/spack/packages/py-pillow/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyPillow(Package): + """Pillow is the friendly PIL fork by Alex Clark and Contributors. PIL is the Python Imaging Library by Fredrik Lundh and Contributors. The Python Imaging Library (PIL) adds image processing capabilities to your Python interpreter. This library supports many file formats, and provides powerful image processing and graphics capabilities.""" + + homepage = "https://python-pillow.github.io/" + url = "https://pypi.python.org/packages/source/P/Pillow/Pillow-3.0.0.tar.gz" + + version('3.0.0', 'fc8ac44e93da09678eac7e30c9b7377d') + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 530e049d4a3f50481e7d55e8b61a9e840f48ad76 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 4 Dec 2015 17:55:20 -0800 Subject: [PATCH 128/910] Added nose 1.3.7 source along with with the lgpl license (renamed from lgpl.txt to LICENSE) --- lib/spack/external/nose/LICENSE | 504 ++++ lib/spack/external/nose/__init__.py | 15 + lib/spack/external/nose/__main__.py | 8 + lib/spack/external/nose/case.py | 397 +++ lib/spack/external/nose/commands.py | 172 ++ lib/spack/external/nose/config.py | 661 +++++ lib/spack/external/nose/core.py | 341 +++ lib/spack/external/nose/exc.py | 9 + lib/spack/external/nose/ext/__init__.py | 3 + lib/spack/external/nose/ext/dtcompat.py | 2272 +++++++++++++++++ lib/spack/external/nose/failure.py | 42 + lib/spack/external/nose/importer.py | 167 ++ lib/spack/external/nose/inspector.py | 207 ++ lib/spack/external/nose/loader.py | 623 +++++ lib/spack/external/nose/plugins/__init__.py | 190 ++ lib/spack/external/nose/plugins/allmodules.py | 45 + lib/spack/external/nose/plugins/attrib.py | 286 +++ lib/spack/external/nose/plugins/base.py | 725 ++++++ lib/spack/external/nose/plugins/builtin.py | 34 + lib/spack/external/nose/plugins/capture.py | 115 + lib/spack/external/nose/plugins/collect.py | 94 + lib/spack/external/nose/plugins/cover.py | 271 ++ lib/spack/external/nose/plugins/debug.py | 67 + lib/spack/external/nose/plugins/deprecated.py | 45 + lib/spack/external/nose/plugins/doctests.py | 455 ++++ lib/spack/external/nose/plugins/errorclass.py | 210 ++ .../external/nose/plugins/failuredetail.py | 49 + lib/spack/external/nose/plugins/isolate.py | 103 + lib/spack/external/nose/plugins/logcapture.py | 245 ++ lib/spack/external/nose/plugins/manager.py | 460 ++++ .../external/nose/plugins/multiprocess.py | 835 ++++++ lib/spack/external/nose/plugins/plugintest.py | 416 +++ lib/spack/external/nose/plugins/prof.py | 154 ++ lib/spack/external/nose/plugins/skip.py | 63 + lib/spack/external/nose/plugins/testid.py | 311 +++ lib/spack/external/nose/plugins/xunit.py | 341 +++ lib/spack/external/nose/proxy.py | 188 ++ lib/spack/external/nose/pyversion.py | 215 ++ lib/spack/external/nose/result.py | 200 ++ lib/spack/external/nose/selector.py | 251 ++ lib/spack/external/nose/sphinx/__init__.py | 1 + lib/spack/external/nose/sphinx/pluginopts.py | 189 ++ lib/spack/external/nose/suite.py | 609 +++++ lib/spack/external/nose/tools/__init__.py | 15 + lib/spack/external/nose/tools/nontrivial.py | 151 ++ lib/spack/external/nose/tools/trivial.py | 54 + lib/spack/external/nose/twistedtools.py | 173 ++ lib/spack/external/nose/usage.txt | 115 + lib/spack/external/nose/util.py | 668 +++++ 49 files changed, 13764 insertions(+) create mode 100644 lib/spack/external/nose/LICENSE create mode 100644 lib/spack/external/nose/__init__.py create mode 100644 lib/spack/external/nose/__main__.py create mode 100644 lib/spack/external/nose/case.py create mode 100644 lib/spack/external/nose/commands.py create mode 100644 lib/spack/external/nose/config.py create mode 100644 lib/spack/external/nose/core.py create mode 100644 lib/spack/external/nose/exc.py create mode 100644 lib/spack/external/nose/ext/__init__.py create mode 100644 lib/spack/external/nose/ext/dtcompat.py create mode 100644 lib/spack/external/nose/failure.py create mode 100644 lib/spack/external/nose/importer.py create mode 100644 lib/spack/external/nose/inspector.py create mode 100644 lib/spack/external/nose/loader.py create mode 100644 lib/spack/external/nose/plugins/__init__.py create mode 100644 lib/spack/external/nose/plugins/allmodules.py create mode 100644 lib/spack/external/nose/plugins/attrib.py create mode 100644 lib/spack/external/nose/plugins/base.py create mode 100644 lib/spack/external/nose/plugins/builtin.py create mode 100644 lib/spack/external/nose/plugins/capture.py create mode 100644 lib/spack/external/nose/plugins/collect.py create mode 100644 lib/spack/external/nose/plugins/cover.py create mode 100644 lib/spack/external/nose/plugins/debug.py create mode 100644 lib/spack/external/nose/plugins/deprecated.py create mode 100644 lib/spack/external/nose/plugins/doctests.py create mode 100644 lib/spack/external/nose/plugins/errorclass.py create mode 100644 lib/spack/external/nose/plugins/failuredetail.py create mode 100644 lib/spack/external/nose/plugins/isolate.py create mode 100644 lib/spack/external/nose/plugins/logcapture.py create mode 100644 lib/spack/external/nose/plugins/manager.py create mode 100644 lib/spack/external/nose/plugins/multiprocess.py create mode 100644 lib/spack/external/nose/plugins/plugintest.py create mode 100644 lib/spack/external/nose/plugins/prof.py create mode 100644 lib/spack/external/nose/plugins/skip.py create mode 100644 lib/spack/external/nose/plugins/testid.py create mode 100644 lib/spack/external/nose/plugins/xunit.py create mode 100644 lib/spack/external/nose/proxy.py create mode 100644 lib/spack/external/nose/pyversion.py create mode 100644 lib/spack/external/nose/result.py create mode 100644 lib/spack/external/nose/selector.py create mode 100644 lib/spack/external/nose/sphinx/__init__.py create mode 100644 lib/spack/external/nose/sphinx/pluginopts.py create mode 100644 lib/spack/external/nose/suite.py create mode 100644 lib/spack/external/nose/tools/__init__.py create mode 100644 lib/spack/external/nose/tools/nontrivial.py create mode 100644 lib/spack/external/nose/tools/trivial.py create mode 100644 lib/spack/external/nose/twistedtools.py create mode 100644 lib/spack/external/nose/usage.txt create mode 100644 lib/spack/external/nose/util.py diff --git a/lib/spack/external/nose/LICENSE b/lib/spack/external/nose/LICENSE new file mode 100644 index 00000000000..8add30ad590 --- /dev/null +++ b/lib/spack/external/nose/LICENSE @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/lib/spack/external/nose/__init__.py b/lib/spack/external/nose/__init__.py new file mode 100644 index 00000000000..1ae1362b7ac --- /dev/null +++ b/lib/spack/external/nose/__init__.py @@ -0,0 +1,15 @@ +from nose.core import collector, main, run, run_exit, runmodule +# backwards compatibility +from nose.exc import SkipTest, DeprecatedTest +from nose.tools import with_setup + +__author__ = 'Jason Pellerin' +__versioninfo__ = (1, 3, 7) +__version__ = '.'.join(map(str, __versioninfo__)) + +__all__ = [ + 'main', 'run', 'run_exit', 'runmodule', 'with_setup', + 'SkipTest', 'DeprecatedTest', 'collector' + ] + + diff --git a/lib/spack/external/nose/__main__.py b/lib/spack/external/nose/__main__.py new file mode 100644 index 00000000000..b402d9df12a --- /dev/null +++ b/lib/spack/external/nose/__main__.py @@ -0,0 +1,8 @@ +import sys + +from nose.core import run_exit + +if sys.argv[0].endswith('__main__.py'): + sys.argv[0] = '%s -m nose' % sys.executable + +run_exit() diff --git a/lib/spack/external/nose/case.py b/lib/spack/external/nose/case.py new file mode 100644 index 00000000000..cffa4ab4c9e --- /dev/null +++ b/lib/spack/external/nose/case.py @@ -0,0 +1,397 @@ +"""nose unittest.TestCase subclasses. It is not necessary to subclass these +classes when writing tests; they are used internally by nose.loader.TestLoader +to create test cases from test functions and methods in test classes. +""" +import logging +import sys +import unittest +from inspect import isfunction +from nose.config import Config +from nose.failure import Failure # for backwards compatibility +from nose.util import resolve_name, test_address, try_run + +log = logging.getLogger(__name__) + + +__all__ = ['Test'] + + +class Test(unittest.TestCase): + """The universal test case wrapper. + + When a plugin sees a test, it will always see an instance of this + class. To access the actual test case that will be run, access the + test property of the nose.case.Test instance. + """ + __test__ = False # do not collect + def __init__(self, test, config=None, resultProxy=None): + # sanity check + if not callable(test): + raise TypeError("nose.case.Test called with argument %r that " + "is not callable. A callable is required." + % test) + self.test = test + if config is None: + config = Config() + self.config = config + self.tbinfo = None + self.capturedOutput = None + self.resultProxy = resultProxy + self.plugins = config.plugins + self.passed = None + unittest.TestCase.__init__(self) + + def __call__(self, *arg, **kwarg): + return self.run(*arg, **kwarg) + + def __str__(self): + name = self.plugins.testName(self) + if name is not None: + return name + return str(self.test) + + def __repr__(self): + return "Test(%r)" % self.test + + def afterTest(self, result): + """Called after test is complete (after result.stopTest) + """ + try: + afterTest = result.afterTest + except AttributeError: + pass + else: + afterTest(self.test) + + def beforeTest(self, result): + """Called before test is run (before result.startTest) + """ + try: + beforeTest = result.beforeTest + except AttributeError: + pass + else: + beforeTest(self.test) + + def exc_info(self): + """Extract exception info. + """ + exc, exv, tb = sys.exc_info() + return (exc, exv, tb) + + def id(self): + """Get a short(er) description of the test + """ + return self.test.id() + + def address(self): + """Return a round-trip name for this test, a name that can be + fed back as input to loadTestByName and (assuming the same + plugin configuration) result in the loading of this test. + """ + if hasattr(self.test, 'address'): + return self.test.address() + else: + # not a nose case + return test_address(self.test) + + def _context(self): + try: + return self.test.context + except AttributeError: + pass + try: + return self.test.__class__ + except AttributeError: + pass + try: + return resolve_name(self.test.__module__) + except AttributeError: + pass + return None + context = property(_context, None, None, + """Get the context object of this test (if any).""") + + def run(self, result): + """Modified run for the test wrapper. + + From here we don't call result.startTest or stopTest or + addSuccess. The wrapper calls addError/addFailure only if its + own setup or teardown fails, or running the wrapped test fails + (eg, if the wrapped "test" is not callable). + + Two additional methods are called, beforeTest and + afterTest. These give plugins a chance to modify the wrapped + test before it is called and do cleanup after it is + called. They are called unconditionally. + """ + if self.resultProxy: + result = self.resultProxy(result, self) + try: + try: + self.beforeTest(result) + self.runTest(result) + except KeyboardInterrupt: + raise + except: + err = sys.exc_info() + result.addError(self, err) + finally: + self.afterTest(result) + + def runTest(self, result): + """Run the test. Plugins may alter the test by returning a + value from prepareTestCase. The value must be callable and + must accept one argument, the result instance. + """ + test = self.test + plug_test = self.config.plugins.prepareTestCase(self) + if plug_test is not None: + test = plug_test + test(result) + + def shortDescription(self): + desc = self.plugins.describeTest(self) + if desc is not None: + return desc + # work around bug in unittest.TestCase.shortDescription + # with multiline docstrings. + test = self.test + try: + test._testMethodDoc = test._testMethodDoc.strip()# 2.5 + except AttributeError: + try: + # 2.4 and earlier + test._TestCase__testMethodDoc = \ + test._TestCase__testMethodDoc.strip() + except AttributeError: + pass + # 2.7 compat: shortDescription() always returns something + # which is a change from 2.6 and below, and breaks the + # testName plugin call. + try: + desc = self.test.shortDescription() + except Exception: + # this is probably caused by a problem in test.__str__() and is + # only triggered by python 3.1's unittest! + pass + try: + if desc == str(self.test): + return + except Exception: + # If str() triggers an exception then ignore it. + # see issue 422 + pass + return desc + + +class TestBase(unittest.TestCase): + """Common functionality for FunctionTestCase and MethodTestCase. + """ + __test__ = False # do not collect + + def id(self): + return str(self) + + def runTest(self): + self.test(*self.arg) + + def shortDescription(self): + if hasattr(self.test, 'description'): + return self.test.description + func, arg = self._descriptors() + doc = getattr(func, '__doc__', None) + if not doc: + doc = str(self) + return doc.strip().split("\n")[0].strip() + + +class FunctionTestCase(TestBase): + """TestCase wrapper for test functions. + + Don't use this class directly; it is used internally in nose to + create test cases for test functions. + """ + __test__ = False # do not collect + + def __init__(self, test, setUp=None, tearDown=None, arg=tuple(), + descriptor=None): + """Initialize the MethodTestCase. + + Required argument: + + * test -- the test function to call. + + Optional arguments: + + * setUp -- function to run at setup. + + * tearDown -- function to run at teardown. + + * arg -- arguments to pass to the test function. This is to support + generator functions that yield arguments. + + * descriptor -- the function, other than the test, that should be used + to construct the test name. This is to support generator functions. + """ + + self.test = test + self.setUpFunc = setUp + self.tearDownFunc = tearDown + self.arg = arg + self.descriptor = descriptor + TestBase.__init__(self) + + def address(self): + """Return a round-trip name for this test, a name that can be + fed back as input to loadTestByName and (assuming the same + plugin configuration) result in the loading of this test. + """ + if self.descriptor is not None: + return test_address(self.descriptor) + else: + return test_address(self.test) + + def _context(self): + return resolve_name(self.test.__module__) + context = property(_context, None, None, + """Get context (module) of this test""") + + def setUp(self): + """Run any setup function attached to the test function + """ + if self.setUpFunc: + self.setUpFunc() + else: + names = ('setup', 'setUp', 'setUpFunc') + try_run(self.test, names) + + def tearDown(self): + """Run any teardown function attached to the test function + """ + if self.tearDownFunc: + self.tearDownFunc() + else: + names = ('teardown', 'tearDown', 'tearDownFunc') + try_run(self.test, names) + + def __str__(self): + func, arg = self._descriptors() + if hasattr(func, 'compat_func_name'): + name = func.compat_func_name + else: + name = func.__name__ + name = "%s.%s" % (func.__module__, name) + if arg: + name = "%s%s" % (name, arg) + # FIXME need to include the full dir path to disambiguate + # in cases where test module of the same name was seen in + # another directory (old fromDirectory) + return name + __repr__ = __str__ + + def _descriptors(self): + """Get the descriptors of the test function: the function and + arguments that will be used to construct the test name. In + most cases, this is the function itself and no arguments. For + tests generated by generator functions, the original + (generator) function and args passed to the generated function + are returned. + """ + if self.descriptor: + return self.descriptor, self.arg + else: + return self.test, self.arg + + +class MethodTestCase(TestBase): + """Test case wrapper for test methods. + + Don't use this class directly; it is used internally in nose to + create test cases for test methods. + """ + __test__ = False # do not collect + + def __init__(self, method, test=None, arg=tuple(), descriptor=None): + """Initialize the MethodTestCase. + + Required argument: + + * method -- the method to call, may be bound or unbound. In either + case, a new instance of the method's class will be instantiated to + make the call. Note: In Python 3.x, if using an unbound method, you + must wrap it using pyversion.unbound_method. + + Optional arguments: + + * test -- the test function to call. If this is passed, it will be + called instead of getting a new bound method of the same name as the + desired method from the test instance. This is to support generator + methods that yield inline functions. + + * arg -- arguments to pass to the test function. This is to support + generator methods that yield arguments. + + * descriptor -- the function, other than the test, that should be used + to construct the test name. This is to support generator methods. + """ + self.method = method + self.test = test + self.arg = arg + self.descriptor = descriptor + if isfunction(method): + raise ValueError("Unbound methods must be wrapped using pyversion.unbound_method before passing to MethodTestCase") + self.cls = method.im_class + self.inst = self.cls() + if self.test is None: + method_name = self.method.__name__ + self.test = getattr(self.inst, method_name) + TestBase.__init__(self) + + def __str__(self): + func, arg = self._descriptors() + if hasattr(func, 'compat_func_name'): + name = func.compat_func_name + else: + name = func.__name__ + name = "%s.%s.%s" % (self.cls.__module__, + self.cls.__name__, + name) + if arg: + name = "%s%s" % (name, arg) + return name + __repr__ = __str__ + + def address(self): + """Return a round-trip name for this test, a name that can be + fed back as input to loadTestByName and (assuming the same + plugin configuration) result in the loading of this test. + """ + if self.descriptor is not None: + return test_address(self.descriptor) + else: + return test_address(self.method) + + def _context(self): + return self.cls + context = property(_context, None, None, + """Get context (class) of this test""") + + def setUp(self): + try_run(self.inst, ('setup', 'setUp')) + + def tearDown(self): + try_run(self.inst, ('teardown', 'tearDown')) + + def _descriptors(self): + """Get the descriptors of the test method: the method and + arguments that will be used to construct the test name. In + most cases, this is the method itself and no arguments. For + tests generated by generator methods, the original + (generator) method and args passed to the generated method + or function are returned. + """ + if self.descriptor: + return self.descriptor, self.arg + else: + return self.method, self.arg diff --git a/lib/spack/external/nose/commands.py b/lib/spack/external/nose/commands.py new file mode 100644 index 00000000000..ef0e9caed4e --- /dev/null +++ b/lib/spack/external/nose/commands.py @@ -0,0 +1,172 @@ +""" +nosetests setuptools command +---------------------------- + +The easiest way to run tests with nose is to use the `nosetests` setuptools +command:: + + python setup.py nosetests + +This command has one *major* benefit over the standard `test` command: *all +nose plugins are supported*. + +To configure the `nosetests` command, add a [nosetests] section to your +setup.cfg. The [nosetests] section can contain any command line arguments that +nosetests supports. The differences between issuing an option on the command +line and adding it to setup.cfg are: + +* In setup.cfg, the -- prefix must be excluded +* In setup.cfg, command line flags that take no arguments must be given an + argument flag (1, T or TRUE for active, 0, F or FALSE for inactive) + +Here's an example [nosetests] setup.cfg section:: + + [nosetests] + verbosity=1 + detailed-errors=1 + with-coverage=1 + cover-package=nose + debug=nose.loader + pdb=1 + pdb-failures=1 + +If you commonly run nosetests with a large number of options, using +the nosetests setuptools command and configuring with setup.cfg can +make running your tests much less tedious. (Note that the same options +and format supported in setup.cfg are supported in all other config +files, and the nosetests script will also load config files.) + +Another reason to run tests with the command is that the command will +install packages listed in your `tests_require`, as well as doing a +complete build of your package before running tests. For packages with +dependencies or that build C extensions, using the setuptools command +can be more convenient than building by hand and running the nosetests +script. + +Bootstrapping +------------- + +If you are distributing your project and want users to be able to run tests +without having to install nose themselves, add nose to the setup_requires +section of your setup():: + + setup( + # ... + setup_requires=['nose>=1.0'] + ) + +This will direct setuptools to download and activate nose during the setup +process, making the ``nosetests`` command available. + +""" +try: + from setuptools import Command +except ImportError: + Command = nosetests = None +else: + from nose.config import Config, option_blacklist, user_config_files, \ + flag, _bool + from nose.core import TestProgram + from nose.plugins import DefaultPluginManager + + + def get_user_options(parser): + """convert a optparse option list into a distutils option tuple list""" + opt_list = [] + for opt in parser.option_list: + if opt._long_opts[0][2:] in option_blacklist: + continue + long_name = opt._long_opts[0][2:] + if opt.action not in ('store_true', 'store_false'): + long_name = long_name + "=" + short_name = None + if opt._short_opts: + short_name = opt._short_opts[0][1:] + opt_list.append((long_name, short_name, opt.help or "")) + return opt_list + + + class nosetests(Command): + description = "Run unit tests using nosetests" + __config = Config(files=user_config_files(), + plugins=DefaultPluginManager()) + __parser = __config.getParser() + user_options = get_user_options(__parser) + + def initialize_options(self): + """create the member variables, but change hyphens to + underscores + """ + + self.option_to_cmds = {} + for opt in self.__parser.option_list: + cmd_name = opt._long_opts[0][2:] + option_name = cmd_name.replace('-', '_') + self.option_to_cmds[option_name] = cmd_name + setattr(self, option_name, None) + self.attr = None + + def finalize_options(self): + """nothing to do here""" + pass + + def run(self): + """ensure tests are capable of being run, then + run nose.main with a reconstructed argument list""" + if getattr(self.distribution, 'use_2to3', False): + # If we run 2to3 we can not do this inplace: + + # Ensure metadata is up-to-date + build_py = self.get_finalized_command('build_py') + build_py.inplace = 0 + build_py.run() + bpy_cmd = self.get_finalized_command("build_py") + build_path = bpy_cmd.build_lib + + # Build extensions + egg_info = self.get_finalized_command('egg_info') + egg_info.egg_base = build_path + egg_info.run() + + build_ext = self.get_finalized_command('build_ext') + build_ext.inplace = 0 + build_ext.run() + else: + self.run_command('egg_info') + + # Build extensions in-place + build_ext = self.get_finalized_command('build_ext') + build_ext.inplace = 1 + build_ext.run() + + if self.distribution.install_requires: + self.distribution.fetch_build_eggs( + self.distribution.install_requires) + if self.distribution.tests_require: + self.distribution.fetch_build_eggs( + self.distribution.tests_require) + + ei_cmd = self.get_finalized_command("egg_info") + argv = ['nosetests', '--where', ei_cmd.egg_base] + for (option_name, cmd_name) in self.option_to_cmds.items(): + if option_name in option_blacklist: + continue + value = getattr(self, option_name) + if value is not None: + argv.extend( + self.cfgToArg(option_name.replace('_', '-'), value)) + TestProgram(argv=argv, config=self.__config) + + def cfgToArg(self, optname, value): + argv = [] + long_optname = '--' + optname + opt = self.__parser.get_option(long_optname) + if opt.action in ('store_true', 'store_false'): + if not flag(value): + raise ValueError("Invalid value '%s' for '%s'" % ( + value, optname)) + if _bool(value): + argv.append(long_optname) + else: + argv.extend([long_optname, value]) + return argv diff --git a/lib/spack/external/nose/config.py b/lib/spack/external/nose/config.py new file mode 100644 index 00000000000..125eb5579d2 --- /dev/null +++ b/lib/spack/external/nose/config.py @@ -0,0 +1,661 @@ +import logging +import optparse +import os +import re +import sys +import ConfigParser +from optparse import OptionParser +from nose.util import absdir, tolist +from nose.plugins.manager import NoPlugins +from warnings import warn, filterwarnings + +log = logging.getLogger(__name__) + +# not allowed in config files +option_blacklist = ['help', 'verbose'] + +config_files = [ + # Linux users will prefer this + "~/.noserc", + # Windows users will prefer this + "~/nose.cfg" + ] + +# plaforms on which the exe check defaults to off +# Windows and IronPython +exe_allowed_platforms = ('win32', 'cli') + +filterwarnings("always", category=DeprecationWarning, + module=r'(.*\.)?nose\.config') + +class NoSuchOptionError(Exception): + def __init__(self, name): + Exception.__init__(self, name) + self.name = name + + +class ConfigError(Exception): + pass + + +class ConfiguredDefaultsOptionParser(object): + """ + Handler for options from commandline and config files. + """ + def __init__(self, parser, config_section, error=None, file_error=None): + self._parser = parser + self._config_section = config_section + if error is None: + error = self._parser.error + self._error = error + if file_error is None: + file_error = lambda msg, **kw: error(msg) + self._file_error = file_error + + def _configTuples(self, cfg, filename): + config = [] + if self._config_section in cfg.sections(): + for name, value in cfg.items(self._config_section): + config.append((name, value, filename)) + return config + + def _readFromFilenames(self, filenames): + config = [] + for filename in filenames: + cfg = ConfigParser.RawConfigParser() + try: + cfg.read(filename) + except ConfigParser.Error, exc: + raise ConfigError("Error reading config file %r: %s" % + (filename, str(exc))) + config.extend(self._configTuples(cfg, filename)) + return config + + def _readFromFileObject(self, fh): + cfg = ConfigParser.RawConfigParser() + try: + filename = fh.name + except AttributeError: + filename = '' + try: + cfg.readfp(fh) + except ConfigParser.Error, exc: + raise ConfigError("Error reading config file %r: %s" % + (filename, str(exc))) + return self._configTuples(cfg, filename) + + def _readConfiguration(self, config_files): + try: + config_files.readline + except AttributeError: + filename_or_filenames = config_files + if isinstance(filename_or_filenames, basestring): + filenames = [filename_or_filenames] + else: + filenames = filename_or_filenames + config = self._readFromFilenames(filenames) + else: + fh = config_files + config = self._readFromFileObject(fh) + return config + + def _processConfigValue(self, name, value, values, parser): + opt_str = '--' + name + option = parser.get_option(opt_str) + if option is None: + raise NoSuchOptionError(name) + else: + option.process(opt_str, value, values, parser) + + def _applyConfigurationToValues(self, parser, config, values): + for name, value, filename in config: + if name in option_blacklist: + continue + try: + self._processConfigValue(name, value, values, parser) + except NoSuchOptionError, exc: + self._file_error( + "Error reading config file %r: " + "no such option %r" % (filename, exc.name), + name=name, filename=filename) + except optparse.OptionValueError, exc: + msg = str(exc).replace('--' + name, repr(name), 1) + self._file_error("Error reading config file %r: " + "%s" % (filename, msg), + name=name, filename=filename) + + def parseArgsAndConfigFiles(self, args, config_files): + values = self._parser.get_default_values() + try: + config = self._readConfiguration(config_files) + except ConfigError, exc: + self._error(str(exc)) + else: + try: + self._applyConfigurationToValues(self._parser, config, values) + except ConfigError, exc: + self._error(str(exc)) + return self._parser.parse_args(args, values) + + +class Config(object): + """nose configuration. + + Instances of Config are used throughout nose to configure + behavior, including plugin lists. Here are the default values for + all config keys:: + + self.env = env = kw.pop('env', {}) + self.args = () + self.testMatch = re.compile(r'(?:^|[\\b_\\.%s-])[Tt]est' % os.sep) + self.addPaths = not env.get('NOSE_NOPATH', False) + self.configSection = 'nosetests' + self.debug = env.get('NOSE_DEBUG') + self.debugLog = env.get('NOSE_DEBUG_LOG') + self.exclude = None + self.getTestCaseNamesCompat = False + self.includeExe = env.get('NOSE_INCLUDE_EXE', + sys.platform in exe_allowed_platforms) + self.ignoreFiles = (re.compile(r'^\.'), + re.compile(r'^_'), + re.compile(r'^setup\.py$') + ) + self.include = None + self.loggingConfig = None + self.logStream = sys.stderr + self.options = NoOptions() + self.parser = None + self.plugins = NoPlugins() + self.srcDirs = ('lib', 'src') + self.runOnInit = True + self.stopOnError = env.get('NOSE_STOP', False) + self.stream = sys.stderr + self.testNames = () + self.verbosity = int(env.get('NOSE_VERBOSE', 1)) + self.where = () + self.py3where = () + self.workingDir = None + """ + + def __init__(self, **kw): + self.env = env = kw.pop('env', {}) + self.args = () + self.testMatchPat = env.get('NOSE_TESTMATCH', + r'(?:^|[\b_\.%s-])[Tt]est' % os.sep) + self.testMatch = re.compile(self.testMatchPat) + self.addPaths = not env.get('NOSE_NOPATH', False) + self.configSection = 'nosetests' + self.debug = env.get('NOSE_DEBUG') + self.debugLog = env.get('NOSE_DEBUG_LOG') + self.exclude = None + self.getTestCaseNamesCompat = False + self.includeExe = env.get('NOSE_INCLUDE_EXE', + sys.platform in exe_allowed_platforms) + self.ignoreFilesDefaultStrings = [r'^\.', + r'^_', + r'^setup\.py$', + ] + self.ignoreFiles = map(re.compile, self.ignoreFilesDefaultStrings) + self.include = None + self.loggingConfig = None + self.logStream = sys.stderr + self.options = NoOptions() + self.parser = None + self.plugins = NoPlugins() + self.srcDirs = ('lib', 'src') + self.runOnInit = True + self.stopOnError = env.get('NOSE_STOP', False) + self.stream = sys.stderr + self.testNames = [] + self.verbosity = int(env.get('NOSE_VERBOSE', 1)) + self.where = () + self.py3where = () + self.workingDir = os.getcwd() + self.traverseNamespace = False + self.firstPackageWins = False + self.parserClass = OptionParser + self.worker = False + + self._default = self.__dict__.copy() + self.update(kw) + self._orig = self.__dict__.copy() + + def __getstate__(self): + state = self.__dict__.copy() + del state['stream'] + del state['_orig'] + del state['_default'] + del state['env'] + del state['logStream'] + # FIXME remove plugins, have only plugin manager class + state['plugins'] = self.plugins.__class__ + return state + + def __setstate__(self, state): + plugincls = state.pop('plugins') + self.update(state) + self.worker = True + # FIXME won't work for static plugin lists + self.plugins = plugincls() + self.plugins.loadPlugins() + # needed so .can_configure gets set appropriately + dummy_parser = self.parserClass() + self.plugins.addOptions(dummy_parser, {}) + self.plugins.configure(self.options, self) + + def __repr__(self): + d = self.__dict__.copy() + # don't expose env, could include sensitive info + d['env'] = {} + keys = [ k for k in d.keys() + if not k.startswith('_') ] + keys.sort() + return "Config(%s)" % ', '.join([ '%s=%r' % (k, d[k]) + for k in keys ]) + __str__ = __repr__ + + def _parseArgs(self, argv, cfg_files): + def warn_sometimes(msg, name=None, filename=None): + if (hasattr(self.plugins, 'excludedOption') and + self.plugins.excludedOption(name)): + msg = ("Option %r in config file %r ignored: " + "excluded by runtime environment" % + (name, filename)) + warn(msg, RuntimeWarning) + else: + raise ConfigError(msg) + parser = ConfiguredDefaultsOptionParser( + self.getParser(), self.configSection, file_error=warn_sometimes) + return parser.parseArgsAndConfigFiles(argv[1:], cfg_files) + + def configure(self, argv=None, doc=None): + """Configure the nose running environment. Execute configure before + collecting tests with nose.TestCollector to enable output capture and + other features. + """ + env = self.env + if argv is None: + argv = sys.argv + + cfg_files = getattr(self, 'files', []) + options, args = self._parseArgs(argv, cfg_files) + # If -c --config has been specified on command line, + # load those config files and reparse + if getattr(options, 'files', []): + options, args = self._parseArgs(argv, options.files) + + self.options = options + if args: + self.testNames = args + if options.testNames is not None: + self.testNames.extend(tolist(options.testNames)) + + if options.py3where is not None: + if sys.version_info >= (3,): + options.where = options.py3where + + # `where` is an append action, so it can't have a default value + # in the parser, or that default will always be in the list + if not options.where: + options.where = env.get('NOSE_WHERE', None) + + # include and exclude also + if not options.ignoreFiles: + options.ignoreFiles = env.get('NOSE_IGNORE_FILES', []) + if not options.include: + options.include = env.get('NOSE_INCLUDE', []) + if not options.exclude: + options.exclude = env.get('NOSE_EXCLUDE', []) + + self.addPaths = options.addPaths + self.stopOnError = options.stopOnError + self.verbosity = options.verbosity + self.includeExe = options.includeExe + self.traverseNamespace = options.traverseNamespace + self.debug = options.debug + self.debugLog = options.debugLog + self.loggingConfig = options.loggingConfig + self.firstPackageWins = options.firstPackageWins + self.configureLogging() + + if not options.byteCompile: + sys.dont_write_bytecode = True + + if options.where is not None: + self.configureWhere(options.where) + + if options.testMatch: + self.testMatch = re.compile(options.testMatch) + + if options.ignoreFiles: + self.ignoreFiles = map(re.compile, tolist(options.ignoreFiles)) + log.info("Ignoring files matching %s", options.ignoreFiles) + else: + log.info("Ignoring files matching %s", self.ignoreFilesDefaultStrings) + + if options.include: + self.include = map(re.compile, tolist(options.include)) + log.info("Including tests matching %s", options.include) + + if options.exclude: + self.exclude = map(re.compile, tolist(options.exclude)) + log.info("Excluding tests matching %s", options.exclude) + + # When listing plugins we don't want to run them + if not options.showPlugins: + self.plugins.configure(options, self) + self.plugins.begin() + + def configureLogging(self): + """Configure logging for nose, or optionally other packages. Any logger + name may be set with the debug option, and that logger will be set to + debug level and be assigned the same handler as the nose loggers, unless + it already has a handler. + """ + if self.loggingConfig: + from logging.config import fileConfig + fileConfig(self.loggingConfig) + return + + format = logging.Formatter('%(name)s: %(levelname)s: %(message)s') + if self.debugLog: + handler = logging.FileHandler(self.debugLog) + else: + handler = logging.StreamHandler(self.logStream) + handler.setFormatter(format) + + logger = logging.getLogger('nose') + logger.propagate = 0 + + # only add our default handler if there isn't already one there + # this avoids annoying duplicate log messages. + found = False + if self.debugLog: + debugLogAbsPath = os.path.abspath(self.debugLog) + for h in logger.handlers: + if type(h) == logging.FileHandler and \ + h.baseFilename == debugLogAbsPath: + found = True + else: + for h in logger.handlers: + if type(h) == logging.StreamHandler and \ + h.stream == self.logStream: + found = True + if not found: + logger.addHandler(handler) + + # default level + lvl = logging.WARNING + if self.verbosity >= 5: + lvl = 0 + elif self.verbosity >= 4: + lvl = logging.DEBUG + elif self.verbosity >= 3: + lvl = logging.INFO + logger.setLevel(lvl) + + # individual overrides + if self.debug: + # no blanks + debug_loggers = [ name for name in self.debug.split(',') + if name ] + for logger_name in debug_loggers: + l = logging.getLogger(logger_name) + l.setLevel(logging.DEBUG) + if not l.handlers and not logger_name.startswith('nose'): + l.addHandler(handler) + + def configureWhere(self, where): + """Configure the working directory or directories for the test run. + """ + from nose.importer import add_path + self.workingDir = None + where = tolist(where) + warned = False + for path in where: + if not self.workingDir: + abs_path = absdir(path) + if abs_path is None: + raise ValueError("Working directory '%s' not found, or " + "not a directory" % path) + log.info("Set working dir to %s", abs_path) + self.workingDir = abs_path + if self.addPaths and \ + os.path.exists(os.path.join(abs_path, '__init__.py')): + log.info("Working directory %s is a package; " + "adding to sys.path" % abs_path) + add_path(abs_path) + continue + if not warned: + warn("Use of multiple -w arguments is deprecated and " + "support may be removed in a future release. You can " + "get the same behavior by passing directories without " + "the -w argument on the command line, or by using the " + "--tests argument in a configuration file.", + DeprecationWarning) + warned = True + self.testNames.append(path) + + def default(self): + """Reset all config values to defaults. + """ + self.__dict__.update(self._default) + + def getParser(self, doc=None): + """Get the command line option parser. + """ + if self.parser: + return self.parser + env = self.env + parser = self.parserClass(doc) + parser.add_option( + "-V","--version", action="store_true", + dest="version", default=False, + help="Output nose version and exit") + parser.add_option( + "-p", "--plugins", action="store_true", + dest="showPlugins", default=False, + help="Output list of available plugins and exit. Combine with " + "higher verbosity for greater detail") + parser.add_option( + "-v", "--verbose", + action="count", dest="verbosity", + default=self.verbosity, + help="Be more verbose. [NOSE_VERBOSE]") + parser.add_option( + "--verbosity", action="store", dest="verbosity", + metavar='VERBOSITY', + type="int", help="Set verbosity; --verbosity=2 is " + "the same as -v") + parser.add_option( + "-q", "--quiet", action="store_const", const=0, dest="verbosity", + help="Be less verbose") + parser.add_option( + "-c", "--config", action="append", dest="files", + metavar="FILES", + help="Load configuration from config file(s). May be specified " + "multiple times; in that case, all config files will be " + "loaded and combined") + parser.add_option( + "-w", "--where", action="append", dest="where", + metavar="WHERE", + help="Look for tests in this directory. " + "May be specified multiple times. The first directory passed " + "will be used as the working directory, in place of the current " + "working directory, which is the default. Others will be added " + "to the list of tests to execute. [NOSE_WHERE]" + ) + parser.add_option( + "--py3where", action="append", dest="py3where", + metavar="PY3WHERE", + help="Look for tests in this directory under Python 3.x. " + "Functions the same as 'where', but only applies if running under " + "Python 3.x or above. Note that, if present under 3.x, this " + "option completely replaces any directories specified with " + "'where', so the 'where' option becomes ineffective. " + "[NOSE_PY3WHERE]" + ) + parser.add_option( + "-m", "--match", "--testmatch", action="store", + dest="testMatch", metavar="REGEX", + help="Files, directories, function names, and class names " + "that match this regular expression are considered tests. " + "Default: %s [NOSE_TESTMATCH]" % self.testMatchPat, + default=self.testMatchPat) + parser.add_option( + "--tests", action="store", dest="testNames", default=None, + metavar='NAMES', + help="Run these tests (comma-separated list). This argument is " + "useful mainly from configuration files; on the command line, " + "just pass the tests to run as additional arguments with no " + "switch.") + parser.add_option( + "-l", "--debug", action="store", + dest="debug", default=self.debug, + help="Activate debug logging for one or more systems. " + "Available debug loggers: nose, nose.importer, " + "nose.inspector, nose.plugins, nose.result and " + "nose.selector. Separate multiple names with a comma.") + parser.add_option( + "--debug-log", dest="debugLog", action="store", + default=self.debugLog, metavar="FILE", + help="Log debug messages to this file " + "(default: sys.stderr)") + parser.add_option( + "--logging-config", "--log-config", + dest="loggingConfig", action="store", + default=self.loggingConfig, metavar="FILE", + help="Load logging config from this file -- bypasses all other" + " logging config settings.") + parser.add_option( + "-I", "--ignore-files", action="append", dest="ignoreFiles", + metavar="REGEX", + help="Completely ignore any file that matches this regular " + "expression. Takes precedence over any other settings or " + "plugins. " + "Specifying this option will replace the default setting. " + "Specify this option multiple times " + "to add more regular expressions [NOSE_IGNORE_FILES]") + parser.add_option( + "-e", "--exclude", action="append", dest="exclude", + metavar="REGEX", + help="Don't run tests that match regular " + "expression [NOSE_EXCLUDE]") + parser.add_option( + "-i", "--include", action="append", dest="include", + metavar="REGEX", + help="This regular expression will be applied to files, " + "directories, function names, and class names for a chance " + "to include additional tests that do not match TESTMATCH. " + "Specify this option multiple times " + "to add more regular expressions [NOSE_INCLUDE]") + parser.add_option( + "-x", "--stop", action="store_true", dest="stopOnError", + default=self.stopOnError, + help="Stop running tests after the first error or failure") + parser.add_option( + "-P", "--no-path-adjustment", action="store_false", + dest="addPaths", + default=self.addPaths, + help="Don't make any changes to sys.path when " + "loading tests [NOSE_NOPATH]") + parser.add_option( + "--exe", action="store_true", dest="includeExe", + default=self.includeExe, + help="Look for tests in python modules that are " + "executable. Normal behavior is to exclude executable " + "modules, since they may not be import-safe " + "[NOSE_INCLUDE_EXE]") + parser.add_option( + "--noexe", action="store_false", dest="includeExe", + help="DO NOT look for tests in python modules that are " + "executable. (The default on the windows platform is to " + "do so.)") + parser.add_option( + "--traverse-namespace", action="store_true", + default=self.traverseNamespace, dest="traverseNamespace", + help="Traverse through all path entries of a namespace package") + parser.add_option( + "--first-package-wins", "--first-pkg-wins", "--1st-pkg-wins", + action="store_true", default=False, dest="firstPackageWins", + help="nose's importer will normally evict a package from sys." + "modules if it sees a package with the same name in a different " + "location. Set this option to disable that behavior.") + parser.add_option( + "--no-byte-compile", + action="store_false", default=True, dest="byteCompile", + help="Prevent nose from byte-compiling the source into .pyc files " + "while nose is scanning for and running tests.") + + self.plugins.loadPlugins() + self.pluginOpts(parser) + + self.parser = parser + return parser + + def help(self, doc=None): + """Return the generated help message + """ + return self.getParser(doc).format_help() + + def pluginOpts(self, parser): + self.plugins.addOptions(parser, self.env) + + def reset(self): + self.__dict__.update(self._orig) + + def todict(self): + return self.__dict__.copy() + + def update(self, d): + self.__dict__.update(d) + + +class NoOptions(object): + """Options container that returns None for all options. + """ + def __getstate__(self): + return {} + + def __setstate__(self, state): + pass + + def __getnewargs__(self): + return () + + def __nonzero__(self): + return False + + +def user_config_files(): + """Return path to any existing user config files + """ + return filter(os.path.exists, + map(os.path.expanduser, config_files)) + + +def all_config_files(): + """Return path to any existing user config files, plus any setup.cfg + in the current working directory. + """ + user = user_config_files() + if os.path.exists('setup.cfg'): + return user + ['setup.cfg'] + return user + + +# used when parsing config files +def flag(val): + """Does the value look like an on/off flag?""" + if val == 1: + return True + elif val == 0: + return False + val = str(val) + if len(val) > 5: + return False + return val.upper() in ('1', '0', 'F', 'T', 'TRUE', 'FALSE', 'ON', 'OFF') + + +def _bool(val): + return str(val).upper() in ('1', 'T', 'TRUE', 'ON') diff --git a/lib/spack/external/nose/core.py b/lib/spack/external/nose/core.py new file mode 100644 index 00000000000..49e7939b986 --- /dev/null +++ b/lib/spack/external/nose/core.py @@ -0,0 +1,341 @@ +"""Implements nose test program and collector. +""" +from __future__ import generators + +import logging +import os +import sys +import time +import unittest + +from nose.config import Config, all_config_files +from nose.loader import defaultTestLoader +from nose.plugins.manager import PluginManager, DefaultPluginManager, \ + RestrictedPluginManager +from nose.result import TextTestResult +from nose.suite import FinalizingSuiteWrapper +from nose.util import isclass, tolist + + +log = logging.getLogger('nose.core') +compat_24 = sys.version_info >= (2, 4) + +__all__ = ['TestProgram', 'main', 'run', 'run_exit', 'runmodule', 'collector', + 'TextTestRunner'] + + +class TextTestRunner(unittest.TextTestRunner): + """Test runner that uses nose's TextTestResult to enable errorClasses, + as well as providing hooks for plugins to override or replace the test + output stream, results, and the test case itself. + """ + def __init__(self, stream=sys.stderr, descriptions=1, verbosity=1, + config=None): + if config is None: + config = Config() + self.config = config + unittest.TextTestRunner.__init__(self, stream, descriptions, verbosity) + + + def _makeResult(self): + return TextTestResult(self.stream, + self.descriptions, + self.verbosity, + self.config) + + def run(self, test): + """Overrides to provide plugin hooks and defer all output to + the test result class. + """ + wrapper = self.config.plugins.prepareTest(test) + if wrapper is not None: + test = wrapper + + # plugins can decorate or capture the output stream + wrapped = self.config.plugins.setOutputStream(self.stream) + if wrapped is not None: + self.stream = wrapped + + result = self._makeResult() + start = time.time() + try: + test(result) + except KeyboardInterrupt: + pass + stop = time.time() + result.printErrors() + result.printSummary(start, stop) + self.config.plugins.finalize(result) + return result + + +class TestProgram(unittest.TestProgram): + """Collect and run tests, returning success or failure. + + The arguments to TestProgram() are the same as to + :func:`main()` and :func:`run()`: + + * module: All tests are in this module (default: None) + * defaultTest: Tests to load (default: '.') + * argv: Command line arguments (default: None; sys.argv is read) + * testRunner: Test runner instance (default: None) + * testLoader: Test loader instance (default: None) + * env: Environment; ignored if config is provided (default: None; + os.environ is read) + * config: :class:`nose.config.Config` instance (default: None) + * suite: Suite or list of tests to run (default: None). Passing a + suite or lists of tests will bypass all test discovery and + loading. *ALSO NOTE* that if you pass a unittest.TestSuite + instance as the suite, context fixtures at the class, module and + package level will not be used, and many plugin hooks will not + be called. If you want normal nose behavior, either pass a list + of tests, or a fully-configured :class:`nose.suite.ContextSuite`. + * exit: Exit after running tests and printing report (default: True) + * plugins: List of plugins to use; ignored if config is provided + (default: load plugins with DefaultPluginManager) + * addplugins: List of **extra** plugins to use. Pass a list of plugin + instances in this argument to make custom plugins available while + still using the DefaultPluginManager. + """ + verbosity = 1 + + def __init__(self, module=None, defaultTest='.', argv=None, + testRunner=None, testLoader=None, env=None, config=None, + suite=None, exit=True, plugins=None, addplugins=None): + if env is None: + env = os.environ + if config is None: + config = self.makeConfig(env, plugins) + if addplugins: + config.plugins.addPlugins(extraplugins=addplugins) + self.config = config + self.suite = suite + self.exit = exit + extra_args = {} + version = sys.version_info[0:2] + if version >= (2,7) and version != (3,0): + extra_args['exit'] = exit + unittest.TestProgram.__init__( + self, module=module, defaultTest=defaultTest, + argv=argv, testRunner=testRunner, testLoader=testLoader, + **extra_args) + + def getAllConfigFiles(self, env=None): + env = env or {} + if env.get('NOSE_IGNORE_CONFIG_FILES', False): + return [] + else: + return all_config_files() + + def makeConfig(self, env, plugins=None): + """Load a Config, pre-filled with user config files if any are + found. + """ + cfg_files = self.getAllConfigFiles(env) + if plugins: + manager = PluginManager(plugins=plugins) + else: + manager = DefaultPluginManager() + return Config( + env=env, files=cfg_files, plugins=manager) + + def parseArgs(self, argv): + """Parse argv and env and configure running environment. + """ + self.config.configure(argv, doc=self.usage()) + log.debug("configured %s", self.config) + + # quick outs: version, plugins (optparse would have already + # caught and exited on help) + if self.config.options.version: + from nose import __version__ + sys.stdout = sys.__stdout__ + print "%s version %s" % (os.path.basename(sys.argv[0]), __version__) + sys.exit(0) + + if self.config.options.showPlugins: + self.showPlugins() + sys.exit(0) + + if self.testLoader is None: + self.testLoader = defaultTestLoader(config=self.config) + elif isclass(self.testLoader): + self.testLoader = self.testLoader(config=self.config) + plug_loader = self.config.plugins.prepareTestLoader(self.testLoader) + if plug_loader is not None: + self.testLoader = plug_loader + log.debug("test loader is %s", self.testLoader) + + # FIXME if self.module is a string, add it to self.testNames? not sure + + if self.config.testNames: + self.testNames = self.config.testNames + else: + self.testNames = tolist(self.defaultTest) + log.debug('defaultTest %s', self.defaultTest) + log.debug('Test names are %s', self.testNames) + if self.config.workingDir is not None: + os.chdir(self.config.workingDir) + self.createTests() + + def createTests(self): + """Create the tests to run. If a self.suite + is set, then that suite will be used. Otherwise, tests will be + loaded from the given test names (self.testNames) using the + test loader. + """ + log.debug("createTests called with %s", self.suite) + if self.suite is not None: + # We were given an explicit suite to run. Make sure it's + # loaded and wrapped correctly. + self.test = self.testLoader.suiteClass(self.suite) + else: + self.test = self.testLoader.loadTestsFromNames(self.testNames) + + def runTests(self): + """Run Tests. Returns true on success, false on failure, and sets + self.success to the same value. + """ + log.debug("runTests called") + if self.testRunner is None: + self.testRunner = TextTestRunner(stream=self.config.stream, + verbosity=self.config.verbosity, + config=self.config) + plug_runner = self.config.plugins.prepareTestRunner(self.testRunner) + if plug_runner is not None: + self.testRunner = plug_runner + result = self.testRunner.run(self.test) + self.success = result.wasSuccessful() + if self.exit: + sys.exit(not self.success) + return self.success + + def showPlugins(self): + """Print list of available plugins. + """ + import textwrap + + class DummyParser: + def __init__(self): + self.options = [] + def add_option(self, *arg, **kw): + self.options.append((arg, kw.pop('help', ''))) + + v = self.config.verbosity + self.config.plugins.sort() + for p in self.config.plugins: + print "Plugin %s" % p.name + if v >= 2: + print " score: %s" % p.score + print '\n'.join(textwrap.wrap(p.help().strip(), + initial_indent=' ', + subsequent_indent=' ')) + if v >= 3: + parser = DummyParser() + p.addOptions(parser) + if len(parser.options): + print + print " Options:" + for opts, help in parser.options: + print ' %s' % (', '.join(opts)) + if help: + print '\n'.join( + textwrap.wrap(help.strip(), + initial_indent=' ', + subsequent_indent=' ')) + print + + def usage(cls): + import nose + try: + ld = nose.__loader__ + text = ld.get_data(os.path.join( + os.path.dirname(__file__), 'usage.txt')) + except AttributeError: + f = open(os.path.join( + os.path.dirname(__file__), 'usage.txt'), 'r') + try: + text = f.read() + finally: + f.close() + # Ensure that we return str, not bytes. + if not isinstance(text, str): + text = text.decode('utf-8') + return text + usage = classmethod(usage) + +# backwards compatibility +run_exit = main = TestProgram + + +def run(*arg, **kw): + """Collect and run tests, returning success or failure. + + The arguments to `run()` are the same as to `main()`: + + * module: All tests are in this module (default: None) + * defaultTest: Tests to load (default: '.') + * argv: Command line arguments (default: None; sys.argv is read) + * testRunner: Test runner instance (default: None) + * testLoader: Test loader instance (default: None) + * env: Environment; ignored if config is provided (default: None; + os.environ is read) + * config: :class:`nose.config.Config` instance (default: None) + * suite: Suite or list of tests to run (default: None). Passing a + suite or lists of tests will bypass all test discovery and + loading. *ALSO NOTE* that if you pass a unittest.TestSuite + instance as the suite, context fixtures at the class, module and + package level will not be used, and many plugin hooks will not + be called. If you want normal nose behavior, either pass a list + of tests, or a fully-configured :class:`nose.suite.ContextSuite`. + * plugins: List of plugins to use; ignored if config is provided + (default: load plugins with DefaultPluginManager) + * addplugins: List of **extra** plugins to use. Pass a list of plugin + instances in this argument to make custom plugins available while + still using the DefaultPluginManager. + + With the exception that the ``exit`` argument is always set + to False. + """ + kw['exit'] = False + return TestProgram(*arg, **kw).success + + +def runmodule(name='__main__', **kw): + """Collect and run tests in a single module only. Defaults to running + tests in __main__. Additional arguments to TestProgram may be passed + as keyword arguments. + """ + main(defaultTest=name, **kw) + + +def collector(): + """TestSuite replacement entry point. Use anywhere you might use a + unittest.TestSuite. The collector will, by default, load options from + all config files and execute loader.loadTestsFromNames() on the + configured testNames, or '.' if no testNames are configured. + """ + # plugins that implement any of these methods are disabled, since + # we don't control the test runner and won't be able to run them + # finalize() is also not called, but plugins that use it aren't disabled, + # because capture needs it. + setuptools_incompat = ('report', 'prepareTest', + 'prepareTestLoader', 'prepareTestRunner', + 'setOutputStream') + + plugins = RestrictedPluginManager(exclude=setuptools_incompat) + conf = Config(files=all_config_files(), + plugins=plugins) + conf.configure(argv=['collector']) + loader = defaultTestLoader(conf) + + if conf.testNames: + suite = loader.loadTestsFromNames(conf.testNames) + else: + suite = loader.loadTestsFromNames(('.',)) + return FinalizingSuiteWrapper(suite, plugins.finalize) + + + +if __name__ == '__main__': + main() diff --git a/lib/spack/external/nose/exc.py b/lib/spack/external/nose/exc.py new file mode 100644 index 00000000000..8b780db0d42 --- /dev/null +++ b/lib/spack/external/nose/exc.py @@ -0,0 +1,9 @@ +"""Exceptions for marking tests as skipped or deprecated. + +This module exists to provide backwards compatibility with previous +versions of nose where skipped and deprecated tests were core +functionality, rather than being provided by plugins. It may be +removed in a future release. +""" +from nose.plugins.skip import SkipTest +from nose.plugins.deprecated import DeprecatedTest diff --git a/lib/spack/external/nose/ext/__init__.py b/lib/spack/external/nose/ext/__init__.py new file mode 100644 index 00000000000..5fd1516a09b --- /dev/null +++ b/lib/spack/external/nose/ext/__init__.py @@ -0,0 +1,3 @@ +""" +External or vendor files +""" diff --git a/lib/spack/external/nose/ext/dtcompat.py b/lib/spack/external/nose/ext/dtcompat.py new file mode 100644 index 00000000000..332cf08c129 --- /dev/null +++ b/lib/spack/external/nose/ext/dtcompat.py @@ -0,0 +1,2272 @@ +# Module doctest. +# Released to the public domain 16-Jan-2001, by Tim Peters (tim@python.org). +# Major enhancements and refactoring by: +# Jim Fulton +# Edward Loper + +# Provided as-is; use at your own risk; no warranty; no promises; enjoy! +# +# Modified for inclusion in nose to provide support for DocFileTest in +# python 2.3: +# +# - all doctests removed from module (they fail under 2.3 and 2.5) +# - now handles the $py.class extension when ran under Jython + +r"""Module doctest -- a framework for running examples in docstrings. + +In simplest use, end each module M to be tested with: + +def _test(): + import doctest + doctest.testmod() + +if __name__ == "__main__": + _test() + +Then running the module as a script will cause the examples in the +docstrings to get executed and verified: + +python M.py + +This won't display anything unless an example fails, in which case the +failing example(s) and the cause(s) of the failure(s) are printed to stdout +(why not stderr? because stderr is a lame hack <0.2 wink>), and the final +line of output is "Test failed.". + +Run it with the -v switch instead: + +python M.py -v + +and a detailed report of all examples tried is printed to stdout, along +with assorted summaries at the end. + +You can force verbose mode by passing "verbose=True" to testmod, or prohibit +it by passing "verbose=False". In either of those cases, sys.argv is not +examined by testmod. + +There are a variety of other ways to run doctests, including integration +with the unittest framework, and support for running non-Python text +files containing doctests. There are also many ways to override parts +of doctest's default behaviors. See the Library Reference Manual for +details. +""" + +__docformat__ = 'reStructuredText en' + +__all__ = [ + # 0, Option Flags + 'register_optionflag', + 'DONT_ACCEPT_TRUE_FOR_1', + 'DONT_ACCEPT_BLANKLINE', + 'NORMALIZE_WHITESPACE', + 'ELLIPSIS', + 'IGNORE_EXCEPTION_DETAIL', + 'COMPARISON_FLAGS', + 'REPORT_UDIFF', + 'REPORT_CDIFF', + 'REPORT_NDIFF', + 'REPORT_ONLY_FIRST_FAILURE', + 'REPORTING_FLAGS', + # 1. Utility Functions + 'is_private', + # 2. Example & DocTest + 'Example', + 'DocTest', + # 3. Doctest Parser + 'DocTestParser', + # 4. Doctest Finder + 'DocTestFinder', + # 5. Doctest Runner + 'DocTestRunner', + 'OutputChecker', + 'DocTestFailure', + 'UnexpectedException', + 'DebugRunner', + # 6. Test Functions + 'testmod', + 'testfile', + 'run_docstring_examples', + # 7. Tester + 'Tester', + # 8. Unittest Support + 'DocTestSuite', + 'DocFileSuite', + 'set_unittest_reportflags', + # 9. Debugging Support + 'script_from_examples', + 'testsource', + 'debug_src', + 'debug', +] + +import __future__ + +import sys, traceback, inspect, linecache, os, re +import unittest, difflib, pdb, tempfile +import warnings +from StringIO import StringIO + +# Don't whine about the deprecated is_private function in this +# module's tests. +warnings.filterwarnings("ignore", "is_private", DeprecationWarning, + __name__, 0) + +# There are 4 basic classes: +# - Example: a pair, plus an intra-docstring line number. +# - DocTest: a collection of examples, parsed from a docstring, plus +# info about where the docstring came from (name, filename, lineno). +# - DocTestFinder: extracts DocTests from a given object's docstring and +# its contained objects' docstrings. +# - DocTestRunner: runs DocTest cases, and accumulates statistics. +# +# So the basic picture is: +# +# list of: +# +------+ +---------+ +-------+ +# |object| --DocTestFinder-> | DocTest | --DocTestRunner-> |results| +# +------+ +---------+ +-------+ +# | Example | +# | ... | +# | Example | +# +---------+ + +# Option constants. + +OPTIONFLAGS_BY_NAME = {} +def register_optionflag(name): + # Create a new flag unless `name` is already known. + return OPTIONFLAGS_BY_NAME.setdefault(name, 1 << len(OPTIONFLAGS_BY_NAME)) + +DONT_ACCEPT_TRUE_FOR_1 = register_optionflag('DONT_ACCEPT_TRUE_FOR_1') +DONT_ACCEPT_BLANKLINE = register_optionflag('DONT_ACCEPT_BLANKLINE') +NORMALIZE_WHITESPACE = register_optionflag('NORMALIZE_WHITESPACE') +ELLIPSIS = register_optionflag('ELLIPSIS') +IGNORE_EXCEPTION_DETAIL = register_optionflag('IGNORE_EXCEPTION_DETAIL') + +COMPARISON_FLAGS = (DONT_ACCEPT_TRUE_FOR_1 | + DONT_ACCEPT_BLANKLINE | + NORMALIZE_WHITESPACE | + ELLIPSIS | + IGNORE_EXCEPTION_DETAIL) + +REPORT_UDIFF = register_optionflag('REPORT_UDIFF') +REPORT_CDIFF = register_optionflag('REPORT_CDIFF') +REPORT_NDIFF = register_optionflag('REPORT_NDIFF') +REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE') + +REPORTING_FLAGS = (REPORT_UDIFF | + REPORT_CDIFF | + REPORT_NDIFF | + REPORT_ONLY_FIRST_FAILURE) + +# Special string markers for use in `want` strings: +BLANKLINE_MARKER = '' +ELLIPSIS_MARKER = '...' + +###################################################################### +## Table of Contents +###################################################################### +# 1. Utility Functions +# 2. Example & DocTest -- store test cases +# 3. DocTest Parser -- extracts examples from strings +# 4. DocTest Finder -- extracts test cases from objects +# 5. DocTest Runner -- runs test cases +# 6. Test Functions -- convenient wrappers for testing +# 7. Tester Class -- for backwards compatibility +# 8. Unittest Support +# 9. Debugging Support +# 10. Example Usage + +###################################################################### +## 1. Utility Functions +###################################################################### + +def is_private(prefix, base): + """prefix, base -> true iff name prefix + "." + base is "private". + + Prefix may be an empty string, and base does not contain a period. + Prefix is ignored (although functions you write conforming to this + protocol may make use of it). + Return true iff base begins with an (at least one) underscore, but + does not both begin and end with (at least) two underscores. + """ + warnings.warn("is_private is deprecated; it wasn't useful; " + "examine DocTestFinder.find() lists instead", + DeprecationWarning, stacklevel=2) + return base[:1] == "_" and not base[:2] == "__" == base[-2:] + +def _extract_future_flags(globs): + """ + Return the compiler-flags associated with the future features that + have been imported into the given namespace (globs). + """ + flags = 0 + for fname in __future__.all_feature_names: + feature = globs.get(fname, None) + if feature is getattr(__future__, fname): + flags |= feature.compiler_flag + return flags + +def _normalize_module(module, depth=2): + """ + Return the module specified by `module`. In particular: + - If `module` is a module, then return module. + - If `module` is a string, then import and return the + module with that name. + - If `module` is None, then return the calling module. + The calling module is assumed to be the module of + the stack frame at the given depth in the call stack. + """ + if inspect.ismodule(module): + return module + elif isinstance(module, (str, unicode)): + return __import__(module, globals(), locals(), ["*"]) + elif module is None: + return sys.modules[sys._getframe(depth).f_globals['__name__']] + else: + raise TypeError("Expected a module, string, or None") + +def _indent(s, indent=4): + """ + Add the given number of space characters to the beginning every + non-blank line in `s`, and return the result. + """ + # This regexp matches the start of non-blank lines: + return re.sub('(?m)^(?!$)', indent*' ', s) + +def _exception_traceback(exc_info): + """ + Return a string containing a traceback message for the given + exc_info tuple (as returned by sys.exc_info()). + """ + # Get a traceback message. + excout = StringIO() + exc_type, exc_val, exc_tb = exc_info + traceback.print_exception(exc_type, exc_val, exc_tb, file=excout) + return excout.getvalue() + +# Override some StringIO methods. +class _SpoofOut(StringIO): + def getvalue(self): + result = StringIO.getvalue(self) + # If anything at all was written, make sure there's a trailing + # newline. There's no way for the expected output to indicate + # that a trailing newline is missing. + if result and not result.endswith("\n"): + result += "\n" + # Prevent softspace from screwing up the next test case, in + # case they used print with a trailing comma in an example. + if hasattr(self, "softspace"): + del self.softspace + return result + + def truncate(self, size=None): + StringIO.truncate(self, size) + if hasattr(self, "softspace"): + del self.softspace + +# Worst-case linear-time ellipsis matching. +def _ellipsis_match(want, got): + if ELLIPSIS_MARKER not in want: + return want == got + + # Find "the real" strings. + ws = want.split(ELLIPSIS_MARKER) + assert len(ws) >= 2 + + # Deal with exact matches possibly needed at one or both ends. + startpos, endpos = 0, len(got) + w = ws[0] + if w: # starts with exact match + if got.startswith(w): + startpos = len(w) + del ws[0] + else: + return False + w = ws[-1] + if w: # ends with exact match + if got.endswith(w): + endpos -= len(w) + del ws[-1] + else: + return False + + if startpos > endpos: + # Exact end matches required more characters than we have, as in + # _ellipsis_match('aa...aa', 'aaa') + return False + + # For the rest, we only need to find the leftmost non-overlapping + # match for each piece. If there's no overall match that way alone, + # there's no overall match period. + for w in ws: + # w may be '' at times, if there are consecutive ellipses, or + # due to an ellipsis at the start or end of `want`. That's OK. + # Search for an empty string succeeds, and doesn't change startpos. + startpos = got.find(w, startpos, endpos) + if startpos < 0: + return False + startpos += len(w) + + return True + +def _comment_line(line): + "Return a commented form of the given line" + line = line.rstrip() + if line: + return '# '+line + else: + return '#' + +class _OutputRedirectingPdb(pdb.Pdb): + """ + A specialized version of the python debugger that redirects stdout + to a given stream when interacting with the user. Stdout is *not* + redirected when traced code is executed. + """ + def __init__(self, out): + self.__out = out + pdb.Pdb.__init__(self) + + def trace_dispatch(self, *args): + # Redirect stdout to the given stream. + save_stdout = sys.stdout + sys.stdout = self.__out + # Call Pdb's trace dispatch method. + try: + return pdb.Pdb.trace_dispatch(self, *args) + finally: + sys.stdout = save_stdout + +# [XX] Normalize with respect to os.path.pardir? +def _module_relative_path(module, path): + if not inspect.ismodule(module): + raise TypeError, 'Expected a module: %r' % module + if path.startswith('/'): + raise ValueError, 'Module-relative files may not have absolute paths' + + # Find the base directory for the path. + if hasattr(module, '__file__'): + # A normal module/package + basedir = os.path.split(module.__file__)[0] + elif module.__name__ == '__main__': + # An interactive session. + if len(sys.argv)>0 and sys.argv[0] != '': + basedir = os.path.split(sys.argv[0])[0] + else: + basedir = os.curdir + else: + # A module w/o __file__ (this includes builtins) + raise ValueError("Can't resolve paths relative to the module " + + module + " (it has no __file__)") + + # Combine the base directory and the path. + return os.path.join(basedir, *(path.split('/'))) + +###################################################################### +## 2. Example & DocTest +###################################################################### +## - An "example" is a pair, where "source" is a +## fragment of source code, and "want" is the expected output for +## "source." The Example class also includes information about +## where the example was extracted from. +## +## - A "doctest" is a collection of examples, typically extracted from +## a string (such as an object's docstring). The DocTest class also +## includes information about where the string was extracted from. + +class Example: + """ + A single doctest example, consisting of source code and expected + output. `Example` defines the following attributes: + + - source: A single Python statement, always ending with a newline. + The constructor adds a newline if needed. + + - want: The expected output from running the source code (either + from stdout, or a traceback in case of exception). `want` ends + with a newline unless it's empty, in which case it's an empty + string. The constructor adds a newline if needed. + + - exc_msg: The exception message generated by the example, if + the example is expected to generate an exception; or `None` if + it is not expected to generate an exception. This exception + message is compared against the return value of + `traceback.format_exception_only()`. `exc_msg` ends with a + newline unless it's `None`. The constructor adds a newline + if needed. + + - lineno: The line number within the DocTest string containing + this Example where the Example begins. This line number is + zero-based, with respect to the beginning of the DocTest. + + - indent: The example's indentation in the DocTest string. + I.e., the number of space characters that preceed the + example's first prompt. + + - options: A dictionary mapping from option flags to True or + False, which is used to override default options for this + example. Any option flags not contained in this dictionary + are left at their default value (as specified by the + DocTestRunner's optionflags). By default, no options are set. + """ + def __init__(self, source, want, exc_msg=None, lineno=0, indent=0, + options=None): + # Normalize inputs. + if not source.endswith('\n'): + source += '\n' + if want and not want.endswith('\n'): + want += '\n' + if exc_msg is not None and not exc_msg.endswith('\n'): + exc_msg += '\n' + # Store properties. + self.source = source + self.want = want + self.lineno = lineno + self.indent = indent + if options is None: options = {} + self.options = options + self.exc_msg = exc_msg + +class DocTest: + """ + A collection of doctest examples that should be run in a single + namespace. Each `DocTest` defines the following attributes: + + - examples: the list of examples. + + - globs: The namespace (aka globals) that the examples should + be run in. + + - name: A name identifying the DocTest (typically, the name of + the object whose docstring this DocTest was extracted from). + + - filename: The name of the file that this DocTest was extracted + from, or `None` if the filename is unknown. + + - lineno: The line number within filename where this DocTest + begins, or `None` if the line number is unavailable. This + line number is zero-based, with respect to the beginning of + the file. + + - docstring: The string that the examples were extracted from, + or `None` if the string is unavailable. + """ + def __init__(self, examples, globs, name, filename, lineno, docstring): + """ + Create a new DocTest containing the given examples. The + DocTest's globals are initialized with a copy of `globs`. + """ + assert not isinstance(examples, basestring), \ + "DocTest no longer accepts str; use DocTestParser instead" + self.examples = examples + self.docstring = docstring + self.globs = globs.copy() + self.name = name + self.filename = filename + self.lineno = lineno + + def __repr__(self): + if len(self.examples) == 0: + examples = 'no examples' + elif len(self.examples) == 1: + examples = '1 example' + else: + examples = '%d examples' % len(self.examples) + return ('' % + (self.name, self.filename, self.lineno, examples)) + + + # This lets us sort tests by name: + def __cmp__(self, other): + if not isinstance(other, DocTest): + return -1 + return cmp((self.name, self.filename, self.lineno, id(self)), + (other.name, other.filename, other.lineno, id(other))) + +###################################################################### +## 3. DocTestParser +###################################################################### + +class DocTestParser: + """ + A class used to parse strings containing doctest examples. + """ + # This regular expression is used to find doctest examples in a + # string. It defines three groups: `source` is the source code + # (including leading indentation and prompts); `indent` is the + # indentation of the first (PS1) line of the source code; and + # `want` is the expected output (including leading indentation). + _EXAMPLE_RE = re.compile(r''' + # Source consists of a PS1 line followed by zero or more PS2 lines. + (?P + (?:^(?P [ ]*) >>> .*) # PS1 line + (?:\n [ ]* \.\.\. .*)*) # PS2 lines + \n? + # Want consists of any non-blank lines that do not start with PS1. + (?P (?:(?![ ]*$) # Not a blank line + (?![ ]*>>>) # Not a line starting with PS1 + .*$\n? # But any other line + )*) + ''', re.MULTILINE | re.VERBOSE) + + # A regular expression for handling `want` strings that contain + # expected exceptions. It divides `want` into three pieces: + # - the traceback header line (`hdr`) + # - the traceback stack (`stack`) + # - the exception message (`msg`), as generated by + # traceback.format_exception_only() + # `msg` may have multiple lines. We assume/require that the + # exception message is the first non-indented line starting with a word + # character following the traceback header line. + _EXCEPTION_RE = re.compile(r""" + # Grab the traceback header. Different versions of Python have + # said different things on the first traceback line. + ^(?P Traceback\ \( + (?: most\ recent\ call\ last + | innermost\ last + ) \) : + ) + \s* $ # toss trailing whitespace on the header. + (?P .*?) # don't blink: absorb stuff until... + ^ (?P \w+ .*) # a line *starts* with alphanum. + """, re.VERBOSE | re.MULTILINE | re.DOTALL) + + # A callable returning a true value iff its argument is a blank line + # or contains a single comment. + _IS_BLANK_OR_COMMENT = re.compile(r'^[ ]*(#.*)?$').match + + def parse(self, string, name=''): + """ + Divide the given string into examples and intervening text, + and return them as a list of alternating Examples and strings. + Line numbers for the Examples are 0-based. The optional + argument `name` is a name identifying this string, and is only + used for error messages. + """ + string = string.expandtabs() + # If all lines begin with the same indentation, then strip it. + min_indent = self._min_indent(string) + if min_indent > 0: + string = '\n'.join([l[min_indent:] for l in string.split('\n')]) + + output = [] + charno, lineno = 0, 0 + # Find all doctest examples in the string: + for m in self._EXAMPLE_RE.finditer(string): + # Add the pre-example text to `output`. + output.append(string[charno:m.start()]) + # Update lineno (lines before this example) + lineno += string.count('\n', charno, m.start()) + # Extract info from the regexp match. + (source, options, want, exc_msg) = \ + self._parse_example(m, name, lineno) + # Create an Example, and add it to the list. + if not self._IS_BLANK_OR_COMMENT(source): + output.append( Example(source, want, exc_msg, + lineno=lineno, + indent=min_indent+len(m.group('indent')), + options=options) ) + # Update lineno (lines inside this example) + lineno += string.count('\n', m.start(), m.end()) + # Update charno. + charno = m.end() + # Add any remaining post-example text to `output`. + output.append(string[charno:]) + return output + + def get_doctest(self, string, globs, name, filename, lineno): + """ + Extract all doctest examples from the given string, and + collect them into a `DocTest` object. + + `globs`, `name`, `filename`, and `lineno` are attributes for + the new `DocTest` object. See the documentation for `DocTest` + for more information. + """ + return DocTest(self.get_examples(string, name), globs, + name, filename, lineno, string) + + def get_examples(self, string, name=''): + """ + Extract all doctest examples from the given string, and return + them as a list of `Example` objects. Line numbers are + 0-based, because it's most common in doctests that nothing + interesting appears on the same line as opening triple-quote, + and so the first interesting line is called \"line 1\" then. + + The optional argument `name` is a name identifying this + string, and is only used for error messages. + """ + return [x for x in self.parse(string, name) + if isinstance(x, Example)] + + def _parse_example(self, m, name, lineno): + """ + Given a regular expression match from `_EXAMPLE_RE` (`m`), + return a pair `(source, want)`, where `source` is the matched + example's source code (with prompts and indentation stripped); + and `want` is the example's expected output (with indentation + stripped). + + `name` is the string's name, and `lineno` is the line number + where the example starts; both are used for error messages. + """ + # Get the example's indentation level. + indent = len(m.group('indent')) + + # Divide source into lines; check that they're properly + # indented; and then strip their indentation & prompts. + source_lines = m.group('source').split('\n') + self._check_prompt_blank(source_lines, indent, name, lineno) + self._check_prefix(source_lines[1:], ' '*indent + '.', name, lineno) + source = '\n'.join([sl[indent+4:] for sl in source_lines]) + + # Divide want into lines; check that it's properly indented; and + # then strip the indentation. Spaces before the last newline should + # be preserved, so plain rstrip() isn't good enough. + want = m.group('want') + want_lines = want.split('\n') + if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]): + del want_lines[-1] # forget final newline & spaces after it + self._check_prefix(want_lines, ' '*indent, name, + lineno + len(source_lines)) + want = '\n'.join([wl[indent:] for wl in want_lines]) + + # If `want` contains a traceback message, then extract it. + m = self._EXCEPTION_RE.match(want) + if m: + exc_msg = m.group('msg') + else: + exc_msg = None + + # Extract options from the source. + options = self._find_options(source, name, lineno) + + return source, options, want, exc_msg + + # This regular expression looks for option directives in the + # source code of an example. Option directives are comments + # starting with "doctest:". Warning: this may give false + # positives for string-literals that contain the string + # "#doctest:". Eliminating these false positives would require + # actually parsing the string; but we limit them by ignoring any + # line containing "#doctest:" that is *followed* by a quote mark. + _OPTION_DIRECTIVE_RE = re.compile(r'#\s*doctest:\s*([^\n\'"]*)$', + re.MULTILINE) + + def _find_options(self, source, name, lineno): + """ + Return a dictionary containing option overrides extracted from + option directives in the given source string. + + `name` is the string's name, and `lineno` is the line number + where the example starts; both are used for error messages. + """ + options = {} + # (note: with the current regexp, this will match at most once:) + for m in self._OPTION_DIRECTIVE_RE.finditer(source): + option_strings = m.group(1).replace(',', ' ').split() + for option in option_strings: + if (option[0] not in '+-' or + option[1:] not in OPTIONFLAGS_BY_NAME): + raise ValueError('line %r of the doctest for %s ' + 'has an invalid option: %r' % + (lineno+1, name, option)) + flag = OPTIONFLAGS_BY_NAME[option[1:]] + options[flag] = (option[0] == '+') + if options and self._IS_BLANK_OR_COMMENT(source): + raise ValueError('line %r of the doctest for %s has an option ' + 'directive on a line with no example: %r' % + (lineno, name, source)) + return options + + # This regular expression finds the indentation of every non-blank + # line in a string. + _INDENT_RE = re.compile('^([ ]*)(?=\S)', re.MULTILINE) + + def _min_indent(self, s): + "Return the minimum indentation of any non-blank line in `s`" + indents = [len(indent) for indent in self._INDENT_RE.findall(s)] + if len(indents) > 0: + return min(indents) + else: + return 0 + + def _check_prompt_blank(self, lines, indent, name, lineno): + """ + Given the lines of a source string (including prompts and + leading indentation), check to make sure that every prompt is + followed by a space character. If any line is not followed by + a space character, then raise ValueError. + """ + for i, line in enumerate(lines): + if len(line) >= indent+4 and line[indent+3] != ' ': + raise ValueError('line %r of the docstring for %s ' + 'lacks blank after %s: %r' % + (lineno+i+1, name, + line[indent:indent+3], line)) + + def _check_prefix(self, lines, prefix, name, lineno): + """ + Check that every line in the given list starts with the given + prefix; if any line does not, then raise a ValueError. + """ + for i, line in enumerate(lines): + if line and not line.startswith(prefix): + raise ValueError('line %r of the docstring for %s has ' + 'inconsistent leading whitespace: %r' % + (lineno+i+1, name, line)) + + +###################################################################### +## 4. DocTest Finder +###################################################################### + +class DocTestFinder: + """ + A class used to extract the DocTests that are relevant to a given + object, from its docstring and the docstrings of its contained + objects. Doctests can currently be extracted from the following + object types: modules, functions, classes, methods, staticmethods, + classmethods, and properties. + """ + + def __init__(self, verbose=False, parser=DocTestParser(), + recurse=True, _namefilter=None, exclude_empty=True): + """ + Create a new doctest finder. + + The optional argument `parser` specifies a class or + function that should be used to create new DocTest objects (or + objects that implement the same interface as DocTest). The + signature for this factory function should match the signature + of the DocTest constructor. + + If the optional argument `recurse` is false, then `find` will + only examine the given object, and not any contained objects. + + If the optional argument `exclude_empty` is false, then `find` + will include tests for objects with empty docstrings. + """ + self._parser = parser + self._verbose = verbose + self._recurse = recurse + self._exclude_empty = exclude_empty + # _namefilter is undocumented, and exists only for temporary backward- + # compatibility support of testmod's deprecated isprivate mess. + self._namefilter = _namefilter + + def find(self, obj, name=None, module=None, globs=None, + extraglobs=None): + """ + Return a list of the DocTests that are defined by the given + object's docstring, or by any of its contained objects' + docstrings. + + The optional parameter `module` is the module that contains + the given object. If the module is not specified or is None, then + the test finder will attempt to automatically determine the + correct module. The object's module is used: + + - As a default namespace, if `globs` is not specified. + - To prevent the DocTestFinder from extracting DocTests + from objects that are imported from other modules. + - To find the name of the file containing the object. + - To help find the line number of the object within its + file. + + Contained objects whose module does not match `module` are ignored. + + If `module` is False, no attempt to find the module will be made. + This is obscure, of use mostly in tests: if `module` is False, or + is None but cannot be found automatically, then all objects are + considered to belong to the (non-existent) module, so all contained + objects will (recursively) be searched for doctests. + + The globals for each DocTest is formed by combining `globs` + and `extraglobs` (bindings in `extraglobs` override bindings + in `globs`). A new copy of the globals dictionary is created + for each DocTest. If `globs` is not specified, then it + defaults to the module's `__dict__`, if specified, or {} + otherwise. If `extraglobs` is not specified, then it defaults + to {}. + + """ + # If name was not specified, then extract it from the object. + if name is None: + name = getattr(obj, '__name__', None) + if name is None: + raise ValueError("DocTestFinder.find: name must be given " + "when obj.__name__ doesn't exist: %r" % + (type(obj),)) + + # Find the module that contains the given object (if obj is + # a module, then module=obj.). Note: this may fail, in which + # case module will be None. + if module is False: + module = None + elif module is None: + module = inspect.getmodule(obj) + + # Read the module's source code. This is used by + # DocTestFinder._find_lineno to find the line number for a + # given object's docstring. + try: + file = inspect.getsourcefile(obj) or inspect.getfile(obj) + source_lines = linecache.getlines(file) + if not source_lines: + source_lines = None + except TypeError: + source_lines = None + + # Initialize globals, and merge in extraglobs. + if globs is None: + if module is None: + globs = {} + else: + globs = module.__dict__.copy() + else: + globs = globs.copy() + if extraglobs is not None: + globs.update(extraglobs) + + # Recursively expore `obj`, extracting DocTests. + tests = [] + self._find(tests, obj, name, module, source_lines, globs, {}) + # Sort the tests by alpha order of names, for consistency in + # verbose-mode output. This was a feature of doctest in Pythons + # <= 2.3 that got lost by accident in 2.4. It was repaired in + # 2.4.4 and 2.5. + tests.sort() + return tests + + def _filter(self, obj, prefix, base): + """ + Return true if the given object should not be examined. + """ + return (self._namefilter is not None and + self._namefilter(prefix, base)) + + def _from_module(self, module, object): + """ + Return true if the given object is defined in the given + module. + """ + if module is None: + return True + elif inspect.isfunction(object): + return module.__dict__ is object.func_globals + elif inspect.isclass(object): + # Some jython classes don't set __module__ + return module.__name__ == getattr(object, '__module__', None) + elif inspect.getmodule(object) is not None: + return module is inspect.getmodule(object) + elif hasattr(object, '__module__'): + return module.__name__ == object.__module__ + elif isinstance(object, property): + return True # [XX] no way not be sure. + else: + raise ValueError("object must be a class or function") + + def _find(self, tests, obj, name, module, source_lines, globs, seen): + """ + Find tests for the given object and any contained objects, and + add them to `tests`. + """ + if self._verbose: + print 'Finding tests in %s' % name + + # If we've already processed this object, then ignore it. + if id(obj) in seen: + return + seen[id(obj)] = 1 + + # Find a test for this object, and add it to the list of tests. + test = self._get_test(obj, name, module, globs, source_lines) + if test is not None: + tests.append(test) + + # Look for tests in a module's contained objects. + if inspect.ismodule(obj) and self._recurse: + for valname, val in obj.__dict__.items(): + # Check if this contained object should be ignored. + if self._filter(val, name, valname): + continue + valname = '%s.%s' % (name, valname) + # Recurse to functions & classes. + if ((inspect.isfunction(val) or inspect.isclass(val)) and + self._from_module(module, val)): + self._find(tests, val, valname, module, source_lines, + globs, seen) + + # Look for tests in a module's __test__ dictionary. + if inspect.ismodule(obj) and self._recurse: + for valname, val in getattr(obj, '__test__', {}).items(): + if not isinstance(valname, basestring): + raise ValueError("DocTestFinder.find: __test__ keys " + "must be strings: %r" % + (type(valname),)) + if not (inspect.isfunction(val) or inspect.isclass(val) or + inspect.ismethod(val) or inspect.ismodule(val) or + isinstance(val, basestring)): + raise ValueError("DocTestFinder.find: __test__ values " + "must be strings, functions, methods, " + "classes, or modules: %r" % + (type(val),)) + valname = '%s.__test__.%s' % (name, valname) + self._find(tests, val, valname, module, source_lines, + globs, seen) + + # Look for tests in a class's contained objects. + if inspect.isclass(obj) and self._recurse: + for valname, val in obj.__dict__.items(): + # Check if this contained object should be ignored. + if self._filter(val, name, valname): + continue + # Special handling for staticmethod/classmethod. + if isinstance(val, staticmethod): + val = getattr(obj, valname) + if isinstance(val, classmethod): + val = getattr(obj, valname).im_func + + # Recurse to methods, properties, and nested classes. + if ((inspect.isfunction(val) or inspect.isclass(val) or + isinstance(val, property)) and + self._from_module(module, val)): + valname = '%s.%s' % (name, valname) + self._find(tests, val, valname, module, source_lines, + globs, seen) + + def _get_test(self, obj, name, module, globs, source_lines): + """ + Return a DocTest for the given object, if it defines a docstring; + otherwise, return None. + """ + # Extract the object's docstring. If it doesn't have one, + # then return None (no test for this object). + if isinstance(obj, basestring): + docstring = obj + else: + try: + if obj.__doc__ is None: + docstring = '' + else: + docstring = obj.__doc__ + if not isinstance(docstring, basestring): + docstring = str(docstring) + except (TypeError, AttributeError): + docstring = '' + + # Find the docstring's location in the file. + lineno = self._find_lineno(obj, source_lines) + + # Don't bother if the docstring is empty. + if self._exclude_empty and not docstring: + return None + + # Return a DocTest for this object. + if module is None: + filename = None + else: + filename = getattr(module, '__file__', module.__name__) + if filename[-4:] in (".pyc", ".pyo"): + filename = filename[:-1] + elif sys.platform.startswith('java') and \ + filename.endswith('$py.class'): + filename = '%s.py' % filename[:-9] + return self._parser.get_doctest(docstring, globs, name, + filename, lineno) + + def _find_lineno(self, obj, source_lines): + """ + Return a line number of the given object's docstring. Note: + this method assumes that the object has a docstring. + """ + lineno = None + + # Find the line number for modules. + if inspect.ismodule(obj): + lineno = 0 + + # Find the line number for classes. + # Note: this could be fooled if a class is defined multiple + # times in a single file. + if inspect.isclass(obj): + if source_lines is None: + return None + pat = re.compile(r'^\s*class\s*%s\b' % + getattr(obj, '__name__', '-')) + for i, line in enumerate(source_lines): + if pat.match(line): + lineno = i + break + + # Find the line number for functions & methods. + if inspect.ismethod(obj): obj = obj.im_func + if inspect.isfunction(obj): obj = obj.func_code + if inspect.istraceback(obj): obj = obj.tb_frame + if inspect.isframe(obj): obj = obj.f_code + if inspect.iscode(obj): + lineno = getattr(obj, 'co_firstlineno', None)-1 + + # Find the line number where the docstring starts. Assume + # that it's the first line that begins with a quote mark. + # Note: this could be fooled by a multiline function + # signature, where a continuation line begins with a quote + # mark. + if lineno is not None: + if source_lines is None: + return lineno+1 + pat = re.compile('(^|.*:)\s*\w*("|\')') + for lineno in range(lineno, len(source_lines)): + if pat.match(source_lines[lineno]): + return lineno + + # We couldn't find the line number. + return None + +###################################################################### +## 5. DocTest Runner +###################################################################### + +class DocTestRunner: + # This divider string is used to separate failure messages, and to + # separate sections of the summary. + DIVIDER = "*" * 70 + + def __init__(self, checker=None, verbose=None, optionflags=0): + """ + Create a new test runner. + + Optional keyword arg `checker` is the `OutputChecker` that + should be used to compare the expected outputs and actual + outputs of doctest examples. + + Optional keyword arg 'verbose' prints lots of stuff if true, + only failures if false; by default, it's true iff '-v' is in + sys.argv. + + Optional argument `optionflags` can be used to control how the + test runner compares expected output to actual output, and how + it displays failures. See the documentation for `testmod` for + more information. + """ + self._checker = checker or OutputChecker() + if verbose is None: + verbose = '-v' in sys.argv + self._verbose = verbose + self.optionflags = optionflags + self.original_optionflags = optionflags + + # Keep track of the examples we've run. + self.tries = 0 + self.failures = 0 + self._name2ft = {} + + # Create a fake output target for capturing doctest output. + self._fakeout = _SpoofOut() + + #///////////////////////////////////////////////////////////////// + # Reporting methods + #///////////////////////////////////////////////////////////////// + + def report_start(self, out, test, example): + """ + Report that the test runner is about to process the given + example. (Only displays a message if verbose=True) + """ + if self._verbose: + if example.want: + out('Trying:\n' + _indent(example.source) + + 'Expecting:\n' + _indent(example.want)) + else: + out('Trying:\n' + _indent(example.source) + + 'Expecting nothing\n') + + def report_success(self, out, test, example, got): + """ + Report that the given example ran successfully. (Only + displays a message if verbose=True) + """ + if self._verbose: + out("ok\n") + + def report_failure(self, out, test, example, got): + """ + Report that the given example failed. + """ + out(self._failure_header(test, example) + + self._checker.output_difference(example, got, self.optionflags)) + + def report_unexpected_exception(self, out, test, example, exc_info): + """ + Report that the given example raised an unexpected exception. + """ + out(self._failure_header(test, example) + + 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) + + def _failure_header(self, test, example): + out = [self.DIVIDER] + if test.filename: + if test.lineno is not None and example.lineno is not None: + lineno = test.lineno + example.lineno + 1 + else: + lineno = '?' + out.append('File "%s", line %s, in %s' % + (test.filename, lineno, test.name)) + else: + out.append('Line %s, in %s' % (example.lineno+1, test.name)) + out.append('Failed example:') + source = example.source + out.append(_indent(source)) + return '\n'.join(out) + + #///////////////////////////////////////////////////////////////// + # DocTest Running + #///////////////////////////////////////////////////////////////// + + def __run(self, test, compileflags, out): + """ + Run the examples in `test`. Write the outcome of each example + with one of the `DocTestRunner.report_*` methods, using the + writer function `out`. `compileflags` is the set of compiler + flags that should be used to execute examples. Return a tuple + `(f, t)`, where `t` is the number of examples tried, and `f` + is the number of examples that failed. The examples are run + in the namespace `test.globs`. + """ + # Keep track of the number of failures and tries. + failures = tries = 0 + + # Save the option flags (since option directives can be used + # to modify them). + original_optionflags = self.optionflags + + SUCCESS, FAILURE, BOOM = range(3) # `outcome` state + + check = self._checker.check_output + + # Process each example. + for examplenum, example in enumerate(test.examples): + + # If REPORT_ONLY_FIRST_FAILURE is set, then supress + # reporting after the first failure. + quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and + failures > 0) + + # Merge in the example's options. + self.optionflags = original_optionflags + if example.options: + for (optionflag, val) in example.options.items(): + if val: + self.optionflags |= optionflag + else: + self.optionflags &= ~optionflag + + # Record that we started this example. + tries += 1 + if not quiet: + self.report_start(out, test, example) + + # Use a special filename for compile(), so we can retrieve + # the source code during interactive debugging (see + # __patched_linecache_getlines). + filename = '' % (test.name, examplenum) + + # Run the example in the given context (globs), and record + # any exception that gets raised. (But don't intercept + # keyboard interrupts.) + try: + # Don't blink! This is where the user's code gets run. + exec compile(example.source, filename, "single", + compileflags, 1) in test.globs + self.debugger.set_continue() # ==== Example Finished ==== + exception = None + except KeyboardInterrupt: + raise + except: + exception = sys.exc_info() + self.debugger.set_continue() # ==== Example Finished ==== + + got = self._fakeout.getvalue() # the actual output + self._fakeout.truncate(0) + outcome = FAILURE # guilty until proved innocent or insane + + # If the example executed without raising any exceptions, + # verify its output. + if exception is None: + if check(example.want, got, self.optionflags): + outcome = SUCCESS + + # The example raised an exception: check if it was expected. + else: + exc_info = sys.exc_info() + exc_msg = traceback.format_exception_only(*exc_info[:2])[-1] + if not quiet: + got += _exception_traceback(exc_info) + + # If `example.exc_msg` is None, then we weren't expecting + # an exception. + if example.exc_msg is None: + outcome = BOOM + + # We expected an exception: see whether it matches. + elif check(example.exc_msg, exc_msg, self.optionflags): + outcome = SUCCESS + + # Another chance if they didn't care about the detail. + elif self.optionflags & IGNORE_EXCEPTION_DETAIL: + m1 = re.match(r'[^:]*:', example.exc_msg) + m2 = re.match(r'[^:]*:', exc_msg) + if m1 and m2 and check(m1.group(0), m2.group(0), + self.optionflags): + outcome = SUCCESS + + # Report the outcome. + if outcome is SUCCESS: + if not quiet: + self.report_success(out, test, example, got) + elif outcome is FAILURE: + if not quiet: + self.report_failure(out, test, example, got) + failures += 1 + elif outcome is BOOM: + if not quiet: + self.report_unexpected_exception(out, test, example, + exc_info) + failures += 1 + else: + assert False, ("unknown outcome", outcome) + + # Restore the option flags (in case they were modified) + self.optionflags = original_optionflags + + # Record and return the number of failures and tries. + self.__record_outcome(test, failures, tries) + return failures, tries + + def __record_outcome(self, test, f, t): + """ + Record the fact that the given DocTest (`test`) generated `f` + failures out of `t` tried examples. + """ + f2, t2 = self._name2ft.get(test.name, (0,0)) + self._name2ft[test.name] = (f+f2, t+t2) + self.failures += f + self.tries += t + + __LINECACHE_FILENAME_RE = re.compile(r'[\w\.]+)' + r'\[(?P\d+)\]>$') + def __patched_linecache_getlines(self, filename): + m = self.__LINECACHE_FILENAME_RE.match(filename) + if m and m.group('name') == self.test.name: + example = self.test.examples[int(m.group('examplenum'))] + return example.source.splitlines(True) + else: + return self.save_linecache_getlines(filename) + + def run(self, test, compileflags=None, out=None, clear_globs=True): + """ + Run the examples in `test`, and display the results using the + writer function `out`. + + The examples are run in the namespace `test.globs`. If + `clear_globs` is true (the default), then this namespace will + be cleared after the test runs, to help with garbage + collection. If you would like to examine the namespace after + the test completes, then use `clear_globs=False`. + + `compileflags` gives the set of flags that should be used by + the Python compiler when running the examples. If not + specified, then it will default to the set of future-import + flags that apply to `globs`. + + The output of each example is checked using + `DocTestRunner.check_output`, and the results are formatted by + the `DocTestRunner.report_*` methods. + """ + self.test = test + + if compileflags is None: + compileflags = _extract_future_flags(test.globs) + + save_stdout = sys.stdout + if out is None: + out = save_stdout.write + sys.stdout = self._fakeout + + # Patch pdb.set_trace to restore sys.stdout during interactive + # debugging (so it's not still redirected to self._fakeout). + # Note that the interactive output will go to *our* + # save_stdout, even if that's not the real sys.stdout; this + # allows us to write test cases for the set_trace behavior. + save_set_trace = pdb.set_trace + self.debugger = _OutputRedirectingPdb(save_stdout) + self.debugger.reset() + pdb.set_trace = self.debugger.set_trace + + # Patch linecache.getlines, so we can see the example's source + # when we're inside the debugger. + self.save_linecache_getlines = linecache.getlines + linecache.getlines = self.__patched_linecache_getlines + + try: + return self.__run(test, compileflags, out) + finally: + sys.stdout = save_stdout + pdb.set_trace = save_set_trace + linecache.getlines = self.save_linecache_getlines + if clear_globs: + test.globs.clear() + + #///////////////////////////////////////////////////////////////// + # Summarization + #///////////////////////////////////////////////////////////////// + def summarize(self, verbose=None): + """ + Print a summary of all the test cases that have been run by + this DocTestRunner, and return a tuple `(f, t)`, where `f` is + the total number of failed examples, and `t` is the total + number of tried examples. + + The optional `verbose` argument controls how detailed the + summary is. If the verbosity is not specified, then the + DocTestRunner's verbosity is used. + """ + if verbose is None: + verbose = self._verbose + notests = [] + passed = [] + failed = [] + totalt = totalf = 0 + for x in self._name2ft.items(): + name, (f, t) = x + assert f <= t + totalt += t + totalf += f + if t == 0: + notests.append(name) + elif f == 0: + passed.append( (name, t) ) + else: + failed.append(x) + if verbose: + if notests: + print len(notests), "items had no tests:" + notests.sort() + for thing in notests: + print " ", thing + if passed: + print len(passed), "items passed all tests:" + passed.sort() + for thing, count in passed: + print " %3d tests in %s" % (count, thing) + if failed: + print self.DIVIDER + print len(failed), "items had failures:" + failed.sort() + for thing, (f, t) in failed: + print " %3d of %3d in %s" % (f, t, thing) + if verbose: + print totalt, "tests in", len(self._name2ft), "items." + print totalt - totalf, "passed and", totalf, "failed." + if totalf: + print "***Test Failed***", totalf, "failures." + elif verbose: + print "Test passed." + return totalf, totalt + + #///////////////////////////////////////////////////////////////// + # Backward compatibility cruft to maintain doctest.master. + #///////////////////////////////////////////////////////////////// + def merge(self, other): + d = self._name2ft + for name, (f, t) in other._name2ft.items(): + if name in d: + print "*** DocTestRunner.merge: '" + name + "' in both" \ + " testers; summing outcomes." + f2, t2 = d[name] + f = f + f2 + t = t + t2 + d[name] = f, t + +class OutputChecker: + """ + A class used to check the whether the actual output from a doctest + example matches the expected output. `OutputChecker` defines two + methods: `check_output`, which compares a given pair of outputs, + and returns true if they match; and `output_difference`, which + returns a string describing the differences between two outputs. + """ + def check_output(self, want, got, optionflags): + """ + Return True iff the actual output from an example (`got`) + matches the expected output (`want`). These strings are + always considered to match if they are identical; but + depending on what option flags the test runner is using, + several non-exact match types are also possible. See the + documentation for `TestRunner` for more information about + option flags. + """ + # Handle the common case first, for efficiency: + # if they're string-identical, always return true. + if got == want: + return True + + # The values True and False replaced 1 and 0 as the return + # value for boolean comparisons in Python 2.3. + if not (optionflags & DONT_ACCEPT_TRUE_FOR_1): + if (got,want) == ("True\n", "1\n"): + return True + if (got,want) == ("False\n", "0\n"): + return True + + # can be used as a special sequence to signify a + # blank line, unless the DONT_ACCEPT_BLANKLINE flag is used. + if not (optionflags & DONT_ACCEPT_BLANKLINE): + # Replace in want with a blank line. + want = re.sub('(?m)^%s\s*?$' % re.escape(BLANKLINE_MARKER), + '', want) + # If a line in got contains only spaces, then remove the + # spaces. + got = re.sub('(?m)^\s*?$', '', got) + if got == want: + return True + + # This flag causes doctest to ignore any differences in the + # contents of whitespace strings. Note that this can be used + # in conjunction with the ELLIPSIS flag. + if optionflags & NORMALIZE_WHITESPACE: + got = ' '.join(got.split()) + want = ' '.join(want.split()) + if got == want: + return True + + # The ELLIPSIS flag says to let the sequence "..." in `want` + # match any substring in `got`. + if optionflags & ELLIPSIS: + if _ellipsis_match(want, got): + return True + + # We didn't find any match; return false. + return False + + # Should we do a fancy diff? + def _do_a_fancy_diff(self, want, got, optionflags): + # Not unless they asked for a fancy diff. + if not optionflags & (REPORT_UDIFF | + REPORT_CDIFF | + REPORT_NDIFF): + return False + + # If expected output uses ellipsis, a meaningful fancy diff is + # too hard ... or maybe not. In two real-life failures Tim saw, + # a diff was a major help anyway, so this is commented out. + # [todo] _ellipsis_match() knows which pieces do and don't match, + # and could be the basis for a kick-ass diff in this case. + ##if optionflags & ELLIPSIS and ELLIPSIS_MARKER in want: + ## return False + + # ndiff does intraline difference marking, so can be useful even + # for 1-line differences. + if optionflags & REPORT_NDIFF: + return True + + # The other diff types need at least a few lines to be helpful. + return want.count('\n') > 2 and got.count('\n') > 2 + + def output_difference(self, example, got, optionflags): + """ + Return a string describing the differences between the + expected output for a given example (`example`) and the actual + output (`got`). `optionflags` is the set of option flags used + to compare `want` and `got`. + """ + want = example.want + # If s are being used, then replace blank lines + # with in the actual output string. + if not (optionflags & DONT_ACCEPT_BLANKLINE): + got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got) + + # Check if we should use diff. + if self._do_a_fancy_diff(want, got, optionflags): + # Split want & got into lines. + want_lines = want.splitlines(True) # True == keep line ends + got_lines = got.splitlines(True) + # Use difflib to find their differences. + if optionflags & REPORT_UDIFF: + diff = difflib.unified_diff(want_lines, got_lines, n=2) + diff = list(diff)[2:] # strip the diff header + kind = 'unified diff with -expected +actual' + elif optionflags & REPORT_CDIFF: + diff = difflib.context_diff(want_lines, got_lines, n=2) + diff = list(diff)[2:] # strip the diff header + kind = 'context diff with expected followed by actual' + elif optionflags & REPORT_NDIFF: + engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK) + diff = list(engine.compare(want_lines, got_lines)) + kind = 'ndiff with -expected +actual' + else: + assert 0, 'Bad diff option' + # Remove trailing whitespace on diff output. + diff = [line.rstrip() + '\n' for line in diff] + return 'Differences (%s):\n' % kind + _indent(''.join(diff)) + + # If we're not using diff, then simply list the expected + # output followed by the actual output. + if want and got: + return 'Expected:\n%sGot:\n%s' % (_indent(want), _indent(got)) + elif want: + return 'Expected:\n%sGot nothing\n' % _indent(want) + elif got: + return 'Expected nothing\nGot:\n%s' % _indent(got) + else: + return 'Expected nothing\nGot nothing\n' + +class DocTestFailure(Exception): + """A DocTest example has failed in debugging mode. + + The exception instance has variables: + + - test: the DocTest object being run + + - excample: the Example object that failed + + - got: the actual output + """ + def __init__(self, test, example, got): + self.test = test + self.example = example + self.got = got + + def __str__(self): + return str(self.test) + +class UnexpectedException(Exception): + """A DocTest example has encountered an unexpected exception + + The exception instance has variables: + + - test: the DocTest object being run + + - excample: the Example object that failed + + - exc_info: the exception info + """ + def __init__(self, test, example, exc_info): + self.test = test + self.example = example + self.exc_info = exc_info + + def __str__(self): + return str(self.test) + +class DebugRunner(DocTestRunner): + + def run(self, test, compileflags=None, out=None, clear_globs=True): + r = DocTestRunner.run(self, test, compileflags, out, False) + if clear_globs: + test.globs.clear() + return r + + def report_unexpected_exception(self, out, test, example, exc_info): + raise UnexpectedException(test, example, exc_info) + + def report_failure(self, out, test, example, got): + raise DocTestFailure(test, example, got) + +###################################################################### +## 6. Test Functions +###################################################################### +# These should be backwards compatible. + +# For backward compatibility, a global instance of a DocTestRunner +# class, updated by testmod. +master = None + +def testmod(m=None, name=None, globs=None, verbose=None, isprivate=None, + report=True, optionflags=0, extraglobs=None, + raise_on_error=False, exclude_empty=False): + """m=None, name=None, globs=None, verbose=None, isprivate=None, + report=True, optionflags=0, extraglobs=None, raise_on_error=False, + exclude_empty=False + + Test examples in docstrings in functions and classes reachable + from module m (or the current module if m is not supplied), starting + with m.__doc__. Unless isprivate is specified, private names + are not skipped. + + Also test examples reachable from dict m.__test__ if it exists and is + not None. m.__test__ maps names to functions, classes and strings; + function and class docstrings are tested even if the name is private; + strings are tested directly, as if they were docstrings. + + Return (#failures, #tests). + + See doctest.__doc__ for an overview. + + Optional keyword arg "name" gives the name of the module; by default + use m.__name__. + + Optional keyword arg "globs" gives a dict to be used as the globals + when executing examples; by default, use m.__dict__. A copy of this + dict is actually used for each docstring, so that each docstring's + examples start with a clean slate. + + Optional keyword arg "extraglobs" gives a dictionary that should be + merged into the globals that are used to execute examples. By + default, no extra globals are used. This is new in 2.4. + + Optional keyword arg "verbose" prints lots of stuff if true, prints + only failures if false; by default, it's true iff "-v" is in sys.argv. + + Optional keyword arg "report" prints a summary at the end when true, + else prints nothing at the end. In verbose mode, the summary is + detailed, else very brief (in fact, empty if all tests passed). + + Optional keyword arg "optionflags" or's together module constants, + and defaults to 0. This is new in 2.3. Possible values (see the + docs for details): + + DONT_ACCEPT_TRUE_FOR_1 + DONT_ACCEPT_BLANKLINE + NORMALIZE_WHITESPACE + ELLIPSIS + IGNORE_EXCEPTION_DETAIL + REPORT_UDIFF + REPORT_CDIFF + REPORT_NDIFF + REPORT_ONLY_FIRST_FAILURE + + Optional keyword arg "raise_on_error" raises an exception on the + first unexpected exception or failure. This allows failures to be + post-mortem debugged. + + Deprecated in Python 2.4: + Optional keyword arg "isprivate" specifies a function used to + determine whether a name is private. The default function is + treat all functions as public. Optionally, "isprivate" can be + set to doctest.is_private to skip over functions marked as private + using the underscore naming convention; see its docs for details. + + Advanced tomfoolery: testmod runs methods of a local instance of + class doctest.Tester, then merges the results into (or creates) + global Tester instance doctest.master. Methods of doctest.master + can be called directly too, if you want to do something unusual. + Passing report=0 to testmod is especially useful then, to delay + displaying a summary. Invoke doctest.master.summarize(verbose) + when you're done fiddling. + """ + global master + + if isprivate is not None: + warnings.warn("the isprivate argument is deprecated; " + "examine DocTestFinder.find() lists instead", + DeprecationWarning) + + # If no module was given, then use __main__. + if m is None: + # DWA - m will still be None if this wasn't invoked from the command + # line, in which case the following TypeError is about as good an error + # as we should expect + m = sys.modules.get('__main__') + + # Check that we were actually given a module. + if not inspect.ismodule(m): + raise TypeError("testmod: module required; %r" % (m,)) + + # If no name was given, then use the module's name. + if name is None: + name = m.__name__ + + # Find, parse, and run all tests in the given module. + finder = DocTestFinder(_namefilter=isprivate, exclude_empty=exclude_empty) + + if raise_on_error: + runner = DebugRunner(verbose=verbose, optionflags=optionflags) + else: + runner = DocTestRunner(verbose=verbose, optionflags=optionflags) + + for test in finder.find(m, name, globs=globs, extraglobs=extraglobs): + runner.run(test) + + if report: + runner.summarize() + + if master is None: + master = runner + else: + master.merge(runner) + + return runner.failures, runner.tries + +def testfile(filename, module_relative=True, name=None, package=None, + globs=None, verbose=None, report=True, optionflags=0, + extraglobs=None, raise_on_error=False, parser=DocTestParser()): + """ + Test examples in the given file. Return (#failures, #tests). + + Optional keyword arg "module_relative" specifies how filenames + should be interpreted: + + - If "module_relative" is True (the default), then "filename" + specifies a module-relative path. By default, this path is + relative to the calling module's directory; but if the + "package" argument is specified, then it is relative to that + package. To ensure os-independence, "filename" should use + "/" characters to separate path segments, and should not + be an absolute path (i.e., it may not begin with "/"). + + - If "module_relative" is False, then "filename" specifies an + os-specific path. The path may be absolute or relative (to + the current working directory). + + Optional keyword arg "name" gives the name of the test; by default + use the file's basename. + + Optional keyword argument "package" is a Python package or the + name of a Python package whose directory should be used as the + base directory for a module relative filename. If no package is + specified, then the calling module's directory is used as the base + directory for module relative filenames. It is an error to + specify "package" if "module_relative" is False. + + Optional keyword arg "globs" gives a dict to be used as the globals + when executing examples; by default, use {}. A copy of this dict + is actually used for each docstring, so that each docstring's + examples start with a clean slate. + + Optional keyword arg "extraglobs" gives a dictionary that should be + merged into the globals that are used to execute examples. By + default, no extra globals are used. + + Optional keyword arg "verbose" prints lots of stuff if true, prints + only failures if false; by default, it's true iff "-v" is in sys.argv. + + Optional keyword arg "report" prints a summary at the end when true, + else prints nothing at the end. In verbose mode, the summary is + detailed, else very brief (in fact, empty if all tests passed). + + Optional keyword arg "optionflags" or's together module constants, + and defaults to 0. Possible values (see the docs for details): + + DONT_ACCEPT_TRUE_FOR_1 + DONT_ACCEPT_BLANKLINE + NORMALIZE_WHITESPACE + ELLIPSIS + IGNORE_EXCEPTION_DETAIL + REPORT_UDIFF + REPORT_CDIFF + REPORT_NDIFF + REPORT_ONLY_FIRST_FAILURE + + Optional keyword arg "raise_on_error" raises an exception on the + first unexpected exception or failure. This allows failures to be + post-mortem debugged. + + Optional keyword arg "parser" specifies a DocTestParser (or + subclass) that should be used to extract tests from the files. + + Advanced tomfoolery: testmod runs methods of a local instance of + class doctest.Tester, then merges the results into (or creates) + global Tester instance doctest.master. Methods of doctest.master + can be called directly too, if you want to do something unusual. + Passing report=0 to testmod is especially useful then, to delay + displaying a summary. Invoke doctest.master.summarize(verbose) + when you're done fiddling. + """ + global master + + if package and not module_relative: + raise ValueError("Package may only be specified for module-" + "relative paths.") + + # Relativize the path + if module_relative: + package = _normalize_module(package) + filename = _module_relative_path(package, filename) + + # If no name was given, then use the file's name. + if name is None: + name = os.path.basename(filename) + + # Assemble the globals. + if globs is None: + globs = {} + else: + globs = globs.copy() + if extraglobs is not None: + globs.update(extraglobs) + + if raise_on_error: + runner = DebugRunner(verbose=verbose, optionflags=optionflags) + else: + runner = DocTestRunner(verbose=verbose, optionflags=optionflags) + + # Read the file, convert it to a test, and run it. + s = open(filename).read() + test = parser.get_doctest(s, globs, name, filename, 0) + runner.run(test) + + if report: + runner.summarize() + + if master is None: + master = runner + else: + master.merge(runner) + + return runner.failures, runner.tries + +def run_docstring_examples(f, globs, verbose=False, name="NoName", + compileflags=None, optionflags=0): + """ + Test examples in the given object's docstring (`f`), using `globs` + as globals. Optional argument `name` is used in failure messages. + If the optional argument `verbose` is true, then generate output + even if there are no failures. + + `compileflags` gives the set of flags that should be used by the + Python compiler when running the examples. If not specified, then + it will default to the set of future-import flags that apply to + `globs`. + + Optional keyword arg `optionflags` specifies options for the + testing and output. See the documentation for `testmod` for more + information. + """ + # Find, parse, and run all tests in the given module. + finder = DocTestFinder(verbose=verbose, recurse=False) + runner = DocTestRunner(verbose=verbose, optionflags=optionflags) + for test in finder.find(f, name, globs=globs): + runner.run(test, compileflags=compileflags) + +###################################################################### +## 7. Tester +###################################################################### +# This is provided only for backwards compatibility. It's not +# actually used in any way. + +class Tester: + def __init__(self, mod=None, globs=None, verbose=None, + isprivate=None, optionflags=0): + + warnings.warn("class Tester is deprecated; " + "use class doctest.DocTestRunner instead", + DeprecationWarning, stacklevel=2) + if mod is None and globs is None: + raise TypeError("Tester.__init__: must specify mod or globs") + if mod is not None and not inspect.ismodule(mod): + raise TypeError("Tester.__init__: mod must be a module; %r" % + (mod,)) + if globs is None: + globs = mod.__dict__ + self.globs = globs + + self.verbose = verbose + self.isprivate = isprivate + self.optionflags = optionflags + self.testfinder = DocTestFinder(_namefilter=isprivate) + self.testrunner = DocTestRunner(verbose=verbose, + optionflags=optionflags) + + def runstring(self, s, name): + test = DocTestParser().get_doctest(s, self.globs, name, None, None) + if self.verbose: + print "Running string", name + (f,t) = self.testrunner.run(test) + if self.verbose: + print f, "of", t, "examples failed in string", name + return (f,t) + + def rundoc(self, object, name=None, module=None): + f = t = 0 + tests = self.testfinder.find(object, name, module=module, + globs=self.globs) + for test in tests: + (f2, t2) = self.testrunner.run(test) + (f,t) = (f+f2, t+t2) + return (f,t) + + def rundict(self, d, name, module=None): + import new + m = new.module(name) + m.__dict__.update(d) + if module is None: + module = False + return self.rundoc(m, name, module) + + def run__test__(self, d, name): + import new + m = new.module(name) + m.__test__ = d + return self.rundoc(m, name) + + def summarize(self, verbose=None): + return self.testrunner.summarize(verbose) + + def merge(self, other): + self.testrunner.merge(other.testrunner) + +###################################################################### +## 8. Unittest Support +###################################################################### + +_unittest_reportflags = 0 + +def set_unittest_reportflags(flags): + global _unittest_reportflags + + if (flags & REPORTING_FLAGS) != flags: + raise ValueError("Only reporting flags allowed", flags) + old = _unittest_reportflags + _unittest_reportflags = flags + return old + + +class DocTestCase(unittest.TestCase): + + def __init__(self, test, optionflags=0, setUp=None, tearDown=None, + checker=None): + + unittest.TestCase.__init__(self) + self._dt_optionflags = optionflags + self._dt_checker = checker + self._dt_test = test + self._dt_setUp = setUp + self._dt_tearDown = tearDown + + def setUp(self): + test = self._dt_test + + if self._dt_setUp is not None: + self._dt_setUp(test) + + def tearDown(self): + test = self._dt_test + + if self._dt_tearDown is not None: + self._dt_tearDown(test) + + test.globs.clear() + + def runTest(self): + test = self._dt_test + old = sys.stdout + new = StringIO() + optionflags = self._dt_optionflags + + if not (optionflags & REPORTING_FLAGS): + # The option flags don't include any reporting flags, + # so add the default reporting flags + optionflags |= _unittest_reportflags + + runner = DocTestRunner(optionflags=optionflags, + checker=self._dt_checker, verbose=False) + + try: + runner.DIVIDER = "-"*70 + failures, tries = runner.run( + test, out=new.write, clear_globs=False) + finally: + sys.stdout = old + + if failures: + raise self.failureException(self.format_failure(new.getvalue())) + + def format_failure(self, err): + test = self._dt_test + if test.lineno is None: + lineno = 'unknown line number' + else: + lineno = '%s' % test.lineno + lname = '.'.join(test.name.split('.')[-1:]) + return ('Failed doctest test for %s\n' + ' File "%s", line %s, in %s\n\n%s' + % (test.name, test.filename, lineno, lname, err) + ) + + def debug(self): + self.setUp() + runner = DebugRunner(optionflags=self._dt_optionflags, + checker=self._dt_checker, verbose=False) + runner.run(self._dt_test) + self.tearDown() + + def id(self): + return self._dt_test.name + + def __repr__(self): + name = self._dt_test.name.split('.') + return "%s (%s)" % (name[-1], '.'.join(name[:-1])) + + __str__ = __repr__ + + def shortDescription(self): + return "Doctest: " + self._dt_test.name + +def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, + **options): + """ + Convert doctest tests for a module to a unittest test suite. + + This converts each documentation string in a module that + contains doctest tests to a unittest test case. If any of the + tests in a doc string fail, then the test case fails. An exception + is raised showing the name of the file containing the test and a + (sometimes approximate) line number. + + The `module` argument provides the module to be tested. The argument + can be either a module or a module name. + + If no argument is given, the calling module is used. + + A number of options may be provided as keyword arguments: + + setUp + A set-up function. This is called before running the + tests in each file. The setUp function will be passed a DocTest + object. The setUp function can access the test globals as the + globs attribute of the test passed. + + tearDown + A tear-down function. This is called after running the + tests in each file. The tearDown function will be passed a DocTest + object. The tearDown function can access the test globals as the + globs attribute of the test passed. + + globs + A dictionary containing initial global variables for the tests. + + optionflags + A set of doctest option flags expressed as an integer. + """ + + if test_finder is None: + test_finder = DocTestFinder() + + module = _normalize_module(module) + tests = test_finder.find(module, globs=globs, extraglobs=extraglobs) + if globs is None: + globs = module.__dict__ + if not tests: + # Why do we want to do this? Because it reveals a bug that might + # otherwise be hidden. + raise ValueError(module, "has no tests") + + tests.sort() + suite = unittest.TestSuite() + for test in tests: + if len(test.examples) == 0: + continue + if not test.filename: + filename = module.__file__ + if filename[-4:] in (".pyc", ".pyo"): + filename = filename[:-1] + elif sys.platform.startswith('java') and \ + filename.endswith('$py.class'): + filename = '%s.py' % filename[:-9] + test.filename = filename + suite.addTest(DocTestCase(test, **options)) + + return suite + +class DocFileCase(DocTestCase): + + def id(self): + return '_'.join(self._dt_test.name.split('.')) + + def __repr__(self): + return self._dt_test.filename + __str__ = __repr__ + + def format_failure(self, err): + return ('Failed doctest test for %s\n File "%s", line 0\n\n%s' + % (self._dt_test.name, self._dt_test.filename, err) + ) + +def DocFileTest(path, module_relative=True, package=None, + globs=None, parser=DocTestParser(), **options): + if globs is None: + globs = {} + + if package and not module_relative: + raise ValueError("Package may only be specified for module-" + "relative paths.") + + # Relativize the path. + if module_relative: + package = _normalize_module(package) + path = _module_relative_path(package, path) + + # Find the file and read it. + name = os.path.basename(path) + doc = open(path).read() + + # Convert it to a test, and wrap it in a DocFileCase. + test = parser.get_doctest(doc, globs, name, path, 0) + return DocFileCase(test, **options) + +def DocFileSuite(*paths, **kw): + """A unittest suite for one or more doctest files. + + The path to each doctest file is given as a string; the + interpretation of that string depends on the keyword argument + "module_relative". + + A number of options may be provided as keyword arguments: + + module_relative + If "module_relative" is True, then the given file paths are + interpreted as os-independent module-relative paths. By + default, these paths are relative to the calling module's + directory; but if the "package" argument is specified, then + they are relative to that package. To ensure os-independence, + "filename" should use "/" characters to separate path + segments, and may not be an absolute path (i.e., it may not + begin with "/"). + + If "module_relative" is False, then the given file paths are + interpreted as os-specific paths. These paths may be absolute + or relative (to the current working directory). + + package + A Python package or the name of a Python package whose directory + should be used as the base directory for module relative paths. + If "package" is not specified, then the calling module's + directory is used as the base directory for module relative + filenames. It is an error to specify "package" if + "module_relative" is False. + + setUp + A set-up function. This is called before running the + tests in each file. The setUp function will be passed a DocTest + object. The setUp function can access the test globals as the + globs attribute of the test passed. + + tearDown + A tear-down function. This is called after running the + tests in each file. The tearDown function will be passed a DocTest + object. The tearDown function can access the test globals as the + globs attribute of the test passed. + + globs + A dictionary containing initial global variables for the tests. + + optionflags + A set of doctest option flags expressed as an integer. + + parser + A DocTestParser (or subclass) that should be used to extract + tests from the files. + """ + suite = unittest.TestSuite() + + # We do this here so that _normalize_module is called at the right + # level. If it were called in DocFileTest, then this function + # would be the caller and we might guess the package incorrectly. + if kw.get('module_relative', True): + kw['package'] = _normalize_module(kw.get('package')) + + for path in paths: + suite.addTest(DocFileTest(path, **kw)) + + return suite + +###################################################################### +## 9. Debugging Support +###################################################################### + +def script_from_examples(s): + output = [] + for piece in DocTestParser().parse(s): + if isinstance(piece, Example): + # Add the example's source code (strip trailing NL) + output.append(piece.source[:-1]) + # Add the expected output: + want = piece.want + if want: + output.append('# Expected:') + output += ['## '+l for l in want.split('\n')[:-1]] + else: + # Add non-example text. + output += [_comment_line(l) + for l in piece.split('\n')[:-1]] + + # Trim junk on both ends. + while output and output[-1] == '#': + output.pop() + while output and output[0] == '#': + output.pop(0) + # Combine the output, and return it. + # Add a courtesy newline to prevent exec from choking (see bug #1172785) + return '\n'.join(output) + '\n' + +def testsource(module, name): + """Extract the test sources from a doctest docstring as a script. + + Provide the module (or dotted name of the module) containing the + test to be debugged and the name (within the module) of the object + with the doc string with tests to be debugged. + """ + module = _normalize_module(module) + tests = DocTestFinder().find(module) + test = [t for t in tests if t.name == name] + if not test: + raise ValueError(name, "not found in tests") + test = test[0] + testsrc = script_from_examples(test.docstring) + return testsrc + +def debug_src(src, pm=False, globs=None): + """Debug a single doctest docstring, in argument `src`'""" + testsrc = script_from_examples(src) + debug_script(testsrc, pm, globs) + +def debug_script(src, pm=False, globs=None): + "Debug a test script. `src` is the script, as a string." + import pdb + + # Note that tempfile.NameTemporaryFile() cannot be used. As the + # docs say, a file so created cannot be opened by name a second time + # on modern Windows boxes, and execfile() needs to open it. + srcfilename = tempfile.mktemp(".py", "doctestdebug") + f = open(srcfilename, 'w') + f.write(src) + f.close() + + try: + if globs: + globs = globs.copy() + else: + globs = {} + + if pm: + try: + execfile(srcfilename, globs, globs) + except: + print sys.exc_info()[1] + pdb.post_mortem(sys.exc_info()[2]) + else: + # Note that %r is vital here. '%s' instead can, e.g., cause + # backslashes to get treated as metacharacters on Windows. + pdb.run("execfile(%r)" % srcfilename, globs, globs) + + finally: + os.remove(srcfilename) + +def debug(module, name, pm=False): + """Debug a single doctest docstring. + + Provide the module (or dotted name of the module) containing the + test to be debugged and the name (within the module) of the object + with the docstring with tests to be debugged. + """ + module = _normalize_module(module) + testsrc = testsource(module, name) + debug_script(testsrc, pm, module.__dict__) + + +__test__ = {} diff --git a/lib/spack/external/nose/failure.py b/lib/spack/external/nose/failure.py new file mode 100644 index 00000000000..c5fabfda5e7 --- /dev/null +++ b/lib/spack/external/nose/failure.py @@ -0,0 +1,42 @@ +import logging +import unittest +from traceback import format_tb +from nose.pyversion import is_base_exception + +log = logging.getLogger(__name__) + + +__all__ = ['Failure'] + + +class Failure(unittest.TestCase): + """Unloadable or unexecutable test. + + A Failure case is placed in a test suite to indicate the presence of a + test that could not be loaded or executed. A common example is a test + module that fails to import. + + """ + __test__ = False # do not collect + def __init__(self, exc_class, exc_val, tb=None, address=None): + log.debug("A failure! %s %s %s", exc_class, exc_val, format_tb(tb)) + self.exc_class = exc_class + self.exc_val = exc_val + self.tb = tb + self._address = address + unittest.TestCase.__init__(self) + + def __str__(self): + return "Failure: %s (%s)" % ( + getattr(self.exc_class, '__name__', self.exc_class), self.exc_val) + + def address(self): + return self._address + + def runTest(self): + if self.tb is not None: + if is_base_exception(self.exc_val): + raise self.exc_val, None, self.tb + raise self.exc_class, self.exc_val, self.tb + else: + raise self.exc_class(self.exc_val) diff --git a/lib/spack/external/nose/importer.py b/lib/spack/external/nose/importer.py new file mode 100644 index 00000000000..e677658ce61 --- /dev/null +++ b/lib/spack/external/nose/importer.py @@ -0,0 +1,167 @@ +"""Implements an importer that looks only in specific path (ignoring +sys.path), and uses a per-path cache in addition to sys.modules. This is +necessary because test modules in different directories frequently have the +same names, which means that the first loaded would mask the rest when using +the builtin importer. +""" +import logging +import os +import sys +from nose.config import Config + +from imp import find_module, load_module, acquire_lock, release_lock + +log = logging.getLogger(__name__) + +try: + _samefile = os.path.samefile +except AttributeError: + def _samefile(src, dst): + return (os.path.normcase(os.path.realpath(src)) == + os.path.normcase(os.path.realpath(dst))) + + +class Importer(object): + """An importer class that does only path-specific imports. That + is, the given module is not searched for on sys.path, but only at + the path or in the directory specified. + """ + def __init__(self, config=None): + if config is None: + config = Config() + self.config = config + + def importFromPath(self, path, fqname): + """Import a dotted-name package whose tail is at path. In other words, + given foo.bar and path/to/foo/bar.py, import foo from path/to/foo then + bar from path/to/foo/bar, returning bar. + """ + # find the base dir of the package + path_parts = os.path.normpath(os.path.abspath(path)).split(os.sep) + name_parts = fqname.split('.') + if path_parts[-1] == '__init__.py': + path_parts.pop() + path_parts = path_parts[:-(len(name_parts))] + dir_path = os.sep.join(path_parts) + # then import fqname starting from that dir + return self.importFromDir(dir_path, fqname) + + def importFromDir(self, dir, fqname): + """Import a module *only* from path, ignoring sys.path and + reloading if the version in sys.modules is not the one we want. + """ + dir = os.path.normpath(os.path.abspath(dir)) + log.debug("Import %s from %s", fqname, dir) + + # FIXME reimplement local per-dir cache? + + # special case for __main__ + if fqname == '__main__': + return sys.modules[fqname] + + if self.config.addPaths: + add_path(dir, self.config) + + path = [dir] + parts = fqname.split('.') + part_fqname = '' + mod = parent = fh = None + + for part in parts: + if part_fqname == '': + part_fqname = part + else: + part_fqname = "%s.%s" % (part_fqname, part) + try: + acquire_lock() + log.debug("find module part %s (%s) in %s", + part, part_fqname, path) + fh, filename, desc = find_module(part, path) + old = sys.modules.get(part_fqname) + if old is not None: + # test modules frequently have name overlap; make sure + # we get a fresh copy of anything we are trying to load + # from a new path + log.debug("sys.modules has %s as %s", part_fqname, old) + if (self.sameModule(old, filename) + or (self.config.firstPackageWins and + getattr(old, '__path__', None))): + mod = old + else: + del sys.modules[part_fqname] + mod = load_module(part_fqname, fh, filename, desc) + else: + mod = load_module(part_fqname, fh, filename, desc) + finally: + if fh: + fh.close() + release_lock() + if parent: + setattr(parent, part, mod) + if hasattr(mod, '__path__'): + path = mod.__path__ + parent = mod + return mod + + def _dirname_if_file(self, filename): + # We only take the dirname if we have a path to a non-dir, + # because taking the dirname of a symlink to a directory does not + # give the actual directory parent. + if os.path.isdir(filename): + return filename + else: + return os.path.dirname(filename) + + def sameModule(self, mod, filename): + mod_paths = [] + if hasattr(mod, '__path__'): + for path in mod.__path__: + mod_paths.append(self._dirname_if_file(path)) + elif hasattr(mod, '__file__'): + mod_paths.append(self._dirname_if_file(mod.__file__)) + else: + # builtin or other module-like object that + # doesn't have __file__; must be new + return False + new_path = self._dirname_if_file(filename) + for mod_path in mod_paths: + log.debug( + "module already loaded? mod: %s new: %s", + mod_path, new_path) + if _samefile(mod_path, new_path): + return True + return False + + +def add_path(path, config=None): + """Ensure that the path, or the root of the current package (if + path is in a package), is in sys.path. + """ + + # FIXME add any src-looking dirs seen too... need to get config for that + + log.debug('Add path %s' % path) + if not path: + return [] + added = [] + parent = os.path.dirname(path) + if (parent + and os.path.exists(os.path.join(path, '__init__.py'))): + added.extend(add_path(parent, config)) + elif not path in sys.path: + log.debug("insert %s into sys.path", path) + sys.path.insert(0, path) + added.append(path) + if config and config.srcDirs: + for dirname in config.srcDirs: + dirpath = os.path.join(path, dirname) + if os.path.isdir(dirpath): + sys.path.insert(0, dirpath) + added.append(dirpath) + return added + + +def remove_path(path): + log.debug('Remove path %s' % path) + if path in sys.path: + sys.path.remove(path) diff --git a/lib/spack/external/nose/inspector.py b/lib/spack/external/nose/inspector.py new file mode 100644 index 00000000000..a6c4a3e3b6a --- /dev/null +++ b/lib/spack/external/nose/inspector.py @@ -0,0 +1,207 @@ +"""Simple traceback introspection. Used to add additional information to +AssertionErrors in tests, so that failure messages may be more informative. +""" +import inspect +import logging +import re +import sys +import textwrap +import tokenize + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +log = logging.getLogger(__name__) + +def inspect_traceback(tb): + """Inspect a traceback and its frame, returning source for the expression + where the exception was raised, with simple variable replacement performed + and the line on which the exception was raised marked with '>>' + """ + log.debug('inspect traceback %s', tb) + + # we only want the innermost frame, where the exception was raised + while tb.tb_next: + tb = tb.tb_next + + frame = tb.tb_frame + lines, exc_line = tbsource(tb) + + # figure out the set of lines to grab. + inspect_lines, mark_line = find_inspectable_lines(lines, exc_line) + src = StringIO(textwrap.dedent(''.join(inspect_lines))) + exp = Expander(frame.f_locals, frame.f_globals) + + while inspect_lines: + try: + for tok in tokenize.generate_tokens(src.readline): + exp(*tok) + except tokenize.TokenError, e: + # this can happen if our inspectable region happens to butt up + # against the end of a construct like a docstring with the closing + # """ on separate line + log.debug("Tokenizer error: %s", e) + inspect_lines.pop(0) + mark_line -= 1 + src = StringIO(textwrap.dedent(''.join(inspect_lines))) + exp = Expander(frame.f_locals, frame.f_globals) + continue + break + padded = [] + if exp.expanded_source: + exp_lines = exp.expanded_source.split('\n') + ep = 0 + for line in exp_lines: + if ep == mark_line: + padded.append('>> ' + line) + else: + padded.append(' ' + line) + ep += 1 + return '\n'.join(padded) + + +def tbsource(tb, context=6): + """Get source from a traceback object. + + A tuple of two things is returned: a list of lines of context from + the source code, and the index of the current line within that list. + The optional second argument specifies the number of lines of context + to return, which are centered around the current line. + + .. Note :: + This is adapted from inspect.py in the python 2.4 standard library, + since a bug in the 2.3 version of inspect prevents it from correctly + locating source lines in a traceback frame. + """ + + lineno = tb.tb_lineno + frame = tb.tb_frame + + if context > 0: + start = lineno - 1 - context//2 + log.debug("lineno: %s start: %s", lineno, start) + + try: + lines, dummy = inspect.findsource(frame) + except IOError: + lines, index = [''], 0 + else: + all_lines = lines + start = max(start, 1) + start = max(0, min(start, len(lines) - context)) + lines = lines[start:start+context] + index = lineno - 1 - start + + # python 2.5 compat: if previous line ends in a continuation, + # decrement start by 1 to match 2.4 behavior + if sys.version_info >= (2, 5) and index > 0: + while lines[index-1].strip().endswith('\\'): + start -= 1 + lines = all_lines[start:start+context] + else: + lines, index = [''], 0 + log.debug("tbsource lines '''%s''' around index %s", lines, index) + return (lines, index) + + +def find_inspectable_lines(lines, pos): + """Find lines in home that are inspectable. + + Walk back from the err line up to 3 lines, but don't walk back over + changes in indent level. + + Walk forward up to 3 lines, counting \ separated lines as 1. Don't walk + over changes in indent level (unless part of an extended line) + """ + cnt = re.compile(r'\\[\s\n]*$') + df = re.compile(r':[\s\n]*$') + ind = re.compile(r'^(\s*)') + toinspect = [] + home = lines[pos] + home_indent = ind.match(home).groups()[0] + + before = lines[max(pos-3, 0):pos] + before.reverse() + after = lines[pos+1:min(pos+4, len(lines))] + + for line in before: + if ind.match(line).groups()[0] == home_indent: + toinspect.append(line) + else: + break + toinspect.reverse() + toinspect.append(home) + home_pos = len(toinspect)-1 + continued = cnt.search(home) + for line in after: + if ((continued or ind.match(line).groups()[0] == home_indent) + and not df.search(line)): + toinspect.append(line) + continued = cnt.search(line) + else: + break + log.debug("Inspecting lines '''%s''' around %s", toinspect, home_pos) + return toinspect, home_pos + + +class Expander: + """Simple expression expander. Uses tokenize to find the names and + expands any that can be looked up in the frame. + """ + def __init__(self, locals, globals): + self.locals = locals + self.globals = globals + self.lpos = None + self.expanded_source = '' + + def __call__(self, ttype, tok, start, end, line): + # TODO + # deal with unicode properly + + # TODO + # Dealing with instance members + # always keep the last thing seen + # if the current token is a dot, + # get ready to getattr(lastthing, this thing) on the + # next call. + + if self.lpos is not None: + if start[1] >= self.lpos: + self.expanded_source += ' ' * (start[1]-self.lpos) + elif start[1] < self.lpos: + # newline, indent correctly + self.expanded_source += ' ' * start[1] + self.lpos = end[1] + + if ttype == tokenize.INDENT: + pass + elif ttype == tokenize.NAME: + # Clean this junk up + try: + val = self.locals[tok] + if callable(val): + val = tok + else: + val = repr(val) + except KeyError: + try: + val = self.globals[tok] + if callable(val): + val = tok + else: + val = repr(val) + + except KeyError: + val = tok + # FIXME... not sure how to handle things like funcs, classes + # FIXME this is broken for some unicode strings + self.expanded_source += val + else: + self.expanded_source += tok + # if this is the end of the line and the line ends with + # \, then tack a \ and newline onto the output + # print line[end[1]:] + if re.match(r'\s+\\\n', line[end[1]:]): + self.expanded_source += ' \\\n' diff --git a/lib/spack/external/nose/loader.py b/lib/spack/external/nose/loader.py new file mode 100644 index 00000000000..3744e54ff69 --- /dev/null +++ b/lib/spack/external/nose/loader.py @@ -0,0 +1,623 @@ +""" +Test Loader +----------- + +nose's test loader implements the same basic functionality as its +superclass, unittest.TestLoader, but extends it by more liberal +interpretations of what may be a test and how a test may be named. +""" +from __future__ import generators + +import logging +import os +import sys +import unittest +import types +from inspect import isfunction +from nose.pyversion import unbound_method, ismethod +from nose.case import FunctionTestCase, MethodTestCase +from nose.failure import Failure +from nose.config import Config +from nose.importer import Importer, add_path, remove_path +from nose.selector import defaultSelector, TestAddress +from nose.util import func_lineno, getpackage, isclass, isgenerator, \ + ispackage, regex_last_key, resolve_name, transplant_func, \ + transplant_class, test_address +from nose.suite import ContextSuiteFactory, ContextList, LazySuite +from nose.pyversion import sort_list, cmp_to_key + + +log = logging.getLogger(__name__) +#log.setLevel(logging.DEBUG) + +# for efficiency and easier mocking +op_normpath = os.path.normpath +op_abspath = os.path.abspath +op_join = os.path.join +op_isdir = os.path.isdir +op_isfile = os.path.isfile + + +__all__ = ['TestLoader', 'defaultTestLoader'] + + +class TestLoader(unittest.TestLoader): + """Test loader that extends unittest.TestLoader to: + + * Load tests from test-like functions and classes that are not + unittest.TestCase subclasses + * Find and load test modules in a directory + * Support tests that are generators + * Support easy extensions of or changes to that behavior through plugins + """ + config = None + importer = None + workingDir = None + selector = None + suiteClass = None + + def __init__(self, config=None, importer=None, workingDir=None, + selector=None): + """Initialize a test loader. + + Parameters (all optional): + + * config: provide a `nose.config.Config`_ or other config class + instance; if not provided a `nose.config.Config`_ with + default values is used. + * importer: provide an importer instance that implements + `importFromPath`. If not provided, a + `nose.importer.Importer`_ is used. + * workingDir: the directory to which file and module names are + relative. If not provided, assumed to be the current working + directory. + * selector: a selector class or instance. If a class is + provided, it will be instantiated with one argument, the + current config. If not provided, a `nose.selector.Selector`_ + is used. + """ + if config is None: + config = Config() + if importer is None: + importer = Importer(config=config) + if workingDir is None: + workingDir = config.workingDir + if selector is None: + selector = defaultSelector(config) + elif isclass(selector): + selector = selector(config) + self.config = config + self.importer = importer + self.workingDir = op_normpath(op_abspath(workingDir)) + self.selector = selector + if config.addPaths: + add_path(workingDir, config) + self.suiteClass = ContextSuiteFactory(config=config) + + self._visitedPaths = set([]) + + unittest.TestLoader.__init__(self) + + def getTestCaseNames(self, testCaseClass): + """Override to select with selector, unless + config.getTestCaseNamesCompat is True + """ + if self.config.getTestCaseNamesCompat: + return unittest.TestLoader.getTestCaseNames(self, testCaseClass) + + def wanted(attr, cls=testCaseClass, sel=self.selector): + item = getattr(cls, attr, None) + if isfunction(item): + item = unbound_method(cls, item) + elif not ismethod(item): + return False + return sel.wantMethod(item) + + cases = filter(wanted, dir(testCaseClass)) + + # add runTest if nothing else picked + if not cases and hasattr(testCaseClass, 'runTest'): + cases = ['runTest'] + if self.sortTestMethodsUsing: + sort_list(cases, cmp_to_key(self.sortTestMethodsUsing)) + return cases + + def _haveVisited(self, path): + # For cases where path is None, we always pretend we haven't visited + # them. + if path is None: + return False + + return path in self._visitedPaths + + def _addVisitedPath(self, path): + if path is not None: + self._visitedPaths.add(path) + + def loadTestsFromDir(self, path): + """Load tests from the directory at path. This is a generator + -- each suite of tests from a module or other file is yielded + and is expected to be executed before the next file is + examined. + """ + log.debug("load from dir %s", path) + plugins = self.config.plugins + plugins.beforeDirectory(path) + if self.config.addPaths: + paths_added = add_path(path, self.config) + + entries = os.listdir(path) + sort_list(entries, regex_last_key(self.config.testMatch)) + for entry in entries: + # this hard-coded initial-dot test will be removed: + # http://code.google.com/p/python-nose/issues/detail?id=82 + if entry.startswith('.'): + continue + entry_path = op_abspath(op_join(path, entry)) + is_file = op_isfile(entry_path) + wanted = False + if is_file: + is_dir = False + wanted = self.selector.wantFile(entry_path) + else: + is_dir = op_isdir(entry_path) + if is_dir: + # this hard-coded initial-underscore test will be removed: + # http://code.google.com/p/python-nose/issues/detail?id=82 + if entry.startswith('_'): + continue + wanted = self.selector.wantDirectory(entry_path) + is_package = ispackage(entry_path) + + # Python 3.3 now implements PEP 420: Implicit Namespace Packages. + # As a result, it's now possible that parent paths that have a + # segment with the same basename as our package ends up + # in module.__path__. So we have to keep track of what we've + # visited, and not-revisit them again. + if wanted and not self._haveVisited(entry_path): + self._addVisitedPath(entry_path) + if is_file: + plugins.beforeContext() + if entry.endswith('.py'): + yield self.loadTestsFromName( + entry_path, discovered=True) + else: + yield self.loadTestsFromFile(entry_path) + plugins.afterContext() + elif is_package: + # Load the entry as a package: given the full path, + # loadTestsFromName() will figure it out + yield self.loadTestsFromName( + entry_path, discovered=True) + else: + # Another test dir in this one: recurse lazily + yield self.suiteClass( + lambda: self.loadTestsFromDir(entry_path)) + tests = [] + for test in plugins.loadTestsFromDir(path): + tests.append(test) + # TODO: is this try/except needed? + try: + if tests: + yield self.suiteClass(tests) + except (KeyboardInterrupt, SystemExit): + raise + except: + yield self.suiteClass([Failure(*sys.exc_info())]) + + # pop paths + if self.config.addPaths: + for p in paths_added: + remove_path(p) + plugins.afterDirectory(path) + + def loadTestsFromFile(self, filename): + """Load tests from a non-module file. Default is to raise a + ValueError; plugins may implement `loadTestsFromFile` to + provide a list of tests loaded from the file. + """ + log.debug("Load from non-module file %s", filename) + try: + tests = [test for test in + self.config.plugins.loadTestsFromFile(filename)] + if tests: + # Plugins can yield False to indicate that they were + # unable to load tests from a file, but it was not an + # error -- the file just had no tests to load. + tests = filter(None, tests) + return self.suiteClass(tests) + else: + # Nothing was able to even try to load from this file + open(filename, 'r').close() # trigger os error + raise ValueError("Unable to load tests from file %s" + % filename) + except (KeyboardInterrupt, SystemExit): + raise + except: + exc = sys.exc_info() + return self.suiteClass( + [Failure(exc[0], exc[1], exc[2], + address=(filename, None, None))]) + + def loadTestsFromGenerator(self, generator, module): + """Lazy-load tests from a generator function. The generator function + may yield either: + + * a callable, or + * a function name resolvable within the same module + """ + def generate(g=generator, m=module): + try: + for test in g(): + test_func, arg = self.parseGeneratedTest(test) + if not callable(test_func): + test_func = getattr(m, test_func) + yield FunctionTestCase(test_func, arg=arg, descriptor=g) + except KeyboardInterrupt: + raise + except: + exc = sys.exc_info() + yield Failure(exc[0], exc[1], exc[2], + address=test_address(generator)) + return self.suiteClass(generate, context=generator, can_split=False) + + def loadTestsFromGeneratorMethod(self, generator, cls): + """Lazy-load tests from a generator method. + + This is more complicated than loading from a generator function, + since a generator method may yield: + + * a function + * a bound or unbound method, or + * a method name + """ + # convert the unbound generator method + # into a bound method so it can be called below + if hasattr(generator, 'im_class'): + cls = generator.im_class + inst = cls() + method = generator.__name__ + generator = getattr(inst, method) + + def generate(g=generator, c=cls): + try: + for test in g(): + test_func, arg = self.parseGeneratedTest(test) + if not callable(test_func): + test_func = unbound_method(c, getattr(c, test_func)) + if ismethod(test_func): + yield MethodTestCase(test_func, arg=arg, descriptor=g) + elif callable(test_func): + # In this case we're forcing the 'MethodTestCase' + # to run the inline function as its test call, + # but using the generator method as the 'method of + # record' (so no need to pass it as the descriptor) + yield MethodTestCase(g, test=test_func, arg=arg) + else: + yield Failure( + TypeError, + "%s is not a callable or method" % test_func) + except KeyboardInterrupt: + raise + except: + exc = sys.exc_info() + yield Failure(exc[0], exc[1], exc[2], + address=test_address(generator)) + return self.suiteClass(generate, context=generator, can_split=False) + + def loadTestsFromModule(self, module, path=None, discovered=False): + """Load all tests from module and return a suite containing + them. If the module has been discovered and is not test-like, + the suite will be empty by default, though plugins may add + their own tests. + """ + log.debug("Load from module %s", module) + tests = [] + test_classes = [] + test_funcs = [] + # For *discovered* modules, we only load tests when the module looks + # testlike. For modules we've been directed to load, we always + # look for tests. (discovered is set to True by loadTestsFromDir) + if not discovered or self.selector.wantModule(module): + for item in dir(module): + test = getattr(module, item, None) + # print "Check %s (%s) in %s" % (item, test, module.__name__) + if isclass(test): + if self.selector.wantClass(test): + test_classes.append(test) + elif isfunction(test) and self.selector.wantFunction(test): + test_funcs.append(test) + sort_list(test_classes, lambda x: x.__name__) + sort_list(test_funcs, func_lineno) + tests = map(lambda t: self.makeTest(t, parent=module), + test_classes + test_funcs) + + # Now, descend into packages + # FIXME can or should this be lazy? + # is this syntax 2.2 compatible? + module_paths = getattr(module, '__path__', []) + + if path: + path = os.path.normcase(os.path.realpath(path)) + + for module_path in module_paths: + log.debug("Load tests from module path %s?", module_path) + log.debug("path: %s os.path.realpath(%s): %s", + path, os.path.normcase(module_path), + os.path.realpath(os.path.normcase(module_path))) + if (self.config.traverseNamespace or not path) or \ + os.path.realpath( + os.path.normcase(module_path)).startswith(path): + # Egg files can be on sys.path, so make sure the path is a + # directory before trying to load from it. + if os.path.isdir(module_path): + tests.extend(self.loadTestsFromDir(module_path)) + + for test in self.config.plugins.loadTestsFromModule(module, path): + tests.append(test) + + return self.suiteClass(ContextList(tests, context=module)) + + def loadTestsFromName(self, name, module=None, discovered=False): + """Load tests from the entity with the given name. + + The name may indicate a file, directory, module, or any object + within a module. See `nose.util.split_test_name` for details on + test name parsing. + """ + # FIXME refactor this method into little bites? + log.debug("load from %s (%s)", name, module) + + suite = self.suiteClass + + # give plugins first crack + plug_tests = self.config.plugins.loadTestsFromName(name, module) + if plug_tests: + return suite(plug_tests) + + addr = TestAddress(name, workingDir=self.workingDir) + if module: + # Two cases: + # name is class.foo + # The addr will be incorrect, since it thinks class.foo is + # a dotted module name. It's actually a dotted attribute + # name. In this case we want to use the full submitted + # name as the name to load from the module. + # name is module:class.foo + # The addr will be correct. The part we want is the part after + # the :, which is in addr.call. + if addr.call: + name = addr.call + parent, obj = self.resolve(name, module) + if (isclass(parent) + and getattr(parent, '__module__', None) != module.__name__ + and not isinstance(obj, Failure)): + parent = transplant_class(parent, module.__name__) + obj = getattr(parent, obj.__name__) + log.debug("parent %s obj %s module %s", parent, obj, module) + if isinstance(obj, Failure): + return suite([obj]) + else: + return suite(ContextList([self.makeTest(obj, parent)], + context=parent)) + else: + if addr.module: + try: + if addr.filename is None: + module = resolve_name(addr.module) + else: + self.config.plugins.beforeImport( + addr.filename, addr.module) + # FIXME: to support module.name names, + # do what resolve-name does and keep trying to + # import, popping tail of module into addr.call, + # until we either get an import or run out of + # module parts + try: + module = self.importer.importFromPath( + addr.filename, addr.module) + finally: + self.config.plugins.afterImport( + addr.filename, addr.module) + except (KeyboardInterrupt, SystemExit): + raise + except: + exc = sys.exc_info() + return suite([Failure(exc[0], exc[1], exc[2], + address=addr.totuple())]) + if addr.call: + return self.loadTestsFromName(addr.call, module) + else: + return self.loadTestsFromModule( + module, addr.filename, + discovered=discovered) + elif addr.filename: + path = addr.filename + if addr.call: + package = getpackage(path) + if package is None: + return suite([ + Failure(ValueError, + "Can't find callable %s in file %s: " + "file is not a python module" % + (addr.call, path), + address=addr.totuple())]) + return self.loadTestsFromName(addr.call, module=package) + else: + if op_isdir(path): + # In this case we *can* be lazy since we know + # that each module in the dir will be fully + # loaded before its tests are executed; we + # also know that we're not going to be asked + # to load from . and ./some_module.py *as part + # of this named test load* + return LazySuite( + lambda: self.loadTestsFromDir(path)) + elif op_isfile(path): + return self.loadTestsFromFile(path) + else: + return suite([ + Failure(OSError, "No such file %s" % path, + address=addr.totuple())]) + else: + # just a function? what to do? I think it can only be + # handled when module is not None + return suite([ + Failure(ValueError, "Unresolvable test name %s" % name, + address=addr.totuple())]) + + def loadTestsFromNames(self, names, module=None): + """Load tests from all names, returning a suite containing all + tests. + """ + plug_res = self.config.plugins.loadTestsFromNames(names, module) + if plug_res: + suite, names = plug_res + if suite: + return self.suiteClass([ + self.suiteClass(suite), + unittest.TestLoader.loadTestsFromNames(self, names, module) + ]) + return unittest.TestLoader.loadTestsFromNames(self, names, module) + + def loadTestsFromTestCase(self, testCaseClass): + """Load tests from a unittest.TestCase subclass. + """ + cases = [] + plugins = self.config.plugins + for case in plugins.loadTestsFromTestCase(testCaseClass): + cases.append(case) + # For efficiency in the most common case, just call and return from + # super. This avoids having to extract cases and rebuild a context + # suite when there are no plugin-contributed cases. + if not cases: + return super(TestLoader, self).loadTestsFromTestCase(testCaseClass) + cases.extend( + [case for case in + super(TestLoader, self).loadTestsFromTestCase(testCaseClass)]) + return self.suiteClass(cases) + + def loadTestsFromTestClass(self, cls): + """Load tests from a test class that is *not* a unittest.TestCase + subclass. + + In this case, we can't depend on the class's `__init__` taking method + name arguments, so we have to compose a MethodTestCase for each + method in the class that looks testlike. + """ + def wanted(attr, cls=cls, sel=self.selector): + item = getattr(cls, attr, None) + if isfunction(item): + item = unbound_method(cls, item) + elif not ismethod(item): + return False + return sel.wantMethod(item) + cases = [self.makeTest(getattr(cls, case), cls) + for case in filter(wanted, dir(cls))] + for test in self.config.plugins.loadTestsFromTestClass(cls): + cases.append(test) + return self.suiteClass(ContextList(cases, context=cls)) + + def makeTest(self, obj, parent=None): + try: + return self._makeTest(obj, parent) + except (KeyboardInterrupt, SystemExit): + raise + except: + exc = sys.exc_info() + try: + addr = test_address(obj) + except KeyboardInterrupt: + raise + except: + addr = None + return Failure(exc[0], exc[1], exc[2], address=addr) + + def _makeTest(self, obj, parent=None): + """Given a test object and its parent, return a test case + or test suite. + """ + plug_tests = [] + try: + addr = test_address(obj) + except KeyboardInterrupt: + raise + except: + addr = None + for test in self.config.plugins.makeTest(obj, parent): + plug_tests.append(test) + # TODO: is this try/except needed? + try: + if plug_tests: + return self.suiteClass(plug_tests) + except (KeyboardInterrupt, SystemExit): + raise + except: + exc = sys.exc_info() + return Failure(exc[0], exc[1], exc[2], address=addr) + + if isfunction(obj) and parent and not isinstance(parent, types.ModuleType): + # This is a Python 3.x 'unbound method'. Wrap it with its + # associated class.. + obj = unbound_method(parent, obj) + + if isinstance(obj, unittest.TestCase): + return obj + elif isclass(obj): + if parent and obj.__module__ != parent.__name__: + obj = transplant_class(obj, parent.__name__) + if issubclass(obj, unittest.TestCase): + return self.loadTestsFromTestCase(obj) + else: + return self.loadTestsFromTestClass(obj) + elif ismethod(obj): + if parent is None: + parent = obj.__class__ + if issubclass(parent, unittest.TestCase): + return parent(obj.__name__) + else: + if isgenerator(obj): + return self.loadTestsFromGeneratorMethod(obj, parent) + else: + return MethodTestCase(obj) + elif isfunction(obj): + if parent and obj.__module__ != parent.__name__: + obj = transplant_func(obj, parent.__name__) + if isgenerator(obj): + return self.loadTestsFromGenerator(obj, parent) + else: + return FunctionTestCase(obj) + else: + return Failure(TypeError, + "Can't make a test from %s" % obj, + address=addr) + + def resolve(self, name, module): + """Resolve name within module + """ + obj = module + parts = name.split('.') + for part in parts: + parent, obj = obj, getattr(obj, part, None) + if obj is None: + # no such test + obj = Failure(ValueError, "No such test %s" % name) + return parent, obj + + def parseGeneratedTest(self, test): + """Given the yield value of a test generator, return a func and args. + + This is used in the two loadTestsFromGenerator* methods. + + """ + if not isinstance(test, tuple): # yield test + test_func, arg = (test, tuple()) + elif len(test) == 1: # yield (test,) + test_func, arg = (test[0], tuple()) + else: # yield test, foo, bar, ... + assert len(test) > 1 # sanity check + test_func, arg = (test[0], test[1:]) + return test_func, arg + +defaultTestLoader = TestLoader + diff --git a/lib/spack/external/nose/plugins/__init__.py b/lib/spack/external/nose/plugins/__init__.py new file mode 100644 index 00000000000..08ee8f3230b --- /dev/null +++ b/lib/spack/external/nose/plugins/__init__.py @@ -0,0 +1,190 @@ +""" +Writing Plugins +--------------- + +nose supports plugins for test collection, selection, observation and +reporting. There are two basic rules for plugins: + +* Plugin classes should subclass :class:`nose.plugins.Plugin`. + +* Plugins may implement any of the methods described in the class + :doc:`IPluginInterface ` in nose.plugins.base. Please note that + this class is for documentary purposes only; plugins may not subclass + IPluginInterface. + +Hello World +=========== + +Here's a basic plugin. It doesn't do much so read on for more ideas or dive +into the :doc:`IPluginInterface ` to see all available hooks. + +.. code-block:: python + + import logging + import os + + from nose.plugins import Plugin + + log = logging.getLogger('nose.plugins.helloworld') + + class HelloWorld(Plugin): + name = 'helloworld' + + def options(self, parser, env=os.environ): + super(HelloWorld, self).options(parser, env=env) + + def configure(self, options, conf): + super(HelloWorld, self).configure(options, conf) + if not self.enabled: + return + + def finalize(self, result): + log.info('Hello pluginized world!') + +Registering +=========== + +.. Note:: + Important note: the following applies only to the default + plugin manager. Other plugin managers may use different means to + locate and load plugins. + +For nose to find a plugin, it must be part of a package that uses +setuptools_, and the plugin must be included in the entry points defined +in the setup.py for the package: + +.. code-block:: python + + setup(name='Some plugin', + # ... + entry_points = { + 'nose.plugins.0.10': [ + 'someplugin = someplugin:SomePlugin' + ] + }, + # ... + ) + +Once the package is installed with install or develop, nose will be able +to load the plugin. + +.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools + +Registering a plugin without setuptools +======================================= + +It is currently possible to register a plugin programmatically by +creating a custom nose runner like this : + +.. code-block:: python + + import nose + from yourplugin import YourPlugin + + if __name__ == '__main__': + nose.main(addplugins=[YourPlugin()]) + +Defining options +================ + +All plugins must implement the methods ``options(self, parser, env)`` +and ``configure(self, options, conf)``. Subclasses of nose.plugins.Plugin +that want the standard options should call the superclass methods. + +nose uses optparse.OptionParser from the standard library to parse +arguments. A plugin's ``options()`` method receives a parser +instance. It's good form for a plugin to use that instance only to add +additional arguments that take only long arguments (--like-this). Most +of nose's built-in arguments get their default value from an environment +variable. + +A plugin's ``configure()`` method receives the parsed ``OptionParser`` options +object, as well as the current config object. Plugins should configure their +behavior based on the user-selected settings, and may raise exceptions +if the configured behavior is nonsensical. + +Logging +======= + +nose uses the logging classes from the standard library. To enable users +to view debug messages easily, plugins should use ``logging.getLogger()`` to +acquire a logger in the ``nose.plugins`` namespace. + +Recipes +======= + +* Writing a plugin that monitors or controls test result output + + Implement any or all of ``addError``, ``addFailure``, etc., to monitor test + results. If you also want to monitor output, implement + ``setOutputStream`` and keep a reference to the output stream. If you + want to prevent the builtin ``TextTestResult`` output, implement + ``setOutputSteam`` and *return a dummy stream*. The default output will go + to the dummy stream, while you send your desired output to the real stream. + + Example: `examples/html_plugin/htmlplug.py`_ + +* Writing a plugin that handles exceptions + + Subclass :doc:`ErrorClassPlugin `. + + Examples: :doc:`nose.plugins.deprecated `, + :doc:`nose.plugins.skip ` + +* Writing a plugin that adds detail to error reports + + Implement ``formatError`` and/or ``formatFailure``. The error tuple + you return (error class, error message, traceback) will replace the + original error tuple. + + Examples: :doc:`nose.plugins.capture `, + :doc:`nose.plugins.failuredetail ` + +* Writing a plugin that loads tests from files other than python modules + + Implement ``wantFile`` and ``loadTestsFromFile``. In ``wantFile``, + return True for files that you want to examine for tests. In + ``loadTestsFromFile``, for those files, return an iterable + containing TestCases (or yield them as you find them; + ``loadTestsFromFile`` may also be a generator). + + Example: :doc:`nose.plugins.doctests ` + +* Writing a plugin that prints a report + + Implement ``begin`` if you need to perform setup before testing + begins. Implement ``report`` and output your report to the provided stream. + + Examples: :doc:`nose.plugins.cover `, :doc:`nose.plugins.prof ` + +* Writing a plugin that selects or rejects tests + + Implement any or all ``want*`` methods. Return False to reject the test + candidate, True to accept it -- which means that the test candidate + will pass through the rest of the system, so you must be prepared to + load tests from it if tests can't be loaded by the core loader or + another plugin -- and None if you don't care. + + Examples: :doc:`nose.plugins.attrib `, + :doc:`nose.plugins.doctests `, :doc:`nose.plugins.testid ` + + +More Examples +============= + +See any builtin plugin or example plugin in the examples_ directory in +the nose source distribution. There is a list of third-party plugins +`on jottit`_. + +.. _examples/html_plugin/htmlplug.py: http://python-nose.googlecode.com/svn/trunk/examples/html_plugin/htmlplug.py +.. _examples: http://python-nose.googlecode.com/svn/trunk/examples +.. _on jottit: http://nose-plugins.jottit.com/ + +""" +from nose.plugins.base import Plugin +from nose.plugins.manager import * +from nose.plugins.plugintest import PluginTester + +if __name__ == '__main__': + import doctest + doctest.testmod() diff --git a/lib/spack/external/nose/plugins/allmodules.py b/lib/spack/external/nose/plugins/allmodules.py new file mode 100644 index 00000000000..1ccd7773a72 --- /dev/null +++ b/lib/spack/external/nose/plugins/allmodules.py @@ -0,0 +1,45 @@ +"""Use the AllModules plugin by passing ``--all-modules`` or setting the +NOSE_ALL_MODULES environment variable to enable collection and execution of +tests in all python modules. Normal nose behavior is to look for tests only in +modules that match testMatch. + +More information: :doc:`../doc_tests/test_allmodules/test_allmodules` + +.. warning :: + + This plugin can have surprising interactions with plugins that load tests + from what nose normally considers non-test modules, such as + the :doc:`doctest plugin `. This is because any given + object in a module can't be loaded both by a plugin and the normal nose + :class:`test loader `. Also, if you have functions + or classes in non-test modules that look like tests but aren't, you will + likely see errors as nose attempts to run them as tests. + +""" + +import os +from nose.plugins.base import Plugin + +class AllModules(Plugin): + """Collect tests from all python modules. + """ + def options(self, parser, env): + """Register commandline options. + """ + env_opt = 'NOSE_ALL_MODULES' + parser.add_option('--all-modules', + action="store_true", + dest=self.enableOpt, + default=env.get(env_opt), + help="Enable plugin %s: %s [%s]" % + (self.__class__.__name__, self.help(), env_opt)) + + def wantFile(self, file): + """Override to return True for all files ending with .py""" + # always want .py files + if file.endswith('.py'): + return True + + def wantModule(self, module): + """Override return True for all modules""" + return True diff --git a/lib/spack/external/nose/plugins/attrib.py b/lib/spack/external/nose/plugins/attrib.py new file mode 100644 index 00000000000..3d4422a23ae --- /dev/null +++ b/lib/spack/external/nose/plugins/attrib.py @@ -0,0 +1,286 @@ +"""Attribute selector plugin. + +Oftentimes when testing you will want to select tests based on +criteria rather then simply by filename. For example, you might want +to run all tests except for the slow ones. You can do this with the +Attribute selector plugin by setting attributes on your test methods. +Here is an example: + +.. code-block:: python + + def test_big_download(): + import urllib + # commence slowness... + + test_big_download.slow = 1 + +Once you've assigned an attribute ``slow = 1`` you can exclude that +test and all other tests having the slow attribute by running :: + + $ nosetests -a '!slow' + +There is also a decorator available for you that will set attributes. +Here's how to set ``slow=1`` like above with the decorator: + +.. code-block:: python + + from nose.plugins.attrib import attr + @attr('slow') + def test_big_download(): + import urllib + # commence slowness... + +And here's how to set an attribute with a specific value: + +.. code-block:: python + + from nose.plugins.attrib import attr + @attr(speed='slow') + def test_big_download(): + import urllib + # commence slowness... + +This test could be run with :: + + $ nosetests -a speed=slow + +In Python 2.6 and higher, ``@attr`` can be used on a class to set attributes +on all its test methods at once. For example: + +.. code-block:: python + + from nose.plugins.attrib import attr + @attr(speed='slow') + class MyTestCase: + def test_long_integration(self): + pass + def test_end_to_end_something(self): + pass + +Below is a reference to the different syntaxes available. + +Simple syntax +------------- + +Examples of using the ``-a`` and ``--attr`` options: + +* ``nosetests -a status=stable`` + Only runs tests with attribute "status" having value "stable" + +* ``nosetests -a priority=2,status=stable`` + Runs tests having both attributes and values + +* ``nosetests -a priority=2 -a slow`` + Runs tests that match either attribute + +* ``nosetests -a tags=http`` + If a test's ``tags`` attribute was a list and it contained the value + ``http`` then it would be run + +* ``nosetests -a slow`` + Runs tests with the attribute ``slow`` if its value does not equal False + (False, [], "", etc...) + +* ``nosetests -a '!slow'`` + Runs tests that do NOT have the attribute ``slow`` or have a ``slow`` + attribute that is equal to False + **NOTE**: + if your shell (like bash) interprets '!' as a special character make sure to + put single quotes around it. + +Expression Evaluation +--------------------- + +Examples using the ``-A`` and ``--eval-attr`` options: + +* ``nosetests -A "not slow"`` + Evaluates the Python expression "not slow" and runs the test if True + +* ``nosetests -A "(priority > 5) and not slow"`` + Evaluates a complex Python expression and runs the test if True + +""" +import inspect +import logging +import os +import sys +from inspect import isfunction +from nose.plugins.base import Plugin +from nose.util import tolist + +log = logging.getLogger('nose.plugins.attrib') +compat_24 = sys.version_info >= (2, 4) + +def attr(*args, **kwargs): + """Decorator that adds attributes to classes or functions + for use with the Attribute (-a) plugin. + """ + def wrap_ob(ob): + for name in args: + setattr(ob, name, True) + for name, value in kwargs.iteritems(): + setattr(ob, name, value) + return ob + return wrap_ob + +def get_method_attr(method, cls, attr_name, default = False): + """Look up an attribute on a method/ function. + If the attribute isn't found there, looking it up in the + method's class, if any. + """ + Missing = object() + value = getattr(method, attr_name, Missing) + if value is Missing and cls is not None: + value = getattr(cls, attr_name, Missing) + if value is Missing: + return default + return value + + +class ContextHelper: + """Object that can act as context dictionary for eval and looks up + names as attributes on a method/ function and its class. + """ + def __init__(self, method, cls): + self.method = method + self.cls = cls + + def __getitem__(self, name): + return get_method_attr(self.method, self.cls, name) + + +class AttributeSelector(Plugin): + """Selects test cases to be run based on their attributes. + """ + + def __init__(self): + Plugin.__init__(self) + self.attribs = [] + + def options(self, parser, env): + """Register command line options""" + parser.add_option("-a", "--attr", + dest="attr", action="append", + default=env.get('NOSE_ATTR'), + metavar="ATTR", + help="Run only tests that have attributes " + "specified by ATTR [NOSE_ATTR]") + # disable in < 2.4: eval can't take needed args + if compat_24: + parser.add_option("-A", "--eval-attr", + dest="eval_attr", metavar="EXPR", action="append", + default=env.get('NOSE_EVAL_ATTR'), + help="Run only tests for whose attributes " + "the Python expression EXPR evaluates " + "to True [NOSE_EVAL_ATTR]") + + def configure(self, options, config): + """Configure the plugin and system, based on selected options. + + attr and eval_attr may each be lists. + + self.attribs will be a list of lists of tuples. In that list, each + list is a group of attributes, all of which must match for the rule to + match. + """ + self.attribs = [] + + # handle python eval-expression parameter + if compat_24 and options.eval_attr: + eval_attr = tolist(options.eval_attr) + for attr in eval_attr: + # "" + # -> eval(expr) in attribute context must be True + def eval_in_context(expr, obj, cls): + return eval(expr, None, ContextHelper(obj, cls)) + self.attribs.append([(attr, eval_in_context)]) + + # attribute requirements are a comma separated list of + # 'key=value' pairs + if options.attr: + std_attr = tolist(options.attr) + for attr in std_attr: + # all attributes within an attribute group must match + attr_group = [] + for attrib in attr.strip().split(","): + # don't die on trailing comma + if not attrib: + continue + items = attrib.split("=", 1) + if len(items) > 1: + # "name=value" + # -> 'str(obj.name) == value' must be True + key, value = items + else: + key = items[0] + if key[0] == "!": + # "!name" + # 'bool(obj.name)' must be False + key = key[1:] + value = False + else: + # "name" + # -> 'bool(obj.name)' must be True + value = True + attr_group.append((key, value)) + self.attribs.append(attr_group) + if self.attribs: + self.enabled = True + + def validateAttrib(self, method, cls = None): + """Verify whether a method has the required attributes + The method is considered a match if it matches all attributes + for any attribute group. + .""" + # TODO: is there a need for case-sensitive value comparison? + any = False + for group in self.attribs: + match = True + for key, value in group: + attr = get_method_attr(method, cls, key) + if callable(value): + if not value(key, method, cls): + match = False + break + elif value is True: + # value must exist and be True + if not bool(attr): + match = False + break + elif value is False: + # value must not exist or be False + if bool(attr): + match = False + break + elif type(attr) in (list, tuple): + # value must be found in the list attribute + if not str(value).lower() in [str(x).lower() + for x in attr]: + match = False + break + else: + # value must match, convert to string and compare + if (value != attr + and str(value).lower() != str(attr).lower()): + match = False + break + any = any or match + if any: + # not True because we don't want to FORCE the selection of the + # item, only say that it is acceptable + return None + return False + + def wantFunction(self, function): + """Accept the function if its attributes match. + """ + return self.validateAttrib(function) + + def wantMethod(self, method): + """Accept the method if its attributes match. + """ + try: + cls = method.im_class + except AttributeError: + return False + return self.validateAttrib(method, cls) diff --git a/lib/spack/external/nose/plugins/base.py b/lib/spack/external/nose/plugins/base.py new file mode 100644 index 00000000000..f09beb696f0 --- /dev/null +++ b/lib/spack/external/nose/plugins/base.py @@ -0,0 +1,725 @@ +import os +import textwrap +from optparse import OptionConflictError +from warnings import warn +from nose.util import tolist + +class Plugin(object): + """Base class for nose plugins. It's recommended but not *necessary* to + subclass this class to create a plugin, but all plugins *must* implement + `options(self, parser, env)` and `configure(self, options, conf)`, and + must have the attributes `enabled`, `name` and `score`. The `name` + attribute may contain hyphens ('-'). + + Plugins should not be enabled by default. + + Subclassing Plugin (and calling the superclass methods in + __init__, configure, and options, if you override them) will give + your plugin some friendly default behavior: + + * A --with-$name option will be added to the command line interface + to enable the plugin, and a corresponding environment variable + will be used as the default value. The plugin class's docstring + will be used as the help for this option. + * The plugin will not be enabled unless this option is selected by + the user. + """ + can_configure = False + enabled = False + enableOpt = None + name = None + score = 100 + + def __init__(self): + if self.name is None: + self.name = self.__class__.__name__.lower() + if self.enableOpt is None: + self.enableOpt = "enable_plugin_%s" % self.name.replace('-', '_') + + def addOptions(self, parser, env=None): + """Add command-line options for this plugin. + + The base plugin class adds --with-$name by default, used to enable the + plugin. + + .. warning :: Don't implement addOptions unless you want to override + all default option handling behavior, including + warnings for conflicting options. Implement + :meth:`options + ` + instead. + """ + self.add_options(parser, env) + + def add_options(self, parser, env=None): + """Non-camel-case version of func name for backwards compatibility. + + .. warning :: + + DEPRECATED: Do not use this method, + use :meth:`options ` + instead. + + """ + # FIXME raise deprecation warning if wasn't called by wrapper + if env is None: + env = os.environ + try: + self.options(parser, env) + self.can_configure = True + except OptionConflictError, e: + warn("Plugin %s has conflicting option string: %s and will " + "be disabled" % (self, e), RuntimeWarning) + self.enabled = False + self.can_configure = False + + def options(self, parser, env): + """Register commandline options. + + Implement this method for normal options behavior with protection from + OptionConflictErrors. If you override this method and want the default + --with-$name option to be registered, be sure to call super(). + """ + env_opt = 'NOSE_WITH_%s' % self.name.upper() + env_opt = env_opt.replace('-', '_') + parser.add_option("--with-%s" % self.name, + action="store_true", + dest=self.enableOpt, + default=env.get(env_opt), + help="Enable plugin %s: %s [%s]" % + (self.__class__.__name__, self.help(), env_opt)) + + def configure(self, options, conf): + """Configure the plugin and system, based on selected options. + + The base plugin class sets the plugin to enabled if the enable option + for the plugin (self.enableOpt) is true. + """ + if not self.can_configure: + return + self.conf = conf + if hasattr(options, self.enableOpt): + self.enabled = getattr(options, self.enableOpt) + + def help(self): + """Return help for this plugin. This will be output as the help + section of the --with-$name option that enables the plugin. + """ + if self.__class__.__doc__: + # doc sections are often indented; compress the spaces + return textwrap.dedent(self.__class__.__doc__) + return "(no help available)" + + # Compatiblity shim + def tolist(self, val): + warn("Plugin.tolist is deprecated. Use nose.util.tolist instead", + DeprecationWarning) + return tolist(val) + + +class IPluginInterface(object): + """ + IPluginInterface describes the plugin API. Do not subclass or use this + class directly. + """ + def __new__(cls, *arg, **kw): + raise TypeError("IPluginInterface class is for documentation only") + + def addOptions(self, parser, env): + """Called to allow plugin to register command-line options with the + parser. DO NOT return a value from this method unless you want to stop + all other plugins from setting their options. + + .. warning :: + + DEPRECATED -- implement + :meth:`options ` instead. + """ + pass + add_options = addOptions + add_options.deprecated = True + + def addDeprecated(self, test): + """Called when a deprecated test is seen. DO NOT return a value + unless you want to stop other plugins from seeing the deprecated + test. + + .. warning :: DEPRECATED -- check error class in addError instead + """ + pass + addDeprecated.deprecated = True + + def addError(self, test, err): + """Called when a test raises an uncaught exception. DO NOT return a + value unless you want to stop other plugins from seeing that the + test has raised an error. + + :param test: the test case + :type test: :class:`nose.case.Test` + :param err: sys.exc_info() tuple + :type err: 3-tuple + """ + pass + addError.changed = True + + def addFailure(self, test, err): + """Called when a test fails. DO NOT return a value unless you + want to stop other plugins from seeing that the test has failed. + + :param test: the test case + :type test: :class:`nose.case.Test` + :param err: 3-tuple + :type err: sys.exc_info() tuple + """ + pass + addFailure.changed = True + + def addSkip(self, test): + """Called when a test is skipped. DO NOT return a value unless + you want to stop other plugins from seeing the skipped test. + + .. warning:: DEPRECATED -- check error class in addError instead + """ + pass + addSkip.deprecated = True + + def addSuccess(self, test): + """Called when a test passes. DO NOT return a value unless you + want to stop other plugins from seeing the passing test. + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + addSuccess.changed = True + + def afterContext(self): + """Called after a context (generally a module) has been + lazy-loaded, imported, setup, had its tests loaded and + executed, and torn down. + """ + pass + afterContext._new = True + + def afterDirectory(self, path): + """Called after all tests have been loaded from directory at path + and run. + + :param path: the directory that has finished processing + :type path: string + """ + pass + afterDirectory._new = True + + def afterImport(self, filename, module): + """Called after module is imported from filename. afterImport + is called even if the import failed. + + :param filename: The file that was loaded + :type filename: string + :param module: The name of the module + :type module: string + """ + pass + afterImport._new = True + + def afterTest(self, test): + """Called after the test has been run and the result recorded + (after stopTest). + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + afterTest._new = True + + def beforeContext(self): + """Called before a context (generally a module) is + examined. Because the context is not yet loaded, plugins don't + get to know what the context is; so any context operations + should use a stack that is pushed in `beforeContext` and popped + in `afterContext` to ensure they operate symmetrically. + + `beforeContext` and `afterContext` are mainly useful for tracking + and restoring global state around possible changes from within a + context, whatever the context may be. If you need to operate on + contexts themselves, see `startContext` and `stopContext`, which + are passed the context in question, but are called after + it has been loaded (imported in the module case). + """ + pass + beforeContext._new = True + + def beforeDirectory(self, path): + """Called before tests are loaded from directory at path. + + :param path: the directory that is about to be processed + """ + pass + beforeDirectory._new = True + + def beforeImport(self, filename, module): + """Called before module is imported from filename. + + :param filename: The file that will be loaded + :param module: The name of the module found in file + :type module: string + """ + beforeImport._new = True + + def beforeTest(self, test): + """Called before the test is run (before startTest). + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + beforeTest._new = True + + def begin(self): + """Called before any tests are collected or run. Use this to + perform any setup needed before testing begins. + """ + pass + + def configure(self, options, conf): + """Called after the command line has been parsed, with the + parsed options and the config container. Here, implement any + config storage or changes to state or operation that are set + by command line options. + + DO NOT return a value from this method unless you want to + stop all other plugins from being configured. + """ + pass + + def finalize(self, result): + """Called after all report output, including output from all + plugins, has been sent to the stream. Use this to print final + test results or perform final cleanup. Return None to allow + other plugins to continue printing, or any other value to stop + them. + + :param result: test result object + + .. Note:: When tests are run under a test runner other than + :class:`nose.core.TextTestRunner`, such as + via ``python setup.py test``, this method may be called + **before** the default report output is sent. + """ + pass + + def describeTest(self, test): + """Return a test description. + + Called by :meth:`nose.case.Test.shortDescription`. + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + describeTest._new = True + + def formatError(self, test, err): + """Called in result.addError, before plugin.addError. If you + want to replace or modify the error tuple, return a new error + tuple, otherwise return err, the original error tuple. + + :param test: the test case + :type test: :class:`nose.case.Test` + :param err: sys.exc_info() tuple + :type err: 3-tuple + """ + pass + formatError._new = True + formatError.chainable = True + # test arg is not chainable + formatError.static_args = (True, False) + + def formatFailure(self, test, err): + """Called in result.addFailure, before plugin.addFailure. If you + want to replace or modify the error tuple, return a new error + tuple, otherwise return err, the original error tuple. + + :param test: the test case + :type test: :class:`nose.case.Test` + :param err: sys.exc_info() tuple + :type err: 3-tuple + """ + pass + formatFailure._new = True + formatFailure.chainable = True + # test arg is not chainable + formatFailure.static_args = (True, False) + + def handleError(self, test, err): + """Called on addError. To handle the error yourself and prevent normal + error processing, return a true value. + + :param test: the test case + :type test: :class:`nose.case.Test` + :param err: sys.exc_info() tuple + :type err: 3-tuple + """ + pass + handleError._new = True + + def handleFailure(self, test, err): + """Called on addFailure. To handle the failure yourself and + prevent normal failure processing, return a true value. + + :param test: the test case + :type test: :class:`nose.case.Test` + :param err: sys.exc_info() tuple + :type err: 3-tuple + """ + pass + handleFailure._new = True + + def loadTestsFromDir(self, path): + """Return iterable of tests from a directory. May be a + generator. Each item returned must be a runnable + unittest.TestCase (or subclass) instance or suite instance. + Return None if your plugin cannot collect any tests from + directory. + + :param path: The path to the directory. + """ + pass + loadTestsFromDir.generative = True + loadTestsFromDir._new = True + + def loadTestsFromModule(self, module, path=None): + """Return iterable of tests in a module. May be a + generator. Each item returned must be a runnable + unittest.TestCase (or subclass) instance. + Return None if your plugin cannot + collect any tests from module. + + :param module: The module object + :type module: python module + :param path: the path of the module to search, to distinguish from + namespace package modules + + .. note:: + + NEW. The ``path`` parameter will only be passed by nose 0.11 + or above. + """ + pass + loadTestsFromModule.generative = True + + def loadTestsFromName(self, name, module=None, importPath=None): + """Return tests in this file or module. Return None if you are not able + to load any tests, or an iterable if you are. May be a + generator. + + :param name: The test name. May be a file or module name plus a test + callable. Use split_test_name to split into parts. Or it might + be some crazy name of your own devising, in which case, do + whatever you want. + :param module: Module from which the name is to be loaded + :param importPath: Path from which file (must be a python module) was + found + + .. warning:: DEPRECATED: this argument will NOT be passed. + """ + pass + loadTestsFromName.generative = True + + def loadTestsFromNames(self, names, module=None): + """Return a tuple of (tests loaded, remaining names). Return + None if you are not able to load any tests. Multiple plugins + may implement loadTestsFromNames; the remaining name list from + each will be passed to the next as input. + + :param names: List of test names. + :type names: iterable + :param module: Module from which the names are to be loaded + """ + pass + loadTestsFromNames._new = True + loadTestsFromNames.chainable = True + + def loadTestsFromFile(self, filename): + """Return tests in this file. Return None if you are not + interested in loading any tests, or an iterable if you are and + can load some. May be a generator. *If you are interested in + loading tests from the file and encounter no errors, but find + no tests, yield False or return [False].* + + .. Note:: This method replaces loadTestsFromPath from the 0.9 + API. + + :param filename: The full path to the file or directory. + """ + pass + loadTestsFromFile.generative = True + loadTestsFromFile._new = True + + def loadTestsFromPath(self, path): + """ + .. warning:: DEPRECATED -- use loadTestsFromFile instead + """ + pass + loadTestsFromPath.deprecated = True + + def loadTestsFromTestCase(self, cls): + """Return tests in this test case class. Return None if you are + not able to load any tests, or an iterable if you are. May be a + generator. + + :param cls: The test case class. Must be subclass of + :class:`unittest.TestCase`. + """ + pass + loadTestsFromTestCase.generative = True + + def loadTestsFromTestClass(self, cls): + """Return tests in this test class. Class will *not* be a + unittest.TestCase subclass. Return None if you are not able to + load any tests, an iterable if you are. May be a generator. + + :param cls: The test case class. Must be **not** be subclass of + :class:`unittest.TestCase`. + """ + pass + loadTestsFromTestClass._new = True + loadTestsFromTestClass.generative = True + + def makeTest(self, obj, parent): + """Given an object and its parent, return or yield one or more + test cases. Each test must be a unittest.TestCase (or subclass) + instance. This is called before default test loading to allow + plugins to load an alternate test case or cases for an + object. May be a generator. + + :param obj: The object to be made into a test + :param parent: The parent of obj (eg, for a method, the class) + """ + pass + makeTest._new = True + makeTest.generative = True + + def options(self, parser, env): + """Called to allow plugin to register command line + options with the parser. + + DO NOT return a value from this method unless you want to stop + all other plugins from setting their options. + + :param parser: options parser instance + :type parser: :class:`ConfigParser.ConfigParser` + :param env: environment, default is os.environ + """ + pass + options._new = True + + def prepareTest(self, test): + """Called before the test is run by the test runner. Please + note the article *the* in the previous sentence: prepareTest + is called *only once*, and is passed the test case or test + suite that the test runner will execute. It is *not* called + for each individual test case. If you return a non-None value, + that return value will be run as the test. Use this hook to + wrap or decorate the test with another function. If you need + to modify or wrap individual test cases, use `prepareTestCase` + instead. + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + + def prepareTestCase(self, test): + """Prepare or wrap an individual test case. Called before + execution of the test. The test passed here is a + nose.case.Test instance; the case to be executed is in the + test attribute of the passed case. To modify the test to be + run, you should return a callable that takes one argument (the + test result object) -- it is recommended that you *do not* + side-effect the nose.case.Test instance you have been passed. + + Keep in mind that when you replace the test callable you are + replacing the run() method of the test case -- including the + exception handling and result calls, etc. + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + prepareTestCase._new = True + + def prepareTestLoader(self, loader): + """Called before tests are loaded. To replace the test loader, + return a test loader. To allow other plugins to process the + test loader, return None. Only one plugin may replace the test + loader. Only valid when using nose.TestProgram. + + :param loader: :class:`nose.loader.TestLoader` + (or other loader) instance + """ + pass + prepareTestLoader._new = True + + def prepareTestResult(self, result): + """Called before the first test is run. To use a different + test result handler for all tests than the given result, + return a test result handler. NOTE however that this handler + will only be seen by tests, that is, inside of the result + proxy system. The TestRunner and TestProgram -- whether nose's + or other -- will continue to see the original result + handler. For this reason, it is usually better to monkeypatch + the result (for instance, if you want to handle some + exceptions in a unique way). Only one plugin may replace the + result, but many may monkeypatch it. If you want to + monkeypatch and stop other plugins from doing so, monkeypatch + and return the patched result. + + :param result: :class:`nose.result.TextTestResult` + (or other result) instance + """ + pass + prepareTestResult._new = True + + def prepareTestRunner(self, runner): + """Called before tests are run. To replace the test runner, + return a test runner. To allow other plugins to process the + test runner, return None. Only valid when using nose.TestProgram. + + :param runner: :class:`nose.core.TextTestRunner` + (or other runner) instance + """ + pass + prepareTestRunner._new = True + + def report(self, stream): + """Called after all error output has been printed. Print your + plugin's report to the provided stream. Return None to allow + other plugins to print reports, any other value to stop them. + + :param stream: stream object; send your output here + :type stream: file-like object + """ + pass + + def setOutputStream(self, stream): + """Called before test output begins. To direct test output to a + new stream, return a stream object, which must implement a + `write(msg)` method. If you only want to note the stream, not + capture or redirect it, then return None. + + :param stream: stream object; send your output here + :type stream: file-like object + """ + + def startContext(self, context): + """Called before context setup and the running of tests in the + context. Note that tests have already been *loaded* from the + context before this call. + + :param context: the context about to be setup. May be a module or + class, or any other object that contains tests. + """ + pass + startContext._new = True + + def startTest(self, test): + """Called before each test is run. DO NOT return a value unless + you want to stop other plugins from seeing the test start. + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + + def stopContext(self, context): + """Called after the tests in a context have run and the + context has been torn down. + + :param context: the context that has been torn down. May be a module or + class, or any other object that contains tests. + """ + pass + stopContext._new = True + + def stopTest(self, test): + """Called after each test is run. DO NOT return a value unless + you want to stop other plugins from seeing that the test has stopped. + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + + def testName(self, test): + """Return a short test name. Called by `nose.case.Test.__str__`. + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + testName._new = True + + def wantClass(self, cls): + """Return true if you want the main test selector to collect + tests from this class, false if you don't, and None if you don't + care. + + :param cls: The class being examined by the selector + """ + pass + + def wantDirectory(self, dirname): + """Return true if you want test collection to descend into this + directory, false if you do not, and None if you don't care. + + :param dirname: Full path to directory being examined by the selector + """ + pass + + def wantFile(self, file): + """Return true if you want to collect tests from this file, + false if you do not and None if you don't care. + + Change from 0.9: The optional package parameter is no longer passed. + + :param file: Full path to file being examined by the selector + """ + pass + + def wantFunction(self, function): + """Return true to collect this function as a test, false to + prevent it from being collected, and None if you don't care. + + :param function: The function object being examined by the selector + """ + pass + + def wantMethod(self, method): + """Return true to collect this method as a test, false to + prevent it from being collected, and None if you don't care. + + :param method: The method object being examined by the selector + :type method: unbound method + """ + pass + + def wantModule(self, module): + """Return true if you want to collection to descend into this + module, false to prevent the collector from descending into the + module, and None if you don't care. + + :param module: The module object being examined by the selector + :type module: python module + """ + pass + + def wantModuleTests(self, module): + """ + .. warning:: DEPRECATED -- this method will not be called, it has + been folded into wantModule. + """ + pass + wantModuleTests.deprecated = True + diff --git a/lib/spack/external/nose/plugins/builtin.py b/lib/spack/external/nose/plugins/builtin.py new file mode 100644 index 00000000000..4fcc0018adc --- /dev/null +++ b/lib/spack/external/nose/plugins/builtin.py @@ -0,0 +1,34 @@ +""" +Lists builtin plugins. +""" +plugins = [] +builtins = ( + ('nose.plugins.attrib', 'AttributeSelector'), + ('nose.plugins.capture', 'Capture'), + ('nose.plugins.logcapture', 'LogCapture'), + ('nose.plugins.cover', 'Coverage'), + ('nose.plugins.debug', 'Pdb'), + ('nose.plugins.deprecated', 'Deprecated'), + ('nose.plugins.doctests', 'Doctest'), + ('nose.plugins.isolate', 'IsolationPlugin'), + ('nose.plugins.failuredetail', 'FailureDetail'), + ('nose.plugins.prof', 'Profile'), + ('nose.plugins.skip', 'Skip'), + ('nose.plugins.testid', 'TestId'), + ('nose.plugins.multiprocess', 'MultiProcess'), + ('nose.plugins.xunit', 'Xunit'), + ('nose.plugins.allmodules', 'AllModules'), + ('nose.plugins.collect', 'CollectOnly'), + ) + +for module, cls in builtins: + try: + plugmod = __import__(module, globals(), locals(), [cls]) + except KeyboardInterrupt: + raise + except: + continue + plug = getattr(plugmod, cls) + plugins.append(plug) + globals()[cls] = plug + diff --git a/lib/spack/external/nose/plugins/capture.py b/lib/spack/external/nose/plugins/capture.py new file mode 100644 index 00000000000..fa4e5dcaaf6 --- /dev/null +++ b/lib/spack/external/nose/plugins/capture.py @@ -0,0 +1,115 @@ +""" +This plugin captures stdout during test execution. If the test fails +or raises an error, the captured output will be appended to the error +or failure output. It is enabled by default but can be disabled with +the options ``-s`` or ``--nocapture``. + +:Options: + ``--nocapture`` + Don't capture stdout (any stdout output will be printed immediately) + +""" +import logging +import os +import sys +from nose.plugins.base import Plugin +from nose.pyversion import exc_to_unicode, force_unicode +from nose.util import ln +from StringIO import StringIO + + +log = logging.getLogger(__name__) + +class Capture(Plugin): + """ + Output capture plugin. Enabled by default. Disable with ``-s`` or + ``--nocapture``. This plugin captures stdout during test execution, + appending any output captured to the error or failure output, + should the test fail or raise an error. + """ + enabled = True + env_opt = 'NOSE_NOCAPTURE' + name = 'capture' + score = 1600 + + def __init__(self): + self.stdout = [] + self._buf = None + + def options(self, parser, env): + """Register commandline options + """ + parser.add_option( + "-s", "--nocapture", action="store_false", + default=not env.get(self.env_opt), dest="capture", + help="Don't capture stdout (any stdout output " + "will be printed immediately) [NOSE_NOCAPTURE]") + + def configure(self, options, conf): + """Configure plugin. Plugin is enabled by default. + """ + self.conf = conf + if not options.capture: + self.enabled = False + + def afterTest(self, test): + """Clear capture buffer. + """ + self.end() + self._buf = None + + def begin(self): + """Replace sys.stdout with capture buffer. + """ + self.start() # get an early handle on sys.stdout + + def beforeTest(self, test): + """Flush capture buffer. + """ + self.start() + + def formatError(self, test, err): + """Add captured output to error report. + """ + test.capturedOutput = output = self.buffer + self._buf = None + if not output: + # Don't return None as that will prevent other + # formatters from formatting and remove earlier formatters + # formats, instead return the err we got + return err + ec, ev, tb = err + return (ec, self.addCaptureToErr(ev, output), tb) + + def formatFailure(self, test, err): + """Add captured output to failure report. + """ + return self.formatError(test, err) + + def addCaptureToErr(self, ev, output): + ev = exc_to_unicode(ev) + output = force_unicode(output) + return u'\n'.join([ev, ln(u'>> begin captured stdout <<'), + output, ln(u'>> end captured stdout <<')]) + + def start(self): + self.stdout.append(sys.stdout) + self._buf = StringIO() + sys.stdout = self._buf + + def end(self): + if self.stdout: + sys.stdout = self.stdout.pop() + + def finalize(self, result): + """Restore stdout. + """ + while self.stdout: + self.end() + + def _get_buffer(self): + if self._buf is not None: + return self._buf.getvalue() + + buffer = property(_get_buffer, None, None, + """Captured stdout output.""") diff --git a/lib/spack/external/nose/plugins/collect.py b/lib/spack/external/nose/plugins/collect.py new file mode 100644 index 00000000000..6f9f0faa779 --- /dev/null +++ b/lib/spack/external/nose/plugins/collect.py @@ -0,0 +1,94 @@ +""" +This plugin bypasses the actual execution of tests, and instead just collects +test names. Fixtures are also bypassed, so running nosetests with the +collection plugin enabled should be very quick. + +This plugin is useful in combination with the testid plugin (``--with-id``). +Run both together to get an indexed list of all tests, which will enable you to +run individual tests by index number. + +This plugin is also useful for counting tests in a test suite, and making +people watching your demo think all of your tests pass. +""" +from nose.plugins.base import Plugin +from nose.case import Test +import logging +import unittest + +log = logging.getLogger(__name__) + + +class CollectOnly(Plugin): + """ + Collect and output test names only, don't run any tests. + """ + name = "collect-only" + enableOpt = 'collect_only' + + def options(self, parser, env): + """Register commandline options. + """ + parser.add_option('--collect-only', + action='store_true', + dest=self.enableOpt, + default=env.get('NOSE_COLLECT_ONLY'), + help="Enable collect-only: %s [COLLECT_ONLY]" % + (self.help())) + + def prepareTestLoader(self, loader): + """Install collect-only suite class in TestLoader. + """ + # Disable context awareness + log.debug("Preparing test loader") + loader.suiteClass = TestSuiteFactory(self.conf) + + def prepareTestCase(self, test): + """Replace actual test with dummy that always passes. + """ + # Return something that always passes + log.debug("Preparing test case %s", test) + if not isinstance(test, Test): + return + def run(result): + # We need to make these plugin calls because there won't be + # a result proxy, due to using a stripped-down test suite + self.conf.plugins.startTest(test) + result.startTest(test) + self.conf.plugins.addSuccess(test) + result.addSuccess(test) + self.conf.plugins.stopTest(test) + result.stopTest(test) + return run + + +class TestSuiteFactory: + """ + Factory for producing configured test suites. + """ + def __init__(self, conf): + self.conf = conf + + def __call__(self, tests=(), **kw): + return TestSuite(tests, conf=self.conf) + + +class TestSuite(unittest.TestSuite): + """ + Basic test suite that bypasses most proxy and plugin calls, but does + wrap tests in a nose.case.Test so prepareTestCase will be called. + """ + def __init__(self, tests=(), conf=None): + self.conf = conf + # Exec lazy suites: makes discovery depth-first + if callable(tests): + tests = tests() + log.debug("TestSuite(%r)", tests) + unittest.TestSuite.__init__(self, tests) + + def addTest(self, test): + log.debug("Add test %s", test) + if isinstance(test, unittest.TestSuite): + self._tests.append(test) + else: + self._tests.append(Test(test, config=self.conf)) + diff --git a/lib/spack/external/nose/plugins/cover.py b/lib/spack/external/nose/plugins/cover.py new file mode 100644 index 00000000000..fbe2e30dcdd --- /dev/null +++ b/lib/spack/external/nose/plugins/cover.py @@ -0,0 +1,271 @@ +"""If you have Ned Batchelder's coverage_ module installed, you may activate a +coverage report with the ``--with-coverage`` switch or NOSE_WITH_COVERAGE +environment variable. The coverage report will cover any python source module +imported after the start of the test run, excluding modules that match +testMatch. If you want to include those modules too, use the ``--cover-tests`` +switch, or set the NOSE_COVER_TESTS environment variable to a true value. To +restrict the coverage report to modules from a particular package or packages, +use the ``--cover-package`` switch or the NOSE_COVER_PACKAGE environment +variable. + +.. _coverage: http://www.nedbatchelder.com/code/modules/coverage.html +""" +import logging +import re +import sys +import StringIO +from nose.plugins.base import Plugin +from nose.util import src, tolist + +log = logging.getLogger(__name__) + + +class Coverage(Plugin): + """ + Activate a coverage report using Ned Batchelder's coverage module. + """ + coverTests = False + coverPackages = None + coverInstance = None + coverErase = False + coverMinPercentage = None + score = 200 + status = {} + + def options(self, parser, env): + """ + Add options to command line. + """ + super(Coverage, self).options(parser, env) + parser.add_option("--cover-package", action="append", + default=env.get('NOSE_COVER_PACKAGE'), + metavar="PACKAGE", + dest="cover_packages", + help="Restrict coverage output to selected packages " + "[NOSE_COVER_PACKAGE]") + parser.add_option("--cover-erase", action="store_true", + default=env.get('NOSE_COVER_ERASE'), + dest="cover_erase", + help="Erase previously collected coverage " + "statistics before run") + parser.add_option("--cover-tests", action="store_true", + dest="cover_tests", + default=env.get('NOSE_COVER_TESTS'), + help="Include test modules in coverage report " + "[NOSE_COVER_TESTS]") + parser.add_option("--cover-min-percentage", action="store", + dest="cover_min_percentage", + default=env.get('NOSE_COVER_MIN_PERCENTAGE'), + help="Minimum percentage of coverage for tests " + "to pass [NOSE_COVER_MIN_PERCENTAGE]") + parser.add_option("--cover-inclusive", action="store_true", + dest="cover_inclusive", + default=env.get('NOSE_COVER_INCLUSIVE'), + help="Include all python files under working " + "directory in coverage report. Useful for " + "discovering holes in test coverage if not all " + "files are imported by the test suite. " + "[NOSE_COVER_INCLUSIVE]") + parser.add_option("--cover-html", action="store_true", + default=env.get('NOSE_COVER_HTML'), + dest='cover_html', + help="Produce HTML coverage information") + parser.add_option('--cover-html-dir', action='store', + default=env.get('NOSE_COVER_HTML_DIR', 'cover'), + dest='cover_html_dir', + metavar='DIR', + help='Produce HTML coverage information in dir') + parser.add_option("--cover-branches", action="store_true", + default=env.get('NOSE_COVER_BRANCHES'), + dest="cover_branches", + help="Include branch coverage in coverage report " + "[NOSE_COVER_BRANCHES]") + parser.add_option("--cover-xml", action="store_true", + default=env.get('NOSE_COVER_XML'), + dest="cover_xml", + help="Produce XML coverage information") + parser.add_option("--cover-xml-file", action="store", + default=env.get('NOSE_COVER_XML_FILE', 'coverage.xml'), + dest="cover_xml_file", + metavar="FILE", + help="Produce XML coverage information in file") + + def configure(self, options, conf): + """ + Configure plugin. + """ + try: + self.status.pop('active') + except KeyError: + pass + super(Coverage, self).configure(options, conf) + if self.enabled: + try: + import coverage + if not hasattr(coverage, 'coverage'): + raise ImportError("Unable to import coverage module") + except ImportError: + log.error("Coverage not available: " + "unable to import coverage module") + self.enabled = False + return + self.conf = conf + self.coverErase = options.cover_erase + self.coverTests = options.cover_tests + self.coverPackages = [] + if options.cover_packages: + if isinstance(options.cover_packages, (list, tuple)): + cover_packages = options.cover_packages + else: + cover_packages = [options.cover_packages] + for pkgs in [tolist(x) for x in cover_packages]: + self.coverPackages.extend(pkgs) + self.coverInclusive = options.cover_inclusive + if self.coverPackages: + log.info("Coverage report will include only packages: %s", + self.coverPackages) + self.coverHtmlDir = None + if options.cover_html: + self.coverHtmlDir = options.cover_html_dir + log.debug('Will put HTML coverage report in %s', self.coverHtmlDir) + self.coverBranches = options.cover_branches + self.coverXmlFile = None + if options.cover_min_percentage: + self.coverMinPercentage = int(options.cover_min_percentage.rstrip('%')) + if options.cover_xml: + self.coverXmlFile = options.cover_xml_file + log.debug('Will put XML coverage report in %s', self.coverXmlFile) + if self.enabled: + self.status['active'] = True + self.coverInstance = coverage.coverage(auto_data=False, + branch=self.coverBranches, data_suffix=conf.worker, + source=self.coverPackages) + self.coverInstance._warn_no_data = False + self.coverInstance.is_worker = conf.worker + self.coverInstance.exclude('#pragma[: ]+[nN][oO] [cC][oO][vV][eE][rR]') + + log.debug("Coverage begin") + self.skipModules = sys.modules.keys()[:] + if self.coverErase: + log.debug("Clearing previously collected coverage statistics") + self.coverInstance.combine() + self.coverInstance.erase() + + if not self.coverInstance.is_worker: + self.coverInstance.load() + self.coverInstance.start() + + + def beforeTest(self, *args, **kwargs): + """ + Begin recording coverage information. + """ + + if self.coverInstance.is_worker: + self.coverInstance.load() + self.coverInstance.start() + + def afterTest(self, *args, **kwargs): + """ + Stop recording coverage information. + """ + + if self.coverInstance.is_worker: + self.coverInstance.stop() + self.coverInstance.save() + + + def report(self, stream): + """ + Output code coverage report. + """ + log.debug("Coverage report") + self.coverInstance.stop() + self.coverInstance.combine() + self.coverInstance.save() + modules = [module + for name, module in sys.modules.items() + if self.wantModuleCoverage(name, module)] + log.debug("Coverage report will cover modules: %s", modules) + self.coverInstance.report(modules, file=stream) + + import coverage + if self.coverHtmlDir: + log.debug("Generating HTML coverage report") + try: + self.coverInstance.html_report(modules, self.coverHtmlDir) + except coverage.misc.CoverageException, e: + log.warning("Failed to generate HTML report: %s" % str(e)) + + if self.coverXmlFile: + log.debug("Generating XML coverage report") + try: + self.coverInstance.xml_report(modules, self.coverXmlFile) + except coverage.misc.CoverageException, e: + log.warning("Failed to generate XML report: %s" % str(e)) + + # make sure we have minimum required coverage + if self.coverMinPercentage: + f = StringIO.StringIO() + self.coverInstance.report(modules, file=f) + + multiPackageRe = (r'-------\s\w+\s+\d+\s+\d+(?:\s+\d+\s+\d+)?' + r'\s+(\d+)%\s+\d*\s{0,1}$') + singlePackageRe = (r'-------\s[\w./]+\s+\d+\s+\d+(?:\s+\d+\s+\d+)?' + r'\s+(\d+)%(?:\s+[-\d, ]+)\s{0,1}$') + + m = re.search(multiPackageRe, f.getvalue()) + if m is None: + m = re.search(singlePackageRe, f.getvalue()) + + if m: + percentage = int(m.groups()[0]) + if percentage < self.coverMinPercentage: + log.error('TOTAL Coverage did not reach minimum ' + 'required: %d%%' % self.coverMinPercentage) + sys.exit(1) + else: + log.error("No total percentage was found in coverage output, " + "something went wrong.") + + + def wantModuleCoverage(self, name, module): + if not hasattr(module, '__file__'): + log.debug("no coverage of %s: no __file__", name) + return False + module_file = src(module.__file__) + if not module_file or not module_file.endswith('.py'): + log.debug("no coverage of %s: not a python file", name) + return False + if self.coverPackages: + for package in self.coverPackages: + if (re.findall(r'^%s\b' % re.escape(package), name) + and (self.coverTests + or not self.conf.testMatch.search(name))): + log.debug("coverage for %s", name) + return True + if name in self.skipModules: + log.debug("no coverage for %s: loaded before coverage start", + name) + return False + if self.conf.testMatch.search(name) and not self.coverTests: + log.debug("no coverage for %s: is a test", name) + return False + # accept any package that passed the previous tests, unless + # coverPackages is on -- in that case, if we wanted this + # module, we would have already returned True + return not self.coverPackages + + def wantFile(self, file, package=None): + """If inclusive coverage enabled, return true for all source files + in wanted packages. + """ + if self.coverInclusive: + if file.endswith(".py"): + if package and self.coverPackages: + for want in self.coverPackages: + if package.startswith(want): + return True + else: + return True + return None diff --git a/lib/spack/external/nose/plugins/debug.py b/lib/spack/external/nose/plugins/debug.py new file mode 100644 index 00000000000..78243e60d09 --- /dev/null +++ b/lib/spack/external/nose/plugins/debug.py @@ -0,0 +1,67 @@ +""" +This plugin provides ``--pdb`` and ``--pdb-failures`` options. The ``--pdb`` +option will drop the test runner into pdb when it encounters an error. To +drop into pdb on failure, use ``--pdb-failures``. +""" + +import pdb +from nose.plugins.base import Plugin + +class Pdb(Plugin): + """ + Provides --pdb and --pdb-failures options that cause the test runner to + drop into pdb if it encounters an error or failure, respectively. + """ + enabled_for_errors = False + enabled_for_failures = False + score = 5 # run last, among builtins + + def options(self, parser, env): + """Register commandline options. + """ + parser.add_option( + "--pdb", action="store_true", dest="debugBoth", + default=env.get('NOSE_PDB', False), + help="Drop into debugger on failures or errors") + parser.add_option( + "--pdb-failures", action="store_true", + dest="debugFailures", + default=env.get('NOSE_PDB_FAILURES', False), + help="Drop into debugger on failures") + parser.add_option( + "--pdb-errors", action="store_true", + dest="debugErrors", + default=env.get('NOSE_PDB_ERRORS', False), + help="Drop into debugger on errors") + + def configure(self, options, conf): + """Configure which kinds of exceptions trigger plugin. + """ + self.conf = conf + self.enabled_for_errors = options.debugErrors or options.debugBoth + self.enabled_for_failures = options.debugFailures or options.debugBoth + self.enabled = self.enabled_for_failures or self.enabled_for_errors + + def addError(self, test, err): + """Enter pdb if configured to debug errors. + """ + if not self.enabled_for_errors: + return + self.debug(err) + + def addFailure(self, test, err): + """Enter pdb if configured to debug failures. + """ + if not self.enabled_for_failures: + return + self.debug(err) + + def debug(self, err): + import sys # FIXME why is this import here? + ec, ev, tb = err + stdout = sys.stdout + sys.stdout = sys.__stdout__ + try: + pdb.post_mortem(tb) + finally: + sys.stdout = stdout diff --git a/lib/spack/external/nose/plugins/deprecated.py b/lib/spack/external/nose/plugins/deprecated.py new file mode 100644 index 00000000000..461a26be631 --- /dev/null +++ b/lib/spack/external/nose/plugins/deprecated.py @@ -0,0 +1,45 @@ +""" +This plugin installs a DEPRECATED error class for the :class:`DeprecatedTest` +exception. When :class:`DeprecatedTest` is raised, the exception will be logged +in the deprecated attribute of the result, ``D`` or ``DEPRECATED`` (verbose) +will be output, and the exception will not be counted as an error or failure. +It is enabled by default, but can be turned off by using ``--no-deprecated``. +""" + +from nose.plugins.errorclass import ErrorClass, ErrorClassPlugin + + +class DeprecatedTest(Exception): + """Raise this exception to mark a test as deprecated. + """ + pass + + +class Deprecated(ErrorClassPlugin): + """ + Installs a DEPRECATED error class for the DeprecatedTest exception. Enabled + by default. + """ + enabled = True + deprecated = ErrorClass(DeprecatedTest, + label='DEPRECATED', + isfailure=False) + + def options(self, parser, env): + """Register commandline options. + """ + env_opt = 'NOSE_WITHOUT_DEPRECATED' + parser.add_option('--no-deprecated', action='store_true', + dest='noDeprecated', default=env.get(env_opt, False), + help="Disable special handling of DeprecatedTest " + "exceptions.") + + def configure(self, options, conf): + """Configure plugin. + """ + if not self.can_configure: + return + self.conf = conf + disable = getattr(options, 'noDeprecated', False) + if disable: + self.enabled = False diff --git a/lib/spack/external/nose/plugins/doctests.py b/lib/spack/external/nose/plugins/doctests.py new file mode 100644 index 00000000000..5ef65799f3a --- /dev/null +++ b/lib/spack/external/nose/plugins/doctests.py @@ -0,0 +1,455 @@ +"""Use the Doctest plugin with ``--with-doctest`` or the NOSE_WITH_DOCTEST +environment variable to enable collection and execution of :mod:`doctests +`. Because doctests are usually included in the tested package +(instead of being grouped into packages or modules of their own), nose only +looks for them in the non-test packages it discovers in the working directory. + +Doctests may also be placed into files other than python modules, in which +case they can be collected and executed by using the ``--doctest-extension`` +switch or NOSE_DOCTEST_EXTENSION environment variable to indicate which file +extension(s) to load. + +When loading doctests from non-module files, use the ``--doctest-fixtures`` +switch to specify how to find modules containing fixtures for the tests. A +module name will be produced by appending the value of that switch to the base +name of each doctest file loaded. For example, a doctest file "widgets.rst" +with the switch ``--doctest_fixtures=_fixt`` will load fixtures from the module +``widgets_fixt.py``. + +A fixtures module may define any or all of the following functions: + +* setup([module]) or setup_module([module]) + + Called before the test runs. You may raise SkipTest to skip all tests. + +* teardown([module]) or teardown_module([module]) + + Called after the test runs, if setup/setup_module did not raise an + unhandled exception. + +* setup_test(test) + + Called before the test. NOTE: the argument passed is a + doctest.DocTest instance, *not* a unittest.TestCase. + +* teardown_test(test) + + Called after the test, if setup_test did not raise an exception. NOTE: the + argument passed is a doctest.DocTest instance, *not* a unittest.TestCase. + +Doctests are run like any other test, with the exception that output +capture does not work; doctest does its own output capture while running a +test. + +.. note :: + + See :doc:`../doc_tests/test_doctest_fixtures/doctest_fixtures` for + additional documentation and examples. + +""" +from __future__ import generators + +import logging +import os +import sys +import unittest +from inspect import getmodule +from nose.plugins.base import Plugin +from nose.suite import ContextList +from nose.util import anyp, getpackage, test_address, resolve_name, \ + src, tolist, isproperty +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO +import sys +import __builtin__ as builtin_mod + +log = logging.getLogger(__name__) + +try: + import doctest + doctest.DocTestCase + # system version of doctest is acceptable, but needs a monkeypatch +except (ImportError, AttributeError): + # system version is too old + import nose.ext.dtcompat as doctest + + +# +# Doctest and coverage don't get along, so we need to create +# a monkeypatch that will replace the part of doctest that +# interferes with coverage reports. +# +# The monkeypatch is based on this zope patch: +# http://svn.zope.org/Zope3/trunk/src/zope/testing/doctest.py?rev=28679&r1=28703&r2=28705 +# +_orp = doctest._OutputRedirectingPdb + +class NoseOutputRedirectingPdb(_orp): + def __init__(self, out): + self.__debugger_used = False + _orp.__init__(self, out) + + def set_trace(self): + self.__debugger_used = True + _orp.set_trace(self, sys._getframe().f_back) + + def set_continue(self): + # Calling set_continue unconditionally would break unit test + # coverage reporting, as Bdb.set_continue calls sys.settrace(None). + if self.__debugger_used: + _orp.set_continue(self) +doctest._OutputRedirectingPdb = NoseOutputRedirectingPdb + + +class DoctestSuite(unittest.TestSuite): + """ + Doctest suites are parallelizable at the module or file level only, + since they may be attached to objects that are not individually + addressable (like properties). This suite subclass is used when + loading doctests from a module to ensure that behavior. + + This class is used only if the plugin is not fully prepared; + in normal use, the loader's suiteClass is used. + + """ + can_split = False + + def __init__(self, tests=(), context=None, can_split=False): + self.context = context + self.can_split = can_split + unittest.TestSuite.__init__(self, tests=tests) + + def address(self): + return test_address(self.context) + + def __iter__(self): + # 2.3 compat + return iter(self._tests) + + def __str__(self): + return str(self._tests) + + +class Doctest(Plugin): + """ + Activate doctest plugin to find and run doctests in non-test modules. + """ + extension = None + suiteClass = DoctestSuite + + def options(self, parser, env): + """Register commmandline options. + """ + Plugin.options(self, parser, env) + parser.add_option('--doctest-tests', action='store_true', + dest='doctest_tests', + default=env.get('NOSE_DOCTEST_TESTS'), + help="Also look for doctests in test modules. " + "Note that classes, methods and functions should " + "have either doctests or non-doctest tests, " + "not both. [NOSE_DOCTEST_TESTS]") + parser.add_option('--doctest-extension', action="append", + dest="doctestExtension", + metavar="EXT", + help="Also look for doctests in files with " + "this extension [NOSE_DOCTEST_EXTENSION]") + parser.add_option('--doctest-result-variable', + dest='doctest_result_var', + default=env.get('NOSE_DOCTEST_RESULT_VAR'), + metavar="VAR", + help="Change the variable name set to the result of " + "the last interpreter command from the default '_'. " + "Can be used to avoid conflicts with the _() " + "function used for text translation. " + "[NOSE_DOCTEST_RESULT_VAR]") + parser.add_option('--doctest-fixtures', action="store", + dest="doctestFixtures", + metavar="SUFFIX", + help="Find fixtures for a doctest file in module " + "with this name appended to the base name " + "of the doctest file") + parser.add_option('--doctest-options', action="append", + dest="doctestOptions", + metavar="OPTIONS", + help="Specify options to pass to doctest. " + + "Eg. '+ELLIPSIS,+NORMALIZE_WHITESPACE'") + # Set the default as a list, if given in env; otherwise + # an additional value set on the command line will cause + # an error. + env_setting = env.get('NOSE_DOCTEST_EXTENSION') + if env_setting is not None: + parser.set_defaults(doctestExtension=tolist(env_setting)) + + def configure(self, options, config): + """Configure plugin. + """ + Plugin.configure(self, options, config) + self.doctest_result_var = options.doctest_result_var + self.doctest_tests = options.doctest_tests + self.extension = tolist(options.doctestExtension) + self.fixtures = options.doctestFixtures + self.finder = doctest.DocTestFinder() + self.optionflags = 0 + if options.doctestOptions: + flags = ",".join(options.doctestOptions).split(',') + for flag in flags: + if not flag or flag[0] not in '+-': + raise ValueError( + "Must specify doctest options with starting " + + "'+' or '-'. Got %s" % (flag,)) + mode, option_name = flag[0], flag[1:] + option_flag = doctest.OPTIONFLAGS_BY_NAME.get(option_name) + if not option_flag: + raise ValueError("Unknown doctest option %s" % + (option_name,)) + if mode == '+': + self.optionflags |= option_flag + elif mode == '-': + self.optionflags &= ~option_flag + + def prepareTestLoader(self, loader): + """Capture loader's suiteClass. + + This is used to create test suites from doctest files. + + """ + self.suiteClass = loader.suiteClass + + def loadTestsFromModule(self, module): + """Load doctests from the module. + """ + log.debug("loading from %s", module) + if not self.matches(module.__name__): + log.debug("Doctest doesn't want module %s", module) + return + try: + tests = self.finder.find(module) + except AttributeError: + log.exception("Attribute error loading from %s", module) + # nose allows module.__test__ = False; doctest does not and throws + # AttributeError + return + if not tests: + log.debug("No tests found in %s", module) + return + tests.sort() + module_file = src(module.__file__) + # FIXME this breaks the id plugin somehow (tests probably don't + # get wrapped in result proxy or something) + cases = [] + for test in tests: + if not test.examples: + continue + if not test.filename: + test.filename = module_file + cases.append(DocTestCase(test, + optionflags=self.optionflags, + result_var=self.doctest_result_var)) + if cases: + yield self.suiteClass(cases, context=module, can_split=False) + + def loadTestsFromFile(self, filename): + """Load doctests from the file. + + Tests are loaded only if filename's extension matches + configured doctest extension. + + """ + if self.extension and anyp(filename.endswith, self.extension): + name = os.path.basename(filename) + dh = open(filename) + try: + doc = dh.read() + finally: + dh.close() + + fixture_context = None + globs = {'__file__': filename} + if self.fixtures: + base, ext = os.path.splitext(name) + dirname = os.path.dirname(filename) + sys.path.append(dirname) + fixt_mod = base + self.fixtures + try: + fixture_context = __import__( + fixt_mod, globals(), locals(), ["nop"]) + except ImportError, e: + log.debug( + "Could not import %s: %s (%s)", fixt_mod, e, sys.path) + log.debug("Fixture module %s resolved to %s", + fixt_mod, fixture_context) + if hasattr(fixture_context, 'globs'): + globs = fixture_context.globs(globs) + parser = doctest.DocTestParser() + test = parser.get_doctest( + doc, globs=globs, name=name, + filename=filename, lineno=0) + if test.examples: + case = DocFileCase( + test, + optionflags=self.optionflags, + setUp=getattr(fixture_context, 'setup_test', None), + tearDown=getattr(fixture_context, 'teardown_test', None), + result_var=self.doctest_result_var) + if fixture_context: + yield ContextList((case,), context=fixture_context) + else: + yield case + else: + yield False # no tests to load + + def makeTest(self, obj, parent): + """Look for doctests in the given object, which will be a + function, method or class. + """ + name = getattr(obj, '__name__', 'Unnammed %s' % type(obj)) + doctests = self.finder.find(obj, module=getmodule(parent), name=name) + if doctests: + for test in doctests: + if len(test.examples) == 0: + continue + yield DocTestCase(test, obj=obj, optionflags=self.optionflags, + result_var=self.doctest_result_var) + + def matches(self, name): + # FIXME this seems wrong -- nothing is ever going to + # fail this test, since we're given a module NAME not FILE + if name == '__init__.py': + return False + # FIXME don't think we need include/exclude checks here? + return ((self.doctest_tests or not self.conf.testMatch.search(name) + or (self.conf.include + and filter(None, + [inc.search(name) + for inc in self.conf.include]))) + and (not self.conf.exclude + or not filter(None, + [exc.search(name) + for exc in self.conf.exclude]))) + + def wantFile(self, file): + """Override to select all modules and any file ending with + configured doctest extension. + """ + # always want .py files + if file.endswith('.py'): + return True + # also want files that match my extension + if (self.extension + and anyp(file.endswith, self.extension) + and (not self.conf.exclude + or not filter(None, + [exc.search(file) + for exc in self.conf.exclude]))): + return True + return None + + +class DocTestCase(doctest.DocTestCase): + """Overrides DocTestCase to + provide an address() method that returns the correct address for + the doctest case. To provide hints for address(), an obj may also + be passed -- this will be used as the test object for purposes of + determining the test address, if it is provided. + """ + def __init__(self, test, optionflags=0, setUp=None, tearDown=None, + checker=None, obj=None, result_var='_'): + self._result_var = result_var + self._nose_obj = obj + super(DocTestCase, self).__init__( + test, optionflags=optionflags, setUp=setUp, tearDown=tearDown, + checker=checker) + + def address(self): + if self._nose_obj is not None: + return test_address(self._nose_obj) + obj = resolve_name(self._dt_test.name) + + if isproperty(obj): + # properties have no connection to the class they are in + # so we can't just look 'em up, we have to first look up + # the class, then stick the prop on the end + parts = self._dt_test.name.split('.') + class_name = '.'.join(parts[:-1]) + cls = resolve_name(class_name) + base_addr = test_address(cls) + return (base_addr[0], base_addr[1], + '.'.join([base_addr[2], parts[-1]])) + else: + return test_address(obj) + + # doctests loaded via find(obj) omit the module name + # so we need to override id, __repr__ and shortDescription + # bonus: this will squash a 2.3 vs 2.4 incompatiblity + def id(self): + name = self._dt_test.name + filename = self._dt_test.filename + if filename is not None: + pk = getpackage(filename) + if pk is None: + return name + if not name.startswith(pk): + name = "%s.%s" % (pk, name) + return name + + def __repr__(self): + name = self.id() + name = name.split('.') + return "%s (%s)" % (name[-1], '.'.join(name[:-1])) + __str__ = __repr__ + + def shortDescription(self): + return 'Doctest: %s' % self.id() + + def setUp(self): + if self._result_var is not None: + self._old_displayhook = sys.displayhook + sys.displayhook = self._displayhook + super(DocTestCase, self).setUp() + + def _displayhook(self, value): + if value is None: + return + setattr(builtin_mod, self._result_var, value) + print repr(value) + + def tearDown(self): + super(DocTestCase, self).tearDown() + if self._result_var is not None: + sys.displayhook = self._old_displayhook + delattr(builtin_mod, self._result_var) + + +class DocFileCase(doctest.DocFileCase): + """Overrides to provide address() method that returns the correct + address for the doc file case. + """ + def __init__(self, test, optionflags=0, setUp=None, tearDown=None, + checker=None, result_var='_'): + self._result_var = result_var + super(DocFileCase, self).__init__( + test, optionflags=optionflags, setUp=setUp, tearDown=tearDown, + checker=None) + + def address(self): + return (self._dt_test.filename, None, None) + + def setUp(self): + if self._result_var is not None: + self._old_displayhook = sys.displayhook + sys.displayhook = self._displayhook + super(DocFileCase, self).setUp() + + def _displayhook(self, value): + if value is None: + return + setattr(builtin_mod, self._result_var, value) + print repr(value) + + def tearDown(self): + super(DocFileCase, self).tearDown() + if self._result_var is not None: + sys.displayhook = self._old_displayhook + delattr(builtin_mod, self._result_var) diff --git a/lib/spack/external/nose/plugins/errorclass.py b/lib/spack/external/nose/plugins/errorclass.py new file mode 100644 index 00000000000..d1540e0070a --- /dev/null +++ b/lib/spack/external/nose/plugins/errorclass.py @@ -0,0 +1,210 @@ +""" +ErrorClass Plugins +------------------ + +ErrorClass plugins provide an easy way to add support for custom +handling of particular classes of exceptions. + +An ErrorClass plugin defines one or more ErrorClasses and how each is +handled and reported on. Each error class is stored in a different +attribute on the result, and reported separately. Each error class must +indicate the exceptions that fall under that class, the label to use +for reporting, and whether exceptions of the class should be +considered as failures for the whole test run. + +ErrorClasses use a declarative syntax. Assign an ErrorClass to the +attribute you wish to add to the result object, defining the +exceptions, label and isfailure attributes. For example, to declare an +ErrorClassPlugin that defines TodoErrors (and subclasses of TodoError) +as an error class with the label 'TODO' that is considered a failure, +do this: + + >>> class Todo(Exception): + ... pass + >>> class TodoError(ErrorClassPlugin): + ... todo = ErrorClass(Todo, label='TODO', isfailure=True) + +The MetaErrorClass metaclass translates the ErrorClass declarations +into the tuples used by the error handling and reporting functions in +the result. This is an internal format and subject to change; you +should always use the declarative syntax for attaching ErrorClasses to +an ErrorClass plugin. + + >>> TodoError.errorClasses # doctest: +ELLIPSIS + ((, ('todo', 'TODO', True)),) + +Let's see the plugin in action. First some boilerplate. + + >>> import sys + >>> import unittest + >>> try: + ... # 2.7+ + ... from unittest.runner import _WritelnDecorator + ... except ImportError: + ... from unittest import _WritelnDecorator + ... + >>> buf = _WritelnDecorator(sys.stdout) + +Now define a test case that raises a Todo. + + >>> class TestTodo(unittest.TestCase): + ... def runTest(self): + ... raise Todo("I need to test something") + >>> case = TestTodo() + +Prepare the result using our plugin. Normally this happens during the +course of test execution within nose -- you won't be doing this +yourself. For the purposes of this testing document, I'm stepping +through the internal process of nose so you can see what happens at +each step. + + >>> plugin = TodoError() + >>> from nose.result import _TextTestResult + >>> result = _TextTestResult(stream=buf, descriptions=0, verbosity=2) + >>> plugin.prepareTestResult(result) + +Now run the test. TODO is printed. + + >>> _ = case(result) # doctest: +ELLIPSIS + runTest (....TestTodo) ... TODO: I need to test something + +Errors and failures are empty, but todo has our test: + + >>> result.errors + [] + >>> result.failures + [] + >>> result.todo # doctest: +ELLIPSIS + [(<....TestTodo testMethod=runTest>, '...Todo: I need to test something\\n')] + >>> result.printErrors() # doctest: +ELLIPSIS + + ====================================================================== + TODO: runTest (....TestTodo) + ---------------------------------------------------------------------- + Traceback (most recent call last): + ... + ...Todo: I need to test something + + +Since we defined a Todo as a failure, the run was not successful. + + >>> result.wasSuccessful() + False +""" + +from nose.pyversion import make_instancemethod +from nose.plugins.base import Plugin +from nose.result import TextTestResult +from nose.util import isclass + +class MetaErrorClass(type): + """Metaclass for ErrorClassPlugins that allows error classes to be + set up in a declarative manner. + """ + def __init__(self, name, bases, attr): + errorClasses = [] + for name, detail in attr.items(): + if isinstance(detail, ErrorClass): + attr.pop(name) + for cls in detail: + errorClasses.append( + (cls, (name, detail.label, detail.isfailure))) + super(MetaErrorClass, self).__init__(name, bases, attr) + self.errorClasses = tuple(errorClasses) + + +class ErrorClass(object): + def __init__(self, *errorClasses, **kw): + self.errorClasses = errorClasses + try: + for key in ('label', 'isfailure'): + setattr(self, key, kw.pop(key)) + except KeyError: + raise TypeError("%r is a required named argument for ErrorClass" + % key) + + def __iter__(self): + return iter(self.errorClasses) + + +class ErrorClassPlugin(Plugin): + """ + Base class for ErrorClass plugins. Subclass this class and declare the + exceptions that you wish to handle as attributes of the subclass. + """ + __metaclass__ = MetaErrorClass + score = 1000 + errorClasses = () + + def addError(self, test, err): + err_cls, a, b = err + if not isclass(err_cls): + return + classes = [e[0] for e in self.errorClasses] + if filter(lambda c: issubclass(err_cls, c), classes): + return True + + def prepareTestResult(self, result): + if not hasattr(result, 'errorClasses'): + self.patchResult(result) + for cls, (storage_attr, label, isfail) in self.errorClasses: + if cls not in result.errorClasses: + storage = getattr(result, storage_attr, []) + setattr(result, storage_attr, storage) + result.errorClasses[cls] = (storage, label, isfail) + + def patchResult(self, result): + result.printLabel = print_label_patch(result) + result._orig_addError, result.addError = \ + result.addError, add_error_patch(result) + result._orig_wasSuccessful, result.wasSuccessful = \ + result.wasSuccessful, wassuccessful_patch(result) + if hasattr(result, 'printErrors'): + result._orig_printErrors, result.printErrors = \ + result.printErrors, print_errors_patch(result) + if hasattr(result, 'addSkip'): + result._orig_addSkip, result.addSkip = \ + result.addSkip, add_skip_patch(result) + result.errorClasses = {} + + +def add_error_patch(result): + """Create a new addError method to patch into a result instance + that recognizes the errorClasses attribute and deals with + errorclasses correctly. + """ + return make_instancemethod(TextTestResult.addError, result) + + +def print_errors_patch(result): + """Create a new printErrors method that prints errorClasses items + as well. + """ + return make_instancemethod(TextTestResult.printErrors, result) + + +def print_label_patch(result): + """Create a new printLabel method that prints errorClasses items + as well. + """ + return make_instancemethod(TextTestResult.printLabel, result) + + +def wassuccessful_patch(result): + """Create a new wasSuccessful method that checks errorClasses for + exceptions that were put into other slots than error or failure + but that still count as not success. + """ + return make_instancemethod(TextTestResult.wasSuccessful, result) + + +def add_skip_patch(result): + """Create a new addSkip method to patch into a result instance + that delegates to addError. + """ + return make_instancemethod(TextTestResult.addSkip, result) + + +if __name__ == '__main__': + import doctest + doctest.testmod() diff --git a/lib/spack/external/nose/plugins/failuredetail.py b/lib/spack/external/nose/plugins/failuredetail.py new file mode 100644 index 00000000000..6462865dd0e --- /dev/null +++ b/lib/spack/external/nose/plugins/failuredetail.py @@ -0,0 +1,49 @@ +""" +This plugin provides assert introspection. When the plugin is enabled +and a test failure occurs, the traceback is displayed with extra context +around the line in which the exception was raised. Simple variable +substitution is also performed in the context output to provide more +debugging information. +""" + +from nose.plugins import Plugin +from nose.pyversion import exc_to_unicode, force_unicode +from nose.inspector import inspect_traceback + +class FailureDetail(Plugin): + """ + Plugin that provides extra information in tracebacks of test failures. + """ + score = 1600 # before capture + + def options(self, parser, env): + """Register commmandline options. + """ + parser.add_option( + "-d", "--detailed-errors", "--failure-detail", + action="store_true", + default=env.get('NOSE_DETAILED_ERRORS'), + dest="detailedErrors", help="Add detail to error" + " output by attempting to evaluate failed" + " asserts [NOSE_DETAILED_ERRORS]") + + def configure(self, options, conf): + """Configure plugin. + """ + if not self.can_configure: + return + self.enabled = options.detailedErrors + self.conf = conf + + def formatFailure(self, test, err): + """Add detail from traceback inspection to error message of a failure. + """ + ec, ev, tb = err + tbinfo, str_ev = None, exc_to_unicode(ev) + + if tb: + tbinfo = force_unicode(inspect_traceback(tb)) + str_ev = '\n'.join([str_ev, tbinfo]) + test.tbinfo = tbinfo + return (ec, str_ev, tb) + diff --git a/lib/spack/external/nose/plugins/isolate.py b/lib/spack/external/nose/plugins/isolate.py new file mode 100644 index 00000000000..13235dfbd1e --- /dev/null +++ b/lib/spack/external/nose/plugins/isolate.py @@ -0,0 +1,103 @@ +"""The isolation plugin resets the contents of sys.modules after running +each test module or package. Use it by setting ``--with-isolation`` or the +NOSE_WITH_ISOLATION environment variable. + +The effects are similar to wrapping the following functions around the +import and execution of each test module:: + + def setup(module): + module._mods = sys.modules.copy() + + def teardown(module): + to_del = [ m for m in sys.modules.keys() if m not in + module._mods ] + for mod in to_del: + del sys.modules[mod] + sys.modules.update(module._mods) + +Isolation works only during lazy loading. In normal use, this is only +during discovery of modules within a directory, where the process of +importing, loading tests and running tests from each module is +encapsulated in a single loadTestsFromName call. This plugin +implements loadTestsFromNames to force the same lazy-loading there, +which allows isolation to work in directed mode as well as discovery, +at the cost of some efficiency: lazy-loading names forces full context +setup and teardown to run for each name, defeating the grouping that +is normally used to ensure that context setup and teardown are run the +fewest possible times for a given set of names. + +.. warning :: + + This plugin should not be used in conjunction with other plugins + that assume that modules, once imported, will stay imported; for + instance, it may cause very odd results when used with the coverage + plugin. + +""" + +import logging +import sys + +from nose.plugins import Plugin + + +log = logging.getLogger('nose.plugins.isolation') + +class IsolationPlugin(Plugin): + """ + Activate the isolation plugin to isolate changes to external + modules to a single test module or package. The isolation plugin + resets the contents of sys.modules after each test module or + package runs to its state before the test. PLEASE NOTE that this + plugin should not be used with the coverage plugin, or in any other case + where module reloading may produce undesirable side-effects. + """ + score = 10 # I want to be last + name = 'isolation' + + def configure(self, options, conf): + """Configure plugin. + """ + Plugin.configure(self, options, conf) + self._mod_stack = [] + + def beforeContext(self): + """Copy sys.modules onto my mod stack + """ + mods = sys.modules.copy() + self._mod_stack.append(mods) + + def afterContext(self): + """Pop my mod stack and restore sys.modules to the state + it was in when mod stack was pushed. + """ + mods = self._mod_stack.pop() + to_del = [ m for m in sys.modules.keys() if m not in mods ] + if to_del: + log.debug('removing sys modules entries: %s', to_del) + for mod in to_del: + del sys.modules[mod] + sys.modules.update(mods) + + def loadTestsFromNames(self, names, module=None): + """Create a lazy suite that calls beforeContext and afterContext + around each name. The side-effect of this is that full context + fixtures will be set up and torn down around each test named. + """ + # Fast path for when we don't care + if not names or len(names) == 1: + return + loader = self.loader + plugins = self.conf.plugins + def lazy(): + for name in names: + plugins.beforeContext() + yield loader.loadTestsFromName(name, module=module) + plugins.afterContext() + return (loader.suiteClass(lazy), []) + + def prepareTestLoader(self, loader): + """Get handle on test loader so we can use it in loadTestsFromNames. + """ + self.loader = loader + diff --git a/lib/spack/external/nose/plugins/logcapture.py b/lib/spack/external/nose/plugins/logcapture.py new file mode 100644 index 00000000000..4c9a79f6fd7 --- /dev/null +++ b/lib/spack/external/nose/plugins/logcapture.py @@ -0,0 +1,245 @@ +""" +This plugin captures logging statements issued during test execution. When an +error or failure occurs, the captured log messages are attached to the running +test in the test.capturedLogging attribute, and displayed with the error failure +output. It is enabled by default but can be turned off with the option +``--nologcapture``. + +You can filter captured logging statements with the ``--logging-filter`` option. +If set, it specifies which logger(s) will be captured; loggers that do not match +will be passed. Example: specifying ``--logging-filter=sqlalchemy,myapp`` +will ensure that only statements logged via sqlalchemy.engine, myapp +or myapp.foo.bar logger will be logged. + +You can remove other installed logging handlers with the +``--logging-clear-handlers`` option. +""" + +import logging +from logging import Handler +import threading + +from nose.plugins.base import Plugin +from nose.util import anyp, ln, safe_str + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +log = logging.getLogger(__name__) + +class FilterSet(object): + def __init__(self, filter_components): + self.inclusive, self.exclusive = self._partition(filter_components) + + # @staticmethod + def _partition(components): + inclusive, exclusive = [], [] + for component in components: + if component.startswith('-'): + exclusive.append(component[1:]) + else: + inclusive.append(component) + return inclusive, exclusive + _partition = staticmethod(_partition) + + def allow(self, record): + """returns whether this record should be printed""" + if not self: + # nothing to filter + return True + return self._allow(record) and not self._deny(record) + + # @staticmethod + def _any_match(matchers, record): + """return the bool of whether `record` starts with + any item in `matchers`""" + def record_matches_key(key): + return record == key or record.startswith(key + '.') + return anyp(bool, map(record_matches_key, matchers)) + _any_match = staticmethod(_any_match) + + def _allow(self, record): + if not self.inclusive: + return True + return self._any_match(self.inclusive, record) + + def _deny(self, record): + if not self.exclusive: + return False + return self._any_match(self.exclusive, record) + + +class MyMemoryHandler(Handler): + def __init__(self, logformat, logdatefmt, filters): + Handler.__init__(self) + fmt = logging.Formatter(logformat, logdatefmt) + self.setFormatter(fmt) + self.filterset = FilterSet(filters) + self.buffer = [] + def emit(self, record): + self.buffer.append(self.format(record)) + def flush(self): + pass # do nothing + def truncate(self): + self.buffer = [] + def filter(self, record): + if self.filterset.allow(record.name): + return Handler.filter(self, record) + def __getstate__(self): + state = self.__dict__.copy() + del state['lock'] + return state + def __setstate__(self, state): + self.__dict__.update(state) + self.lock = threading.RLock() + + +class LogCapture(Plugin): + """ + Log capture plugin. Enabled by default. Disable with --nologcapture. + This plugin captures logging statements issued during test execution, + appending any output captured to the error or failure output, + should the test fail or raise an error. + """ + enabled = True + env_opt = 'NOSE_NOLOGCAPTURE' + name = 'logcapture' + score = 500 + logformat = '%(name)s: %(levelname)s: %(message)s' + logdatefmt = None + clear = False + filters = ['-nose'] + + def options(self, parser, env): + """Register commandline options. + """ + parser.add_option( + "--nologcapture", action="store_false", + default=not env.get(self.env_opt), dest="logcapture", + help="Disable logging capture plugin. " + "Logging configuration will be left intact." + " [NOSE_NOLOGCAPTURE]") + parser.add_option( + "--logging-format", action="store", dest="logcapture_format", + default=env.get('NOSE_LOGFORMAT') or self.logformat, + metavar="FORMAT", + help="Specify custom format to print statements. " + "Uses the same format as used by standard logging handlers." + " [NOSE_LOGFORMAT]") + parser.add_option( + "--logging-datefmt", action="store", dest="logcapture_datefmt", + default=env.get('NOSE_LOGDATEFMT') or self.logdatefmt, + metavar="FORMAT", + help="Specify custom date/time format to print statements. " + "Uses the same format as used by standard logging handlers." + " [NOSE_LOGDATEFMT]") + parser.add_option( + "--logging-filter", action="store", dest="logcapture_filters", + default=env.get('NOSE_LOGFILTER'), + metavar="FILTER", + help="Specify which statements to filter in/out. " + "By default, everything is captured. If the output is too" + " verbose,\nuse this option to filter out needless output.\n" + "Example: filter=foo will capture statements issued ONLY to\n" + " foo or foo.what.ever.sub but not foobar or other logger.\n" + "Specify multiple loggers with comma: filter=foo,bar,baz.\n" + "If any logger name is prefixed with a minus, eg filter=-foo,\n" + "it will be excluded rather than included. Default: " + "exclude logging messages from nose itself (-nose)." + " [NOSE_LOGFILTER]\n") + parser.add_option( + "--logging-clear-handlers", action="store_true", + default=False, dest="logcapture_clear", + help="Clear all other logging handlers") + parser.add_option( + "--logging-level", action="store", + default='NOTSET', dest="logcapture_level", + help="Set the log level to capture") + + def configure(self, options, conf): + """Configure plugin. + """ + self.conf = conf + # Disable if explicitly disabled, or if logging is + # configured via logging config file + if not options.logcapture or conf.loggingConfig: + self.enabled = False + self.logformat = options.logcapture_format + self.logdatefmt = options.logcapture_datefmt + self.clear = options.logcapture_clear + self.loglevel = options.logcapture_level + if options.logcapture_filters: + self.filters = options.logcapture_filters.split(',') + + def setupLoghandler(self): + # setup our handler with root logger + root_logger = logging.getLogger() + if self.clear: + if hasattr(root_logger, "handlers"): + for handler in root_logger.handlers: + root_logger.removeHandler(handler) + for logger in logging.Logger.manager.loggerDict.values(): + if hasattr(logger, "handlers"): + for handler in logger.handlers: + logger.removeHandler(handler) + # make sure there isn't one already + # you can't simply use "if self.handler not in root_logger.handlers" + # since at least in unit tests this doesn't work -- + # LogCapture() is instantiated for each test case while root_logger + # is module global + # so we always add new MyMemoryHandler instance + for handler in root_logger.handlers[:]: + if isinstance(handler, MyMemoryHandler): + root_logger.handlers.remove(handler) + root_logger.addHandler(self.handler) + # to make sure everything gets captured + loglevel = getattr(self, "loglevel", "NOTSET") + root_logger.setLevel(getattr(logging, loglevel)) + + def begin(self): + """Set up logging handler before test run begins. + """ + self.start() + + def start(self): + self.handler = MyMemoryHandler(self.logformat, self.logdatefmt, + self.filters) + self.setupLoghandler() + + def end(self): + pass + + def beforeTest(self, test): + """Clear buffers and handlers before test. + """ + self.setupLoghandler() + + def afterTest(self, test): + """Clear buffers after test. + """ + self.handler.truncate() + + def formatFailure(self, test, err): + """Add captured log messages to failure output. + """ + return self.formatError(test, err) + + def formatError(self, test, err): + """Add captured log messages to error output. + """ + # logic flow copied from Capture.formatError + test.capturedLogging = records = self.formatLogRecords() + if not records: + return err + ec, ev, tb = err + return (ec, self.addCaptureToErr(ev, records), tb) + + def formatLogRecords(self): + return map(safe_str, self.handler.buffer) + + def addCaptureToErr(self, ev, records): + return '\n'.join([safe_str(ev), ln('>> begin captured logging <<')] + \ + records + \ + [ln('>> end captured logging <<')]) diff --git a/lib/spack/external/nose/plugins/manager.py b/lib/spack/external/nose/plugins/manager.py new file mode 100644 index 00000000000..4d2ed22b6fe --- /dev/null +++ b/lib/spack/external/nose/plugins/manager.py @@ -0,0 +1,460 @@ +""" +Plugin Manager +-------------- + +A plugin manager class is used to load plugins, manage the list of +loaded plugins, and proxy calls to those plugins. + +The plugin managers provided with nose are: + +:class:`PluginManager` + This manager doesn't implement loadPlugins, so it can only work + with a static list of plugins. + +:class:`BuiltinPluginManager` + This manager loads plugins referenced in ``nose.plugins.builtin``. + +:class:`EntryPointPluginManager` + This manager uses setuptools entrypoints to load plugins. + +:class:`ExtraPluginsPluginManager` + This manager loads extra plugins specified with the keyword + `addplugins`. + +:class:`DefaultPluginMananger` + This is the manager class that will be used by default. If + setuptools is installed, it is a subclass of + :class:`EntryPointPluginManager` and :class:`BuiltinPluginManager`; + otherwise, an alias to :class:`BuiltinPluginManager`. + +:class:`RestrictedPluginManager` + This manager is for use in test runs where some plugin calls are + not available, such as runs started with ``python setup.py test``, + where the test runner is the default unittest :class:`TextTestRunner`. It + is a subclass of :class:`DefaultPluginManager`. + +Writing a plugin manager +======================== + +If you want to load plugins via some other means, you can write a +plugin manager and pass an instance of your plugin manager class when +instantiating the :class:`nose.config.Config` instance that you pass to +:class:`TestProgram` (or :func:`main` or :func:`run`). + +To implement your plugin loading scheme, implement ``loadPlugins()``, +and in that method, call ``addPlugin()`` with an instance of each plugin +you wish to make available. Make sure to call +``super(self).loadPlugins()`` as well if have subclassed a manager +other than ``PluginManager``. + +""" +import inspect +import logging +import os +import sys +from itertools import chain as iterchain +from warnings import warn +import nose.config +from nose.failure import Failure +from nose.plugins.base import IPluginInterface +from nose.pyversion import sort_list + +try: + import cPickle as pickle +except: + import pickle +try: + from cStringIO import StringIO +except: + from StringIO import StringIO + + +__all__ = ['DefaultPluginManager', 'PluginManager', 'EntryPointPluginManager', + 'BuiltinPluginManager', 'RestrictedPluginManager'] + +log = logging.getLogger(__name__) + + +class PluginProxy(object): + """Proxy for plugin calls. Essentially a closure bound to the + given call and plugin list. + + The plugin proxy also must be bound to a particular plugin + interface specification, so that it knows what calls are available + and any special handling that is required for each call. + """ + interface = IPluginInterface + def __init__(self, call, plugins): + try: + self.method = getattr(self.interface, call) + except AttributeError: + raise AttributeError("%s is not a valid %s method" + % (call, self.interface.__name__)) + self.call = self.makeCall(call) + self.plugins = [] + for p in plugins: + self.addPlugin(p, call) + + def __call__(self, *arg, **kw): + return self.call(*arg, **kw) + + def addPlugin(self, plugin, call): + """Add plugin to my list of plugins to call, if it has the attribute + I'm bound to. + """ + meth = getattr(plugin, call, None) + if meth is not None: + if call == 'loadTestsFromModule' and \ + len(inspect.getargspec(meth)[0]) == 2: + orig_meth = meth + meth = lambda module, path, **kwargs: orig_meth(module) + self.plugins.append((plugin, meth)) + + def makeCall(self, call): + if call == 'loadTestsFromNames': + # special case -- load tests from names behaves somewhat differently + # from other chainable calls, because plugins return a tuple, only + # part of which can be chained to the next plugin. + return self._loadTestsFromNames + + meth = self.method + if getattr(meth, 'generative', False): + # call all plugins and yield a flattened iterator of their results + return lambda *arg, **kw: list(self.generate(*arg, **kw)) + elif getattr(meth, 'chainable', False): + return self.chain + else: + # return a value from the first plugin that returns non-None + return self.simple + + def chain(self, *arg, **kw): + """Call plugins in a chain, where the result of each plugin call is + sent to the next plugin as input. The final output result is returned. + """ + result = None + # extract the static arguments (if any) from arg so they can + # be passed to each plugin call in the chain + static = [a for (static, a) + in zip(getattr(self.method, 'static_args', []), arg) + if static] + for p, meth in self.plugins: + result = meth(*arg, **kw) + arg = static[:] + arg.append(result) + return result + + def generate(self, *arg, **kw): + """Call all plugins, yielding each item in each non-None result. + """ + for p, meth in self.plugins: + result = None + try: + result = meth(*arg, **kw) + if result is not None: + for r in result: + yield r + except (KeyboardInterrupt, SystemExit): + raise + except: + exc = sys.exc_info() + yield Failure(*exc) + continue + + def simple(self, *arg, **kw): + """Call all plugins, returning the first non-None result. + """ + for p, meth in self.plugins: + result = meth(*arg, **kw) + if result is not None: + return result + + def _loadTestsFromNames(self, names, module=None): + """Chainable but not quite normal. Plugins return a tuple of + (tests, names) after processing the names. The tests are added + to a suite that is accumulated throughout the full call, while + names are input for the next plugin in the chain. + """ + suite = [] + for p, meth in self.plugins: + result = meth(names, module=module) + if result is not None: + suite_part, names = result + if suite_part: + suite.extend(suite_part) + return suite, names + + +class NoPlugins(object): + """Null Plugin manager that has no plugins.""" + interface = IPluginInterface + def __init__(self): + self._plugins = self.plugins = () + + def __iter__(self): + return () + + def _doNothing(self, *args, **kwds): + pass + + def _emptyIterator(self, *args, **kwds): + return () + + def __getattr__(self, call): + method = getattr(self.interface, call) + if getattr(method, "generative", False): + return self._emptyIterator + else: + return self._doNothing + + def addPlugin(self, plug): + raise NotImplementedError() + + def addPlugins(self, plugins): + raise NotImplementedError() + + def configure(self, options, config): + pass + + def loadPlugins(self): + pass + + def sort(self): + pass + + +class PluginManager(object): + """Base class for plugin managers. PluginManager is intended to be + used only with a static list of plugins. The loadPlugins() implementation + only reloads plugins from _extraplugins to prevent those from being + overridden by a subclass. + + The basic functionality of a plugin manager is to proxy all unknown + attributes through a ``PluginProxy`` to a list of plugins. + + Note that the list of plugins *may not* be changed after the first plugin + call. + """ + proxyClass = PluginProxy + + def __init__(self, plugins=(), proxyClass=None): + self._plugins = [] + self._extraplugins = () + self._proxies = {} + if plugins: + self.addPlugins(plugins) + if proxyClass is not None: + self.proxyClass = proxyClass + + def __getattr__(self, call): + try: + return self._proxies[call] + except KeyError: + proxy = self.proxyClass(call, self._plugins) + self._proxies[call] = proxy + return proxy + + def __iter__(self): + return iter(self.plugins) + + def addPlugin(self, plug): + # allow, for instance, plugins loaded via entry points to + # supplant builtin plugins. + new_name = getattr(plug, 'name', object()) + self._plugins[:] = [p for p in self._plugins + if getattr(p, 'name', None) != new_name] + self._plugins.append(plug) + + def addPlugins(self, plugins=(), extraplugins=()): + """extraplugins are maintained in a separate list and + re-added by loadPlugins() to prevent their being overwritten + by plugins added by a subclass of PluginManager + """ + self._extraplugins = extraplugins + for plug in iterchain(plugins, extraplugins): + self.addPlugin(plug) + + def configure(self, options, config): + """Configure the set of plugins with the given options + and config instance. After configuration, disabled plugins + are removed from the plugins list. + """ + log.debug("Configuring plugins") + self.config = config + cfg = PluginProxy('configure', self._plugins) + cfg(options, config) + enabled = [plug for plug in self._plugins if plug.enabled] + self.plugins = enabled + self.sort() + log.debug("Plugins enabled: %s", enabled) + + def loadPlugins(self): + for plug in self._extraplugins: + self.addPlugin(plug) + + def sort(self): + return sort_list(self._plugins, lambda x: getattr(x, 'score', 1), reverse=True) + + def _get_plugins(self): + return self._plugins + + def _set_plugins(self, plugins): + self._plugins = [] + self.addPlugins(plugins) + + plugins = property(_get_plugins, _set_plugins, None, + """Access the list of plugins managed by + this plugin manager""") + + +class ZeroNinePlugin: + """Proxy for 0.9 plugins, adapts 0.10 calls to 0.9 standard. + """ + def __init__(self, plugin): + self.plugin = plugin + + def options(self, parser, env=os.environ): + self.plugin.add_options(parser, env) + + def addError(self, test, err): + if not hasattr(self.plugin, 'addError'): + return + # switch off to addSkip, addDeprecated if those types + from nose.exc import SkipTest, DeprecatedTest + ec, ev, tb = err + if issubclass(ec, SkipTest): + if not hasattr(self.plugin, 'addSkip'): + return + return self.plugin.addSkip(test.test) + elif issubclass(ec, DeprecatedTest): + if not hasattr(self.plugin, 'addDeprecated'): + return + return self.plugin.addDeprecated(test.test) + # add capt + capt = test.capturedOutput + return self.plugin.addError(test.test, err, capt) + + def loadTestsFromFile(self, filename): + if hasattr(self.plugin, 'loadTestsFromPath'): + return self.plugin.loadTestsFromPath(filename) + + def addFailure(self, test, err): + if not hasattr(self.plugin, 'addFailure'): + return + # add capt and tbinfo + capt = test.capturedOutput + tbinfo = test.tbinfo + return self.plugin.addFailure(test.test, err, capt, tbinfo) + + def addSuccess(self, test): + if not hasattr(self.plugin, 'addSuccess'): + return + capt = test.capturedOutput + self.plugin.addSuccess(test.test, capt) + + def startTest(self, test): + if not hasattr(self.plugin, 'startTest'): + return + return self.plugin.startTest(test.test) + + def stopTest(self, test): + if not hasattr(self.plugin, 'stopTest'): + return + return self.plugin.stopTest(test.test) + + def __getattr__(self, val): + return getattr(self.plugin, val) + + +class EntryPointPluginManager(PluginManager): + """Plugin manager that loads plugins from the `nose.plugins` and + `nose.plugins.0.10` entry points. + """ + entry_points = (('nose.plugins.0.10', None), + ('nose.plugins', ZeroNinePlugin)) + + def loadPlugins(self): + """Load plugins by iterating the `nose.plugins` entry point. + """ + from pkg_resources import iter_entry_points + loaded = {} + for entry_point, adapt in self.entry_points: + for ep in iter_entry_points(entry_point): + if ep.name in loaded: + continue + loaded[ep.name] = True + log.debug('%s load plugin %s', self.__class__.__name__, ep) + try: + plugcls = ep.load() + except KeyboardInterrupt: + raise + except Exception, e: + # never want a plugin load to kill the test run + # but we can't log here because the logger is not yet + # configured + warn("Unable to load plugin %s: %s" % (ep, e), + RuntimeWarning) + continue + if adapt: + plug = adapt(plugcls()) + else: + plug = plugcls() + self.addPlugin(plug) + super(EntryPointPluginManager, self).loadPlugins() + + +class BuiltinPluginManager(PluginManager): + """Plugin manager that loads plugins from the list in + `nose.plugins.builtin`. + """ + def loadPlugins(self): + """Load plugins in nose.plugins.builtin + """ + from nose.plugins import builtin + for plug in builtin.plugins: + self.addPlugin(plug()) + super(BuiltinPluginManager, self).loadPlugins() + +try: + import pkg_resources + class DefaultPluginManager(EntryPointPluginManager, BuiltinPluginManager): + pass + +except ImportError: + class DefaultPluginManager(BuiltinPluginManager): + pass + +class RestrictedPluginManager(DefaultPluginManager): + """Plugin manager that restricts the plugin list to those not + excluded by a list of exclude methods. Any plugin that implements + an excluded method will be removed from the manager's plugin list + after plugins are loaded. + """ + def __init__(self, plugins=(), exclude=(), load=True): + DefaultPluginManager.__init__(self, plugins) + self.load = load + self.exclude = exclude + self.excluded = [] + self._excludedOpts = None + + def excludedOption(self, name): + if self._excludedOpts is None: + from optparse import OptionParser + self._excludedOpts = OptionParser(add_help_option=False) + for plugin in self.excluded: + plugin.options(self._excludedOpts, env={}) + return self._excludedOpts.get_option('--' + name) + + def loadPlugins(self): + if self.load: + DefaultPluginManager.loadPlugins(self) + allow = [] + for plugin in self.plugins: + ok = True + for method in self.exclude: + if hasattr(plugin, method): + ok = False + self.excluded.append(plugin) + break + if ok: + allow.append(plugin) + self.plugins = allow diff --git a/lib/spack/external/nose/plugins/multiprocess.py b/lib/spack/external/nose/plugins/multiprocess.py new file mode 100644 index 00000000000..2cae744a111 --- /dev/null +++ b/lib/spack/external/nose/plugins/multiprocess.py @@ -0,0 +1,835 @@ +""" +Overview +======== + +The multiprocess plugin enables you to distribute your test run among a set of +worker processes that run tests in parallel. This can speed up CPU-bound test +runs (as long as the number of work processeses is around the number of +processors or cores available), but is mainly useful for IO-bound tests that +spend most of their time waiting for data to arrive from someplace else. + +.. note :: + + See :doc:`../doc_tests/test_multiprocess/multiprocess` for + additional documentation and examples. Use of this plugin on python + 2.5 or earlier requires the multiprocessing_ module, also available + from PyPI. + +.. _multiprocessing : http://code.google.com/p/python-multiprocessing/ + +How tests are distributed +========================= + +The ideal case would be to dispatch each test to a worker process +separately. This ideal is not attainable in all cases, however, because many +test suites depend on context (class, module or package) fixtures. + +The plugin can't know (unless you tell it -- see below!) if a context fixture +can be called many times concurrently (is re-entrant), or if it can be shared +among tests running in different processes. Therefore, if a context has +fixtures, the default behavior is to dispatch the entire suite to a worker as +a unit. + +Controlling distribution +^^^^^^^^^^^^^^^^^^^^^^^^ + +There are two context-level variables that you can use to control this default +behavior. + +If a context's fixtures are re-entrant, set ``_multiprocess_can_split_ = True`` +in the context, and the plugin will dispatch tests in suites bound to that +context as if the context had no fixtures. This means that the fixtures will +execute concurrently and multiple times, typically once per test. + +If a context's fixtures can be shared by tests running in different processes +-- such as a package-level fixture that starts an external http server or +initializes a shared database -- then set ``_multiprocess_shared_ = True`` in +the context. These fixtures will then execute in the primary nose process, and +tests in those contexts will be individually dispatched to run in parallel. + +How results are collected and reported +====================================== + +As each test or suite executes in a worker process, results (failures, errors, +and specially handled exceptions like SkipTest) are collected in that +process. When the worker process finishes, it returns results to the main +nose process. There, any progress output is printed (dots!), and the +results from the test run are combined into a consolidated result +set. When results have been received for all dispatched tests, or all +workers have died, the result summary is output as normal. + +Beware! +======= + +Not all test suites will benefit from, or even operate correctly using, this +plugin. For example, CPU-bound tests will run more slowly if you don't have +multiple processors. There are also some differences in plugin +interactions and behaviors due to the way in which tests are dispatched and +loaded. In general, test loading under this plugin operates as if it were +always in directed mode instead of discovered mode. For instance, doctests +in test modules will always be found when using this plugin with the doctest +plugin. + +But the biggest issue you will face is probably concurrency. Unless you +have kept your tests as religiously pure unit tests, with no side-effects, no +ordering issues, and no external dependencies, chances are you will experience +odd, intermittent and unexplainable failures and errors when using this +plugin. This doesn't necessarily mean the plugin is broken; it may mean that +your test suite is not safe for concurrency. + +New Features in 1.1.0 +===================== + +* functions generated by test generators are now added to the worker queue + making them multi-threaded. +* fixed timeout functionality, now functions will be terminated with a + TimedOutException exception when they exceed their execution time. The + worker processes are not terminated. +* added ``--process-restartworker`` option to restart workers once they are + done, this helps control memory usage. Sometimes memory leaks can accumulate + making long runs very difficult. +* added global _instantiate_plugins to configure which plugins are started + on the worker processes. + +""" + +import logging +import os +import sys +import time +import traceback +import unittest +import pickle +import signal +import nose.case +from nose.core import TextTestRunner +from nose import failure +from nose import loader +from nose.plugins.base import Plugin +from nose.pyversion import bytes_ +from nose.result import TextTestResult +from nose.suite import ContextSuite +from nose.util import test_address +try: + # 2.7+ + from unittest.runner import _WritelnDecorator +except ImportError: + from unittest import _WritelnDecorator +from Queue import Empty +from warnings import warn +try: + from cStringIO import StringIO +except ImportError: + import StringIO + +# this is a list of plugin classes that will be checked for and created inside +# each worker process +_instantiate_plugins = None + +log = logging.getLogger(__name__) + +Process = Queue = Pool = Event = Value = Array = None + +# have to inherit KeyboardInterrupt to it will interrupt process properly +class TimedOutException(KeyboardInterrupt): + def __init__(self, value = "Timed Out"): + self.value = value + def __str__(self): + return repr(self.value) + +def _import_mp(): + global Process, Queue, Pool, Event, Value, Array + try: + from multiprocessing import Manager, Process + #prevent the server process created in the manager which holds Python + #objects and allows other processes to manipulate them using proxies + #to interrupt on SIGINT (keyboardinterrupt) so that the communication + #channel between subprocesses and main process is still usable after + #ctrl+C is received in the main process. + old=signal.signal(signal.SIGINT, signal.SIG_IGN) + m = Manager() + #reset it back so main process will receive a KeyboardInterrupt + #exception on ctrl+c + signal.signal(signal.SIGINT, old) + Queue, Pool, Event, Value, Array = ( + m.Queue, m.Pool, m.Event, m.Value, m.Array + ) + except ImportError: + warn("multiprocessing module is not available, multiprocess plugin " + "cannot be used", RuntimeWarning) + + +class TestLet: + def __init__(self, case): + try: + self._id = case.id() + except AttributeError: + pass + self._short_description = case.shortDescription() + self._str = str(case) + + def id(self): + return self._id + + def shortDescription(self): + return self._short_description + + def __str__(self): + return self._str + +class MultiProcess(Plugin): + """ + Run tests in multiple processes. Requires processing module. + """ + score = 1000 + status = {} + + def options(self, parser, env): + """ + Register command-line options. + """ + parser.add_option("--processes", action="store", + default=env.get('NOSE_PROCESSES', 0), + dest="multiprocess_workers", + metavar="NUM", + help="Spread test run among this many processes. " + "Set a number equal to the number of processors " + "or cores in your machine for best results. " + "Pass a negative number to have the number of " + "processes automatically set to the number of " + "cores. Passing 0 means to disable parallel " + "testing. Default is 0 unless NOSE_PROCESSES is " + "set. " + "[NOSE_PROCESSES]") + parser.add_option("--process-timeout", action="store", + default=env.get('NOSE_PROCESS_TIMEOUT', 10), + dest="multiprocess_timeout", + metavar="SECONDS", + help="Set timeout for return of results from each " + "test runner process. Default is 10. " + "[NOSE_PROCESS_TIMEOUT]") + parser.add_option("--process-restartworker", action="store_true", + default=env.get('NOSE_PROCESS_RESTARTWORKER', False), + dest="multiprocess_restartworker", + help="If set, will restart each worker process once" + " their tests are done, this helps control memory " + "leaks from killing the system. " + "[NOSE_PROCESS_RESTARTWORKER]") + + def configure(self, options, config): + """ + Configure plugin. + """ + try: + self.status.pop('active') + except KeyError: + pass + if not hasattr(options, 'multiprocess_workers'): + self.enabled = False + return + # don't start inside of a worker process + if config.worker: + return + self.config = config + try: + workers = int(options.multiprocess_workers) + except (TypeError, ValueError): + workers = 0 + if workers: + _import_mp() + if Process is None: + self.enabled = False + return + # Negative number of workers will cause multiprocessing to hang. + # Set the number of workers to the CPU count to avoid this. + if workers < 0: + try: + import multiprocessing + workers = multiprocessing.cpu_count() + except NotImplementedError: + self.enabled = False + return + self.enabled = True + self.config.multiprocess_workers = workers + t = float(options.multiprocess_timeout) + self.config.multiprocess_timeout = t + r = int(options.multiprocess_restartworker) + self.config.multiprocess_restartworker = r + self.status['active'] = True + + def prepareTestLoader(self, loader): + """Remember loader class so MultiProcessTestRunner can instantiate + the right loader. + """ + self.loaderClass = loader.__class__ + + def prepareTestRunner(self, runner): + """Replace test runner with MultiProcessTestRunner. + """ + # replace with our runner class + return MultiProcessTestRunner(stream=runner.stream, + verbosity=self.config.verbosity, + config=self.config, + loaderClass=self.loaderClass) + +def signalhandler(sig, frame): + raise TimedOutException() + +class MultiProcessTestRunner(TextTestRunner): + waitkilltime = 5.0 # max time to wait to terminate a process that does not + # respond to SIGILL + def __init__(self, **kw): + self.loaderClass = kw.pop('loaderClass', loader.defaultTestLoader) + super(MultiProcessTestRunner, self).__init__(**kw) + + def collect(self, test, testQueue, tasks, to_teardown, result): + # dispatch and collect results + # put indexes only on queue because tests aren't picklable + for case in self.nextBatch(test): + log.debug("Next batch %s (%s)", case, type(case)) + if (isinstance(case, nose.case.Test) and + isinstance(case.test, failure.Failure)): + log.debug("Case is a Failure") + case(result) # run here to capture the failure + continue + # handle shared fixtures + if isinstance(case, ContextSuite) and case.context is failure.Failure: + log.debug("Case is a Failure") + case(result) # run here to capture the failure + continue + elif isinstance(case, ContextSuite) and self.sharedFixtures(case): + log.debug("%s has shared fixtures", case) + try: + case.setUp() + except (KeyboardInterrupt, SystemExit): + raise + except: + log.debug("%s setup failed", sys.exc_info()) + result.addError(case, sys.exc_info()) + else: + to_teardown.append(case) + if case.factory: + ancestors=case.factory.context.get(case, []) + for an in ancestors[:2]: + #log.debug('reset ancestor %s', an) + if getattr(an, '_multiprocess_shared_', False): + an._multiprocess_can_split_=True + #an._multiprocess_shared_=False + self.collect(case, testQueue, tasks, to_teardown, result) + + else: + test_addr = self.addtask(testQueue,tasks,case) + log.debug("Queued test %s (%s) to %s", + len(tasks), test_addr, testQueue) + + def startProcess(self, iworker, testQueue, resultQueue, shouldStop, result): + currentaddr = Value('c',bytes_('')) + currentstart = Value('d',time.time()) + keyboardCaught = Event() + p = Process(target=runner, + args=(iworker, testQueue, + resultQueue, + currentaddr, + currentstart, + keyboardCaught, + shouldStop, + self.loaderClass, + result.__class__, + pickle.dumps(self.config))) + p.currentaddr = currentaddr + p.currentstart = currentstart + p.keyboardCaught = keyboardCaught + old = signal.signal(signal.SIGILL, signalhandler) + p.start() + signal.signal(signal.SIGILL, old) + return p + + def run(self, test): + """ + Execute the test (which may be a test suite). If the test is a suite, + distribute it out among as many processes as have been configured, at + as fine a level as is possible given the context fixtures defined in + the suite or any sub-suites. + + """ + log.debug("%s.run(%s) (%s)", self, test, os.getpid()) + wrapper = self.config.plugins.prepareTest(test) + if wrapper is not None: + test = wrapper + + # plugins can decorate or capture the output stream + wrapped = self.config.plugins.setOutputStream(self.stream) + if wrapped is not None: + self.stream = wrapped + + testQueue = Queue() + resultQueue = Queue() + tasks = [] + completed = [] + workers = [] + to_teardown = [] + shouldStop = Event() + + result = self._makeResult() + start = time.time() + + self.collect(test, testQueue, tasks, to_teardown, result) + + log.debug("Starting %s workers", self.config.multiprocess_workers) + for i in range(self.config.multiprocess_workers): + p = self.startProcess(i, testQueue, resultQueue, shouldStop, result) + workers.append(p) + log.debug("Started worker process %s", i+1) + + total_tasks = len(tasks) + # need to keep track of the next time to check for timeouts in case + # more than one process times out at the same time. + nexttimeout=self.config.multiprocess_timeout + thrownError = None + + try: + while tasks: + log.debug("Waiting for results (%s/%s tasks), next timeout=%.3fs", + len(completed), total_tasks,nexttimeout) + try: + iworker, addr, newtask_addrs, batch_result = resultQueue.get( + timeout=nexttimeout) + log.debug('Results received for worker %d, %s, new tasks: %d', + iworker,addr,len(newtask_addrs)) + try: + try: + tasks.remove(addr) + except ValueError: + log.warn('worker %s failed to remove from tasks: %s', + iworker,addr) + total_tasks += len(newtask_addrs) + tasks.extend(newtask_addrs) + except KeyError: + log.debug("Got result for unknown task? %s", addr) + log.debug("current: %s",str(list(tasks)[0])) + else: + completed.append([addr,batch_result]) + self.consolidate(result, batch_result) + if (self.config.stopOnError + and not result.wasSuccessful()): + # set the stop condition + shouldStop.set() + break + if self.config.multiprocess_restartworker: + log.debug('joining worker %s',iworker) + # wait for working, but not that important if worker + # cannot be joined in fact, for workers that add to + # testQueue, they will not terminate until all their + # items are read + workers[iworker].join(timeout=1) + if not shouldStop.is_set() and not testQueue.empty(): + log.debug('starting new process on worker %s',iworker) + workers[iworker] = self.startProcess(iworker, testQueue, resultQueue, shouldStop, result) + except Empty: + log.debug("Timed out with %s tasks pending " + "(empty testQueue=%r): %s", + len(tasks),testQueue.empty(),str(tasks)) + any_alive = False + for iworker, w in enumerate(workers): + if w.is_alive(): + worker_addr = bytes_(w.currentaddr.value,'ascii') + timeprocessing = time.time() - w.currentstart.value + if ( len(worker_addr) == 0 + and timeprocessing > self.config.multiprocess_timeout-0.1): + log.debug('worker %d has finished its work item, ' + 'but is not exiting? do we wait for it?', + iworker) + else: + any_alive = True + if (len(worker_addr) > 0 + and timeprocessing > self.config.multiprocess_timeout-0.1): + log.debug('timed out worker %s: %s', + iworker,worker_addr) + w.currentaddr.value = bytes_('') + # If the process is in C++ code, sending a SIGILL + # might not send a python KeybordInterrupt exception + # therefore, send multiple signals until an + # exception is caught. If this takes too long, then + # terminate the process + w.keyboardCaught.clear() + startkilltime = time.time() + while not w.keyboardCaught.is_set() and w.is_alive(): + if time.time()-startkilltime > self.waitkilltime: + # have to terminate... + log.error("terminating worker %s",iworker) + w.terminate() + # there is a small probability that the + # terminated process might send a result, + # which has to be specially handled or + # else processes might get orphaned. + workers[iworker] = w = self.startProcess(iworker, testQueue, resultQueue, shouldStop, result) + break + os.kill(w.pid, signal.SIGILL) + time.sleep(0.1) + if not any_alive and testQueue.empty(): + log.debug("All workers dead") + break + nexttimeout=self.config.multiprocess_timeout + for w in workers: + if w.is_alive() and len(w.currentaddr.value) > 0: + timeprocessing = time.time()-w.currentstart.value + if timeprocessing <= self.config.multiprocess_timeout: + nexttimeout = min(nexttimeout, + self.config.multiprocess_timeout-timeprocessing) + log.debug("Completed %s tasks (%s remain)", len(completed), len(tasks)) + + except (KeyboardInterrupt, SystemExit), e: + log.info('parent received ctrl-c when waiting for test results') + thrownError = e + #resultQueue.get(False) + + result.addError(test, sys.exc_info()) + + try: + for case in to_teardown: + log.debug("Tearing down shared fixtures for %s", case) + try: + case.tearDown() + except (KeyboardInterrupt, SystemExit): + raise + except: + result.addError(case, sys.exc_info()) + + stop = time.time() + + # first write since can freeze on shutting down processes + result.printErrors() + result.printSummary(start, stop) + self.config.plugins.finalize(result) + + if thrownError is None: + log.debug("Tell all workers to stop") + for w in workers: + if w.is_alive(): + testQueue.put('STOP', block=False) + + # wait for the workers to end + for iworker,worker in enumerate(workers): + if worker.is_alive(): + log.debug('joining worker %s',iworker) + worker.join() + if worker.is_alive(): + log.debug('failed to join worker %s',iworker) + except (KeyboardInterrupt, SystemExit): + log.info('parent received ctrl-c when shutting down: stop all processes') + for worker in workers: + if worker.is_alive(): + worker.terminate() + + if thrownError: raise thrownError + else: raise + + return result + + def addtask(testQueue,tasks,case): + arg = None + if isinstance(case,nose.case.Test) and hasattr(case.test,'arg'): + # this removes the top level descriptor and allows real function + # name to be returned + case.test.descriptor = None + arg = case.test.arg + test_addr = MultiProcessTestRunner.address(case) + testQueue.put((test_addr,arg), block=False) + if arg is not None: + test_addr += str(arg) + if tasks is not None: + tasks.append(test_addr) + return test_addr + addtask = staticmethod(addtask) + + def address(case): + if hasattr(case, 'address'): + file, mod, call = case.address() + elif hasattr(case, 'context'): + file, mod, call = test_address(case.context) + else: + raise Exception("Unable to convert %s to address" % case) + parts = [] + if file is None: + if mod is None: + raise Exception("Unaddressable case %s" % case) + else: + parts.append(mod) + else: + # strip __init__.py(c) from end of file part + # if present, having it there confuses loader + dirname, basename = os.path.split(file) + if basename.startswith('__init__'): + file = dirname + parts.append(file) + if call is not None: + parts.append(call) + return ':'.join(map(str, parts)) + address = staticmethod(address) + + def nextBatch(self, test): + # allows tests or suites to mark themselves as not safe + # for multiprocess execution + if hasattr(test, 'context'): + if not getattr(test.context, '_multiprocess_', True): + return + + if ((isinstance(test, ContextSuite) + and test.hasFixtures(self.checkCanSplit)) + or not getattr(test, 'can_split', True) + or not isinstance(test, unittest.TestSuite)): + # regular test case, or a suite with context fixtures + + # special case: when run like nosetests path/to/module.py + # the top-level suite has only one item, and it shares + # the same context as that item. In that case, we want the + # item, not the top-level suite + if isinstance(test, ContextSuite): + contained = list(test) + if (len(contained) == 1 + and getattr(contained[0], + 'context', None) == test.context): + test = contained[0] + yield test + else: + # Suite is without fixtures at this level; but it may have + # fixtures at any deeper level, so we need to examine it all + # the way down to the case level + for case in test: + for batch in self.nextBatch(case): + yield batch + + def checkCanSplit(context, fixt): + """ + Callback that we use to check whether the fixtures found in a + context or ancestor are ones we care about. + + Contexts can tell us that their fixtures are reentrant by setting + _multiprocess_can_split_. So if we see that, we return False to + disregard those fixtures. + """ + if not fixt: + return False + if getattr(context, '_multiprocess_can_split_', False): + return False + return True + checkCanSplit = staticmethod(checkCanSplit) + + def sharedFixtures(self, case): + context = getattr(case, 'context', None) + if not context: + return False + return getattr(context, '_multiprocess_shared_', False) + + def consolidate(self, result, batch_result): + log.debug("batch result is %s" , batch_result) + try: + output, testsRun, failures, errors, errorClasses = batch_result + except ValueError: + log.debug("result in unexpected format %s", batch_result) + failure.Failure(*sys.exc_info())(result) + return + self.stream.write(output) + result.testsRun += testsRun + result.failures.extend(failures) + result.errors.extend(errors) + for key, (storage, label, isfail) in errorClasses.items(): + if key not in result.errorClasses: + # Ordinarily storage is result attribute + # but it's only processed through the errorClasses + # dict, so it's ok to fake it here + result.errorClasses[key] = ([], label, isfail) + mystorage, _junk, _junk = result.errorClasses[key] + mystorage.extend(storage) + log.debug("Ran %s tests (total: %s)", testsRun, result.testsRun) + + +def runner(ix, testQueue, resultQueue, currentaddr, currentstart, + keyboardCaught, shouldStop, loaderClass, resultClass, config): + try: + try: + return __runner(ix, testQueue, resultQueue, currentaddr, currentstart, + keyboardCaught, shouldStop, loaderClass, resultClass, config) + except KeyboardInterrupt: + log.debug('Worker %s keyboard interrupt, stopping',ix) + except Empty: + log.debug("Worker %s timed out waiting for tasks", ix) + +def __runner(ix, testQueue, resultQueue, currentaddr, currentstart, + keyboardCaught, shouldStop, loaderClass, resultClass, config): + + config = pickle.loads(config) + dummy_parser = config.parserClass() + if _instantiate_plugins is not None: + for pluginclass in _instantiate_plugins: + plugin = pluginclass() + plugin.addOptions(dummy_parser,{}) + config.plugins.addPlugin(plugin) + config.plugins.configure(config.options,config) + config.plugins.begin() + log.debug("Worker %s executing, pid=%d", ix,os.getpid()) + loader = loaderClass(config=config) + loader.suiteClass.suiteClass = NoSharedFixtureContextSuite + + def get(): + return testQueue.get(timeout=config.multiprocess_timeout) + + def makeResult(): + stream = _WritelnDecorator(StringIO()) + result = resultClass(stream, descriptions=1, + verbosity=config.verbosity, + config=config) + plug_result = config.plugins.prepareTestResult(result) + if plug_result: + return plug_result + return result + + def batch(result): + failures = [(TestLet(c), err) for c, err in result.failures] + errors = [(TestLet(c), err) for c, err in result.errors] + errorClasses = {} + for key, (storage, label, isfail) in result.errorClasses.items(): + errorClasses[key] = ([(TestLet(c), err) for c, err in storage], + label, isfail) + return ( + result.stream.getvalue(), + result.testsRun, + failures, + errors, + errorClasses) + for test_addr, arg in iter(get, 'STOP'): + if shouldStop.is_set(): + log.exception('Worker %d STOPPED',ix) + break + result = makeResult() + test = loader.loadTestsFromNames([test_addr]) + test.testQueue = testQueue + test.tasks = [] + test.arg = arg + log.debug("Worker %s Test is %s (%s)", ix, test_addr, test) + try: + if arg is not None: + test_addr = test_addr + str(arg) + currentaddr.value = bytes_(test_addr) + currentstart.value = time.time() + test(result) + currentaddr.value = bytes_('') + resultQueue.put((ix, test_addr, test.tasks, batch(result))) + except KeyboardInterrupt, e: #TimedOutException: + timeout = isinstance(e, TimedOutException) + if timeout: + keyboardCaught.set() + if len(currentaddr.value): + if timeout: + msg = 'Worker %s timed out, failing current test %s' + else: + msg = 'Worker %s keyboard interrupt, failing current test %s' + log.exception(msg,ix,test_addr) + currentaddr.value = bytes_('') + failure.Failure(*sys.exc_info())(result) + resultQueue.put((ix, test_addr, test.tasks, batch(result))) + else: + if timeout: + msg = 'Worker %s test %s timed out' + else: + msg = 'Worker %s test %s keyboard interrupt' + log.debug(msg,ix,test_addr) + resultQueue.put((ix, test_addr, test.tasks, batch(result))) + if not timeout: + raise + except SystemExit: + currentaddr.value = bytes_('') + log.exception('Worker %s system exit',ix) + raise + except: + currentaddr.value = bytes_('') + log.exception("Worker %s error running test or returning " + "results",ix) + failure.Failure(*sys.exc_info())(result) + resultQueue.put((ix, test_addr, test.tasks, batch(result))) + if config.multiprocess_restartworker: + break + log.debug("Worker %s ending", ix) + + +class NoSharedFixtureContextSuite(ContextSuite): + """ + Context suite that never fires shared fixtures. + + When a context sets _multiprocess_shared_, fixtures in that context + are executed by the main process. Using this suite class prevents them + from executing in the runner process as well. + + """ + testQueue = None + tasks = None + arg = None + def setupContext(self, context): + if getattr(context, '_multiprocess_shared_', False): + return + super(NoSharedFixtureContextSuite, self).setupContext(context) + + def teardownContext(self, context): + if getattr(context, '_multiprocess_shared_', False): + return + super(NoSharedFixtureContextSuite, self).teardownContext(context) + def run(self, result): + """Run tests in suite inside of suite fixtures. + """ + # proxy the result for myself + log.debug("suite %s (%s) run called, tests: %s", + id(self), self, self._tests) + if self.resultProxy: + result, orig = self.resultProxy(result, self), result + else: + result, orig = result, result + try: + #log.debug('setUp for %s', id(self)); + self.setUp() + except KeyboardInterrupt: + raise + except: + self.error_context = 'setup' + result.addError(self, self._exc_info()) + return + try: + for test in self._tests: + if (isinstance(test,nose.case.Test) + and self.arg is not None): + test.test.arg = self.arg + else: + test.arg = self.arg + test.testQueue = self.testQueue + test.tasks = self.tasks + if result.shouldStop: + log.debug("stopping") + break + # each nose.case.Test will create its own result proxy + # so the cases need the original result, to avoid proxy + # chains + #log.debug('running test %s in suite %s', test, self); + try: + test(orig) + except KeyboardInterrupt, e: + timeout = isinstance(e, TimedOutException) + if timeout: + msg = 'Timeout when running test %s in suite %s' + else: + msg = 'KeyboardInterrupt when running test %s in suite %s' + log.debug(msg, test, self) + err = (TimedOutException,TimedOutException(str(test)), + sys.exc_info()[2]) + test.config.plugins.addError(test,err) + orig.addError(test,err) + if not timeout: + raise + finally: + self.has_run = True + try: + #log.debug('tearDown for %s', id(self)); + self.tearDown() + except KeyboardInterrupt: + raise + except: + self.error_context = 'teardown' + result.addError(self, self._exc_info()) diff --git a/lib/spack/external/nose/plugins/plugintest.py b/lib/spack/external/nose/plugins/plugintest.py new file mode 100644 index 00000000000..76d0d2c48cf --- /dev/null +++ b/lib/spack/external/nose/plugins/plugintest.py @@ -0,0 +1,416 @@ +""" +Testing Plugins +=============== + +The plugin interface is well-tested enough to safely unit test your +use of its hooks with some level of confidence. However, there is also +a mixin for unittest.TestCase called PluginTester that's designed to +test plugins in their native runtime environment. + +Here's a simple example with a do-nothing plugin and a composed suite. + + >>> import unittest + >>> from nose.plugins import Plugin, PluginTester + >>> class FooPlugin(Plugin): + ... pass + >>> class TestPluginFoo(PluginTester, unittest.TestCase): + ... activate = '--with-foo' + ... plugins = [FooPlugin()] + ... def test_foo(self): + ... for line in self.output: + ... # i.e. check for patterns + ... pass + ... + ... # or check for a line containing ... + ... assert "ValueError" in self.output + ... def makeSuite(self): + ... class TC(unittest.TestCase): + ... def runTest(self): + ... raise ValueError("I hate foo") + ... return [TC('runTest')] + ... + >>> res = unittest.TestResult() + >>> case = TestPluginFoo('test_foo') + >>> _ = case(res) + >>> res.errors + [] + >>> res.failures + [] + >>> res.wasSuccessful() + True + >>> res.testsRun + 1 + +And here is a more complex example of testing a plugin that has extra +arguments and reads environment variables. + + >>> import unittest, os + >>> from nose.plugins import Plugin, PluginTester + >>> class FancyOutputter(Plugin): + ... name = "fancy" + ... def configure(self, options, conf): + ... Plugin.configure(self, options, conf) + ... if not self.enabled: + ... return + ... self.fanciness = 1 + ... if options.more_fancy: + ... self.fanciness = 2 + ... if 'EVEN_FANCIER' in self.env: + ... self.fanciness = 3 + ... + ... def options(self, parser, env=os.environ): + ... self.env = env + ... parser.add_option('--more-fancy', action='store_true') + ... Plugin.options(self, parser, env=env) + ... + ... def report(self, stream): + ... stream.write("FANCY " * self.fanciness) + ... + >>> class TestFancyOutputter(PluginTester, unittest.TestCase): + ... activate = '--with-fancy' # enables the plugin + ... plugins = [FancyOutputter()] + ... args = ['--more-fancy'] + ... env = {'EVEN_FANCIER': '1'} + ... + ... def test_fancy_output(self): + ... assert "FANCY FANCY FANCY" in self.output, ( + ... "got: %s" % self.output) + ... def makeSuite(self): + ... class TC(unittest.TestCase): + ... def runTest(self): + ... raise ValueError("I hate fancy stuff") + ... return [TC('runTest')] + ... + >>> res = unittest.TestResult() + >>> case = TestFancyOutputter('test_fancy_output') + >>> _ = case(res) + >>> res.errors + [] + >>> res.failures + [] + >>> res.wasSuccessful() + True + >>> res.testsRun + 1 + +""" + +import re +import sys +from warnings import warn + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +__all__ = ['PluginTester', 'run'] + +from os import getpid +class MultiProcessFile(object): + """ + helper for testing multiprocessing + + multiprocessing poses a problem for doctests, since the strategy + of replacing sys.stdout/stderr with file-like objects then + inspecting the results won't work: the child processes will + write to the objects, but the data will not be reflected + in the parent doctest-ing process. + + The solution is to create file-like objects which will interact with + multiprocessing in a more desirable way. + + All processes can write to this object, but only the creator can read. + This allows the testing system to see a unified picture of I/O. + """ + def __init__(self): + # per advice at: + # http://docs.python.org/library/multiprocessing.html#all-platforms + self.__master = getpid() + self.__queue = Manager().Queue() + self.__buffer = StringIO() + self.softspace = 0 + + def buffer(self): + if getpid() != self.__master: + return + + from Queue import Empty + from collections import defaultdict + cache = defaultdict(str) + while True: + try: + pid, data = self.__queue.get_nowait() + except Empty: + break + if pid == (): + #show parent output after children + #this is what users see, usually + pid = ( 1e100, ) # googol! + cache[pid] += data + for pid in sorted(cache): + #self.__buffer.write( '%s wrote: %r\n' % (pid, cache[pid]) ) #DEBUG + self.__buffer.write( cache[pid] ) + def write(self, data): + # note that these pids are in the form of current_process()._identity + # rather than OS pids + from multiprocessing import current_process + pid = current_process()._identity + self.__queue.put((pid, data)) + def __iter__(self): + "getattr doesn't work for iter()" + self.buffer() + return self.__buffer + def seek(self, offset, whence=0): + self.buffer() + return self.__buffer.seek(offset, whence) + def getvalue(self): + self.buffer() + return self.__buffer.getvalue() + def __getattr__(self, attr): + return getattr(self.__buffer, attr) + +try: + from multiprocessing import Manager + Buffer = MultiProcessFile +except ImportError: + Buffer = StringIO + +class PluginTester(object): + """A mixin for testing nose plugins in their runtime environment. + + Subclass this and mix in unittest.TestCase to run integration/functional + tests on your plugin. When setUp() is called, the stub test suite is + executed with your plugin so that during an actual test you can inspect the + artifacts of how your plugin interacted with the stub test suite. + + - activate + + - the argument to send nosetests to activate the plugin + + - suitepath + + - if set, this is the path of the suite to test. Otherwise, you + will need to use the hook, makeSuite() + + - plugins + + - the list of plugins to make available during the run. Note + that this does not mean these plugins will be *enabled* during + the run -- only the plugins enabled by the activate argument + or other settings in argv or env will be enabled. + + - args + + - a list of arguments to add to the nosetests command, in addition to + the activate argument + + - env + + - optional dict of environment variables to send nosetests + + """ + activate = None + suitepath = None + args = None + env = {} + argv = None + plugins = [] + ignoreFiles = None + + def makeSuite(self): + """returns a suite object of tests to run (unittest.TestSuite()) + + If self.suitepath is None, this must be implemented. The returned suite + object will be executed with all plugins activated. It may return + None. + + Here is an example of a basic suite object you can return :: + + >>> import unittest + >>> class SomeTest(unittest.TestCase): + ... def runTest(self): + ... raise ValueError("Now do something, plugin!") + ... + >>> unittest.TestSuite([SomeTest()]) # doctest: +ELLIPSIS + ]> + + """ + raise NotImplementedError + + def _execPlugin(self): + """execute the plugin on the internal test suite. + """ + from nose.config import Config + from nose.core import TestProgram + from nose.plugins.manager import PluginManager + + suite = None + stream = Buffer() + conf = Config(env=self.env, + stream=stream, + plugins=PluginManager(plugins=self.plugins)) + if self.ignoreFiles is not None: + conf.ignoreFiles = self.ignoreFiles + if not self.suitepath: + suite = self.makeSuite() + + self.nose = TestProgram(argv=self.argv, config=conf, suite=suite, + exit=False) + self.output = AccessDecorator(stream) + + def setUp(self): + """runs nosetests with the specified test suite, all plugins + activated. + """ + self.argv = ['nosetests', self.activate] + if self.args: + self.argv.extend(self.args) + if self.suitepath: + self.argv.append(self.suitepath) + + self._execPlugin() + + +class AccessDecorator(object): + stream = None + _buf = None + def __init__(self, stream): + self.stream = stream + stream.seek(0) + self._buf = stream.read() + stream.seek(0) + def __contains__(self, val): + return val in self._buf + def __iter__(self): + return iter(self.stream) + def __str__(self): + return self._buf + + +def blankline_separated_blocks(text): + "a bunch of === characters is also considered a blank line" + block = [] + for line in text.splitlines(True): + block.append(line) + line = line.strip() + if not line or line.startswith('===') and not line.strip('='): + yield "".join(block) + block = [] + if block: + yield "".join(block) + + +def remove_stack_traces(out): + # this regexp taken from Python 2.5's doctest + traceback_re = re.compile(r""" + # Grab the traceback header. Different versions of Python have + # said different things on the first traceback line. + ^(?P Traceback\ \( + (?: most\ recent\ call\ last + | innermost\ last + ) \) : + ) + \s* $ # toss trailing whitespace on the header. + (?P .*?) # don't blink: absorb stuff until... + ^(?=\w) # a line *starts* with alphanum. + .*?(?P \w+ ) # exception name + (?P [:\n] .*) # the rest + """, re.VERBOSE | re.MULTILINE | re.DOTALL) + blocks = [] + for block in blankline_separated_blocks(out): + blocks.append(traceback_re.sub(r"\g\n...\n\g\g", block)) + return "".join(blocks) + + +def simplify_warnings(out): + warn_re = re.compile(r""" + # Cut the file and line no, up to the warning name + ^.*:\d+:\s + (?P\w+): \s+ # warning category + (?P.+) $ \n? # warning message + ^ .* $ # stack frame + """, re.VERBOSE | re.MULTILINE) + return warn_re.sub(r"\g: \g", out) + + +def remove_timings(out): + return re.sub( + r"Ran (\d+ tests?) in [0-9.]+s", r"Ran \1 in ...s", out) + + +def munge_nose_output_for_doctest(out): + """Modify nose output to make it easy to use in doctests.""" + out = remove_stack_traces(out) + out = simplify_warnings(out) + out = remove_timings(out) + return out.strip() + + +def run(*arg, **kw): + """ + Specialized version of nose.run for use inside of doctests that + test test runs. + + This version of run() prints the result output to stdout. Before + printing, the output is processed by replacing the timing + information with an ellipsis (...), removing traceback stacks, and + removing trailing whitespace. + + Use this version of run wherever you are writing a doctest that + tests nose (or unittest) test result output. + + Note: do not use doctest: +ELLIPSIS when testing nose output, + since ellipses ("test_foo ... ok") in your expected test runner + output may match multiple lines of output, causing spurious test + passes! + """ + from nose import run + from nose.config import Config + from nose.plugins.manager import PluginManager + + buffer = Buffer() + if 'config' not in kw: + plugins = kw.pop('plugins', []) + if isinstance(plugins, list): + plugins = PluginManager(plugins=plugins) + env = kw.pop('env', {}) + kw['config'] = Config(env=env, plugins=plugins) + if 'argv' not in kw: + kw['argv'] = ['nosetests', '-v'] + kw['config'].stream = buffer + + # Set up buffering so that all output goes to our buffer, + # or warn user if deprecated behavior is active. If this is not + # done, prints and warnings will either be out of place or + # disappear. + stderr = sys.stderr + stdout = sys.stdout + if kw.pop('buffer_all', False): + sys.stdout = sys.stderr = buffer + restore = True + else: + restore = False + warn("The behavior of nose.plugins.plugintest.run() will change in " + "the next release of nose. The current behavior does not " + "correctly account for output to stdout and stderr. To enable " + "correct behavior, use run_buffered() instead, or pass " + "the keyword argument buffer_all=True to run().", + DeprecationWarning, stacklevel=2) + try: + run(*arg, **kw) + finally: + if restore: + sys.stderr = stderr + sys.stdout = stdout + out = buffer.getvalue() + print munge_nose_output_for_doctest(out) + + +def run_buffered(*arg, **kw): + kw['buffer_all'] = True + run(*arg, **kw) + +if __name__ == '__main__': + import doctest + doctest.testmod() diff --git a/lib/spack/external/nose/plugins/prof.py b/lib/spack/external/nose/plugins/prof.py new file mode 100644 index 00000000000..4d304a934bd --- /dev/null +++ b/lib/spack/external/nose/plugins/prof.py @@ -0,0 +1,154 @@ +"""This plugin will run tests using the hotshot profiler, which is part +of the standard library. To turn it on, use the ``--with-profile`` option +or set the NOSE_WITH_PROFILE environment variable. Profiler output can be +controlled with the ``--profile-sort`` and ``--profile-restrict`` options, +and the profiler output file may be changed with ``--profile-stats-file``. + +See the `hotshot documentation`_ in the standard library documentation for +more details on the various output options. + +.. _hotshot documentation: http://docs.python.org/library/hotshot.html +""" + +try: + import hotshot + from hotshot import stats +except ImportError: + hotshot, stats = None, None +import logging +import os +import sys +import tempfile +from nose.plugins.base import Plugin +from nose.util import tolist + +log = logging.getLogger('nose.plugins') + +class Profile(Plugin): + """ + Use this plugin to run tests using the hotshot profiler. + """ + pfile = None + clean_stats_file = False + def options(self, parser, env): + """Register commandline options. + """ + if not self.available(): + return + Plugin.options(self, parser, env) + parser.add_option('--profile-sort', action='store', dest='profile_sort', + default=env.get('NOSE_PROFILE_SORT', 'cumulative'), + metavar="SORT", + help="Set sort order for profiler output") + parser.add_option('--profile-stats-file', action='store', + dest='profile_stats_file', + metavar="FILE", + default=env.get('NOSE_PROFILE_STATS_FILE'), + help='Profiler stats file; default is a new ' + 'temp file on each run') + parser.add_option('--profile-restrict', action='append', + dest='profile_restrict', + metavar="RESTRICT", + default=env.get('NOSE_PROFILE_RESTRICT'), + help="Restrict profiler output. See help for " + "pstats.Stats for details") + + def available(cls): + return hotshot is not None + available = classmethod(available) + + def begin(self): + """Create profile stats file and load profiler. + """ + if not self.available(): + return + self._create_pfile() + self.prof = hotshot.Profile(self.pfile) + + def configure(self, options, conf): + """Configure plugin. + """ + if not self.available(): + self.enabled = False + return + Plugin.configure(self, options, conf) + self.conf = conf + if options.profile_stats_file: + self.pfile = options.profile_stats_file + self.clean_stats_file = False + else: + self.pfile = None + self.clean_stats_file = True + self.fileno = None + self.sort = options.profile_sort + self.restrict = tolist(options.profile_restrict) + + def prepareTest(self, test): + """Wrap entire test run in :func:`prof.runcall`. + """ + if not self.available(): + return + log.debug('preparing test %s' % test) + def run_and_profile(result, prof=self.prof, test=test): + self._create_pfile() + prof.runcall(test, result) + return run_and_profile + + def report(self, stream): + """Output profiler report. + """ + log.debug('printing profiler report') + self.prof.close() + prof_stats = stats.load(self.pfile) + prof_stats.sort_stats(self.sort) + + # 2.5 has completely different stream handling from 2.4 and earlier. + # Before 2.5, stats objects have no stream attribute; in 2.5 and later + # a reference sys.stdout is stored before we can tweak it. + compat_25 = hasattr(prof_stats, 'stream') + if compat_25: + tmp = prof_stats.stream + prof_stats.stream = stream + else: + tmp = sys.stdout + sys.stdout = stream + try: + if self.restrict: + log.debug('setting profiler restriction to %s', self.restrict) + prof_stats.print_stats(*self.restrict) + else: + prof_stats.print_stats() + finally: + if compat_25: + prof_stats.stream = tmp + else: + sys.stdout = tmp + + def finalize(self, result): + """Clean up stats file, if configured to do so. + """ + if not self.available(): + return + try: + self.prof.close() + except AttributeError: + # TODO: is this trying to catch just the case where not + # hasattr(self.prof, "close")? If so, the function call should be + # moved out of the try: suite. + pass + if self.clean_stats_file: + if self.fileno: + try: + os.close(self.fileno) + except OSError: + pass + try: + os.unlink(self.pfile) + except OSError: + pass + return None + + def _create_pfile(self): + if not self.pfile: + self.fileno, self.pfile = tempfile.mkstemp() + self.clean_stats_file = True diff --git a/lib/spack/external/nose/plugins/skip.py b/lib/spack/external/nose/plugins/skip.py new file mode 100644 index 00000000000..9d1ac8f6042 --- /dev/null +++ b/lib/spack/external/nose/plugins/skip.py @@ -0,0 +1,63 @@ +""" +This plugin installs a SKIP error class for the SkipTest exception. +When SkipTest is raised, the exception will be logged in the skipped +attribute of the result, 'S' or 'SKIP' (verbose) will be output, and +the exception will not be counted as an error or failure. This plugin +is enabled by default but may be disabled with the ``--no-skip`` option. +""" + +from nose.plugins.errorclass import ErrorClass, ErrorClassPlugin + + +# on SkipTest: +# - unittest SkipTest is first preference, but it's only available +# for >= 2.7 +# - unittest2 SkipTest is second preference for older pythons. This +# mirrors logic for choosing SkipTest exception in testtools +# - if none of the above, provide custom class +try: + from unittest.case import SkipTest +except ImportError: + try: + from unittest2.case import SkipTest + except ImportError: + class SkipTest(Exception): + """Raise this exception to mark a test as skipped. + """ + pass + + +class Skip(ErrorClassPlugin): + """ + Plugin that installs a SKIP error class for the SkipTest + exception. When SkipTest is raised, the exception will be logged + in the skipped attribute of the result, 'S' or 'SKIP' (verbose) + will be output, and the exception will not be counted as an error + or failure. + """ + enabled = True + skipped = ErrorClass(SkipTest, + label='SKIP', + isfailure=False) + + def options(self, parser, env): + """ + Add my options to command line. + """ + env_opt = 'NOSE_WITHOUT_SKIP' + parser.add_option('--no-skip', action='store_true', + dest='noSkip', default=env.get(env_opt, False), + help="Disable special handling of SkipTest " + "exceptions.") + + def configure(self, options, conf): + """ + Configure plugin. Skip plugin is enabled by default. + """ + if not self.can_configure: + return + self.conf = conf + disable = getattr(options, 'noSkip', False) + if disable: + self.enabled = False + diff --git a/lib/spack/external/nose/plugins/testid.py b/lib/spack/external/nose/plugins/testid.py new file mode 100644 index 00000000000..ae8119bd010 --- /dev/null +++ b/lib/spack/external/nose/plugins/testid.py @@ -0,0 +1,311 @@ +""" +This plugin adds a test id (like #1) to each test name output. After +you've run once to generate test ids, you can re-run individual +tests by activating the plugin and passing the ids (with or +without the # prefix) instead of test names. + +For example, if your normal test run looks like:: + + % nosetests -v + tests.test_a ... ok + tests.test_b ... ok + tests.test_c ... ok + +When adding ``--with-id`` you'll see:: + + % nosetests -v --with-id + #1 tests.test_a ... ok + #2 tests.test_b ... ok + #3 tests.test_c ... ok + +Then you can re-run individual tests by supplying just an id number:: + + % nosetests -v --with-id 2 + #2 tests.test_b ... ok + +You can also pass multiple id numbers:: + + % nosetests -v --with-id 2 3 + #2 tests.test_b ... ok + #3 tests.test_c ... ok + +Since most shells consider '#' a special character, you can leave it out when +specifying a test id. + +Note that when run without the -v switch, no special output is displayed, but +the ids file is still written. + +Looping over failed tests +------------------------- + +This plugin also adds a mode that will direct the test runner to record +failed tests. Subsequent test runs will then run only the tests that failed +last time. Activate this mode with the ``--failed`` switch:: + + % nosetests -v --failed + #1 test.test_a ... ok + #2 test.test_b ... ERROR + #3 test.test_c ... FAILED + #4 test.test_d ... ok + +On the second run, only tests #2 and #3 will run:: + + % nosetests -v --failed + #2 test.test_b ... ERROR + #3 test.test_c ... FAILED + +As you correct errors and tests pass, they'll drop out of subsequent runs. + +First:: + + % nosetests -v --failed + #2 test.test_b ... ok + #3 test.test_c ... FAILED + +Second:: + + % nosetests -v --failed + #3 test.test_c ... FAILED + +When all tests pass, the full set will run on the next invocation. + +First:: + + % nosetests -v --failed + #3 test.test_c ... ok + +Second:: + + % nosetests -v --failed + #1 test.test_a ... ok + #2 test.test_b ... ok + #3 test.test_c ... ok + #4 test.test_d ... ok + +.. note :: + + If you expect to use ``--failed`` regularly, it's a good idea to always run + using the ``--with-id`` option. This will ensure that an id file is always + created, allowing you to add ``--failed`` to the command line as soon as + you have failing tests. Otherwise, your first run using ``--failed`` will + (perhaps surprisingly) run *all* tests, because there won't be an id file + containing the record of failed tests from your previous run. + +""" +__test__ = False + +import logging +import os +from nose.plugins import Plugin +from nose.util import src, set + +try: + from cPickle import dump, load +except ImportError: + from pickle import dump, load + +log = logging.getLogger(__name__) + + +class TestId(Plugin): + """ + Activate to add a test id (like #1) to each test name output. Activate + with --failed to rerun failing tests only. + """ + name = 'id' + idfile = None + collecting = True + loopOnFailed = False + + def options(self, parser, env): + """Register commandline options. + """ + Plugin.options(self, parser, env) + parser.add_option('--id-file', action='store', dest='testIdFile', + default='.noseids', metavar="FILE", + help="Store test ids found in test runs in this " + "file. Default is the file .noseids in the " + "working directory.") + parser.add_option('--failed', action='store_true', + dest='failed', default=False, + help="Run the tests that failed in the last " + "test run.") + + def configure(self, options, conf): + """Configure plugin. + """ + Plugin.configure(self, options, conf) + if options.failed: + self.enabled = True + self.loopOnFailed = True + log.debug("Looping on failed tests") + self.idfile = os.path.expanduser(options.testIdFile) + if not os.path.isabs(self.idfile): + self.idfile = os.path.join(conf.workingDir, self.idfile) + self.id = 1 + # Ids and tests are mirror images: ids are {id: test address} and + # tests are {test address: id} + self.ids = {} + self.tests = {} + self.failed = [] + self.source_names = [] + # used to track ids seen when tests is filled from + # loaded ids file + self._seen = {} + self._write_hashes = conf.verbosity >= 2 + + def finalize(self, result): + """Save new ids file, if needed. + """ + if result.wasSuccessful(): + self.failed = [] + if self.collecting: + ids = dict(list(zip(list(self.tests.values()), list(self.tests.keys())))) + else: + ids = self.ids + fh = open(self.idfile, 'wb') + dump({'ids': ids, + 'failed': self.failed, + 'source_names': self.source_names}, fh) + fh.close() + log.debug('Saved test ids: %s, failed %s to %s', + ids, self.failed, self.idfile) + + def loadTestsFromNames(self, names, module=None): + """Translate ids in the list of requested names into their + test addresses, if they are found in my dict of tests. + """ + log.debug('ltfn %s %s', names, module) + try: + fh = open(self.idfile, 'rb') + data = load(fh) + if 'ids' in data: + self.ids = data['ids'] + self.failed = data['failed'] + self.source_names = data['source_names'] + else: + # old ids field + self.ids = data + self.failed = [] + self.source_names = names + if self.ids: + self.id = max(self.ids) + 1 + self.tests = dict(list(zip(list(self.ids.values()), list(self.ids.keys())))) + else: + self.id = 1 + log.debug( + 'Loaded test ids %s tests %s failed %s sources %s from %s', + self.ids, self.tests, self.failed, self.source_names, + self.idfile) + fh.close() + except ValueError, e: + # load() may throw a ValueError when reading the ids file, if it + # was generated with a newer version of Python than we are currently + # running. + log.debug('Error loading %s : %s', self.idfile, str(e)) + except IOError: + log.debug('IO error reading %s', self.idfile) + + if self.loopOnFailed and self.failed: + self.collecting = False + names = self.failed + self.failed = [] + # I don't load any tests myself, only translate names like '#2' + # into the associated test addresses + translated = [] + new_source = [] + really_new = [] + for name in names: + trans = self.tr(name) + if trans != name: + translated.append(trans) + else: + new_source.append(name) + # names that are not ids and that are not in the current + # list of source names go into the list for next time + if new_source: + new_set = set(new_source) + old_set = set(self.source_names) + log.debug("old: %s new: %s", old_set, new_set) + really_new = [s for s in new_source + if not s in old_set] + if really_new: + # remember new sources + self.source_names.extend(really_new) + if not translated: + # new set of source names, no translations + # means "run the requested tests" + names = new_source + else: + # no new names to translate and add to id set + self.collecting = False + log.debug("translated: %s new sources %s names %s", + translated, really_new, names) + return (None, translated + really_new or names) + + def makeName(self, addr): + log.debug("Make name %s", addr) + filename, module, call = addr + if filename is not None: + head = src(filename) + else: + head = module + if call is not None: + return "%s:%s" % (head, call) + return head + + def setOutputStream(self, stream): + """Get handle on output stream so the plugin can print id #s + """ + self.stream = stream + + def startTest(self, test): + """Maybe output an id # before the test name. + + Example output:: + + #1 test.test ... ok + #2 test.test_two ... ok + + """ + adr = test.address() + log.debug('start test %s (%s)', adr, adr in self.tests) + if adr in self.tests: + if adr in self._seen: + self.write(' ') + else: + self.write('#%s ' % self.tests[adr]) + self._seen[adr] = 1 + return + self.tests[adr] = self.id + self.write('#%s ' % self.id) + self.id += 1 + + def afterTest(self, test): + # None means test never ran, False means failed/err + if test.passed is False: + try: + key = str(self.tests[test.address()]) + except KeyError: + # never saw this test -- startTest didn't run + pass + else: + if key not in self.failed: + self.failed.append(key) + + def tr(self, name): + log.debug("tr '%s'", name) + try: + key = int(name.replace('#', '')) + except ValueError: + return name + log.debug("Got key %s", key) + # I'm running tests mapped from the ids file, + # not collecting new ones + if key in self.ids: + return self.makeName(self.ids[key]) + return name + + def write(self, output): + if self._write_hashes: + self.stream.write(output) diff --git a/lib/spack/external/nose/plugins/xunit.py b/lib/spack/external/nose/plugins/xunit.py new file mode 100644 index 00000000000..90b52f5f610 --- /dev/null +++ b/lib/spack/external/nose/plugins/xunit.py @@ -0,0 +1,341 @@ +"""This plugin provides test results in the standard XUnit XML format. + +It's designed for the `Jenkins`_ (previously Hudson) continuous build +system, but will probably work for anything else that understands an +XUnit-formatted XML representation of test results. + +Add this shell command to your builder :: + + nosetests --with-xunit + +And by default a file named nosetests.xml will be written to the +working directory. + +In a Jenkins builder, tick the box named "Publish JUnit test result report" +under the Post-build Actions and enter this value for Test report XMLs:: + + **/nosetests.xml + +If you need to change the name or location of the file, you can set the +``--xunit-file`` option. + +If you need to change the name of the test suite, you can set the +``--xunit-testsuite-name`` option. + +Here is an abbreviated version of what an XML test report might look like:: + + + + + + Traceback (most recent call last): + ... + TypeError: oops, wrong type + + + + +.. _Jenkins: http://jenkins-ci.org/ + +""" +import codecs +import doctest +import os +import sys +import traceback +import re +import inspect +from StringIO import StringIO +from time import time +from xml.sax import saxutils + +from nose.plugins.base import Plugin +from nose.exc import SkipTest +from nose.pyversion import force_unicode, format_exception + +# Invalid XML characters, control characters 0-31 sans \t, \n and \r +CONTROL_CHARACTERS = re.compile(r"[\000-\010\013\014\016-\037]") + +TEST_ID = re.compile(r'^(.*?)(\(.*\))$') + +def xml_safe(value): + """Replaces invalid XML characters with '?'.""" + return CONTROL_CHARACTERS.sub('?', value) + +def escape_cdata(cdata): + """Escape a string for an XML CDATA section.""" + return xml_safe(cdata).replace(']]>', ']]>]]>>> nice_classname(Exception()) # doctest: +ELLIPSIS + '...Exception' + >>> nice_classname(Exception) # doctest: +ELLIPSIS + '...Exception' + + """ + if inspect.isclass(obj): + cls_name = obj.__name__ + else: + cls_name = obj.__class__.__name__ + mod = inspect.getmodule(obj) + if mod: + name = mod.__name__ + # jython + if name.startswith('org.python.core.'): + name = name[len('org.python.core.'):] + return "%s.%s" % (name, cls_name) + else: + return cls_name + +def exc_message(exc_info): + """Return the exception's message.""" + exc = exc_info[1] + if exc is None: + # str exception + result = exc_info[0] + else: + try: + result = str(exc) + except UnicodeEncodeError: + try: + result = unicode(exc) + except UnicodeError: + # Fallback to args as neither str nor + # unicode(Exception(u'\xe6')) work in Python < 2.6 + result = exc.args[0] + result = force_unicode(result, 'UTF-8') + return xml_safe(result) + +class Tee(object): + def __init__(self, encoding, *args): + self._encoding = encoding + self._streams = args + + def write(self, data): + data = force_unicode(data, self._encoding) + for s in self._streams: + s.write(data) + + def writelines(self, lines): + for line in lines: + self.write(line) + + def flush(self): + for s in self._streams: + s.flush() + + def isatty(self): + return False + + +class Xunit(Plugin): + """This plugin provides test results in the standard XUnit XML format.""" + name = 'xunit' + score = 1500 + encoding = 'UTF-8' + error_report_file = None + + def __init__(self): + super(Xunit, self).__init__() + self._capture_stack = [] + self._currentStdout = None + self._currentStderr = None + + def _timeTaken(self): + if hasattr(self, '_timer'): + taken = time() - self._timer + else: + # test died before it ran (probably error in setup()) + # or success/failure added before test started probably + # due to custom TestResult munging + taken = 0.0 + return taken + + def _quoteattr(self, attr): + """Escape an XML attribute. Value can be unicode.""" + attr = xml_safe(attr) + return saxutils.quoteattr(attr) + + def options(self, parser, env): + """Sets additional command line options.""" + Plugin.options(self, parser, env) + parser.add_option( + '--xunit-file', action='store', + dest='xunit_file', metavar="FILE", + default=env.get('NOSE_XUNIT_FILE', 'nosetests.xml'), + help=("Path to xml file to store the xunit report in. " + "Default is nosetests.xml in the working directory " + "[NOSE_XUNIT_FILE]")) + + parser.add_option( + '--xunit-testsuite-name', action='store', + dest='xunit_testsuite_name', metavar="PACKAGE", + default=env.get('NOSE_XUNIT_TESTSUITE_NAME', 'nosetests'), + help=("Name of the testsuite in the xunit xml, generated by plugin. " + "Default test suite name is nosetests.")) + + def configure(self, options, config): + """Configures the xunit plugin.""" + Plugin.configure(self, options, config) + self.config = config + if self.enabled: + self.stats = {'errors': 0, + 'failures': 0, + 'passes': 0, + 'skipped': 0 + } + self.errorlist = [] + self.error_report_file_name = os.path.realpath(options.xunit_file) + self.xunit_testsuite_name = options.xunit_testsuite_name + + def report(self, stream): + """Writes an Xunit-formatted XML file + + The file includes a report of test errors and failures. + + """ + self.error_report_file = codecs.open(self.error_report_file_name, 'w', + self.encoding, 'replace') + self.stats['encoding'] = self.encoding + self.stats['testsuite_name'] = self.xunit_testsuite_name + self.stats['total'] = (self.stats['errors'] + self.stats['failures'] + + self.stats['passes'] + self.stats['skipped']) + self.error_report_file.write( + u'' + u'' % self.stats) + self.error_report_file.write(u''.join([force_unicode(e, self.encoding) + for e in self.errorlist])) + self.error_report_file.write(u'') + self.error_report_file.close() + if self.config.verbosity > 1: + stream.writeln("-" * 70) + stream.writeln("XML: %s" % self.error_report_file.name) + + def _startCapture(self): + self._capture_stack.append((sys.stdout, sys.stderr)) + self._currentStdout = StringIO() + self._currentStderr = StringIO() + sys.stdout = Tee(self.encoding, self._currentStdout, sys.stdout) + sys.stderr = Tee(self.encoding, self._currentStderr, sys.stderr) + + def startContext(self, context): + self._startCapture() + + def stopContext(self, context): + self._endCapture() + + def beforeTest(self, test): + """Initializes a timer before starting a test.""" + self._timer = time() + self._startCapture() + + def _endCapture(self): + if self._capture_stack: + sys.stdout, sys.stderr = self._capture_stack.pop() + + def afterTest(self, test): + self._endCapture() + self._currentStdout = None + self._currentStderr = None + + def finalize(self, test): + while self._capture_stack: + self._endCapture() + + def _getCapturedStdout(self): + if self._currentStdout: + value = self._currentStdout.getvalue() + if value: + return '' % escape_cdata( + value) + return '' + + def _getCapturedStderr(self): + if self._currentStderr: + value = self._currentStderr.getvalue() + if value: + return '' % escape_cdata( + value) + return '' + + def addError(self, test, err, capt=None): + """Add error output to Xunit report. + """ + taken = self._timeTaken() + + if issubclass(err[0], SkipTest): + type = 'skipped' + self.stats['skipped'] += 1 + else: + type = 'error' + self.stats['errors'] += 1 + + tb = format_exception(err, self.encoding) + id = test.id() + + self.errorlist.append( + u'' + u'<%(type)s type=%(errtype)s message=%(message)s>' + u'%(systemout)s%(systemerr)s' % + {'cls': self._quoteattr(id_split(id)[0]), + 'name': self._quoteattr(id_split(id)[-1]), + 'taken': taken, + 'type': type, + 'errtype': self._quoteattr(nice_classname(err[0])), + 'message': self._quoteattr(exc_message(err)), + 'tb': escape_cdata(tb), + 'systemout': self._getCapturedStdout(), + 'systemerr': self._getCapturedStderr(), + }) + + def addFailure(self, test, err, capt=None, tb_info=None): + """Add failure output to Xunit report. + """ + taken = self._timeTaken() + tb = format_exception(err, self.encoding) + self.stats['failures'] += 1 + id = test.id() + + self.errorlist.append( + u'' + u'' + u'%(systemout)s%(systemerr)s' % + {'cls': self._quoteattr(id_split(id)[0]), + 'name': self._quoteattr(id_split(id)[-1]), + 'taken': taken, + 'errtype': self._quoteattr(nice_classname(err[0])), + 'message': self._quoteattr(exc_message(err)), + 'tb': escape_cdata(tb), + 'systemout': self._getCapturedStdout(), + 'systemerr': self._getCapturedStderr(), + }) + + def addSuccess(self, test, capt=None): + """Add success output to Xunit report. + """ + taken = self._timeTaken() + self.stats['passes'] += 1 + id = test.id() + self.errorlist.append( + '%(systemout)s%(systemerr)s' % + {'cls': self._quoteattr(id_split(id)[0]), + 'name': self._quoteattr(id_split(id)[-1]), + 'taken': taken, + 'systemout': self._getCapturedStdout(), + 'systemerr': self._getCapturedStderr(), + }) diff --git a/lib/spack/external/nose/proxy.py b/lib/spack/external/nose/proxy.py new file mode 100644 index 00000000000..c2676cb1952 --- /dev/null +++ b/lib/spack/external/nose/proxy.py @@ -0,0 +1,188 @@ +""" +Result Proxy +------------ + +The result proxy wraps the result instance given to each test. It +performs two functions: enabling extended error/failure reporting +and calling plugins. + +As each result event is fired, plugins are called with the same event; +however, plugins are called with the nose.case.Test instance that +wraps the actual test. So when a test fails and calls +result.addFailure(self, err), the result proxy calls +addFailure(self.test, err) for each plugin. This allows plugins to +have a single stable interface for all test types, and also to +manipulate the test object itself by setting the `test` attribute of +the nose.case.Test that they receive. +""" +import logging +from nose.config import Config + + +log = logging.getLogger(__name__) + + +def proxied_attribute(local_attr, proxied_attr, doc): + """Create a property that proxies attribute ``proxied_attr`` through + the local attribute ``local_attr``. + """ + def fget(self): + return getattr(getattr(self, local_attr), proxied_attr) + def fset(self, value): + setattr(getattr(self, local_attr), proxied_attr, value) + def fdel(self): + delattr(getattr(self, local_attr), proxied_attr) + return property(fget, fset, fdel, doc) + + +class ResultProxyFactory(object): + """Factory for result proxies. Generates a ResultProxy bound to each test + and the result passed to the test. + """ + def __init__(self, config=None): + if config is None: + config = Config() + self.config = config + self.__prepared = False + self.__result = None + + def __call__(self, result, test): + """Return a ResultProxy for the current test. + + On first call, plugins are given a chance to replace the + result used for the remaining tests. If a plugin returns a + value from prepareTestResult, that object will be used as the + result for all tests. + """ + if not self.__prepared: + self.__prepared = True + plug_result = self.config.plugins.prepareTestResult(result) + if plug_result is not None: + self.__result = result = plug_result + if self.__result is not None: + result = self.__result + return ResultProxy(result, test, config=self.config) + + +class ResultProxy(object): + """Proxy to TestResults (or other results handler). + + One ResultProxy is created for each nose.case.Test. The result + proxy calls plugins with the nose.case.Test instance (instead of + the wrapped test case) as each result call is made. Finally, the + real result method is called, also with the nose.case.Test + instance as the test parameter. + + """ + def __init__(self, result, test, config=None): + if config is None: + config = Config() + self.config = config + self.plugins = config.plugins + self.result = result + self.test = test + + def __repr__(self): + return repr(self.result) + + def _prepareErr(self, err): + if not isinstance(err[1], Exception) and isinstance(err[0], type): + # Turn value back into an Exception (required in Python 3.x). + # Plugins do all sorts of crazy things with exception values. + # Convert it to a custom subclass of Exception with the same + # name as the actual exception to make it print correctly. + value = type(err[0].__name__, (Exception,), {})(err[1]) + err = (err[0], value, err[2]) + return err + + def assertMyTest(self, test): + # The test I was called with must be my .test or my + # .test's .test. or my .test.test's .case + + case = getattr(self.test, 'test', None) + assert (test is self.test + or test is case + or test is getattr(case, '_nose_case', None)), ( + "ResultProxy for %r (%s) was called with test %r (%s)" + % (self.test, id(self.test), test, id(test))) + + def afterTest(self, test): + self.assertMyTest(test) + self.plugins.afterTest(self.test) + if hasattr(self.result, "afterTest"): + self.result.afterTest(self.test) + + def beforeTest(self, test): + self.assertMyTest(test) + self.plugins.beforeTest(self.test) + if hasattr(self.result, "beforeTest"): + self.result.beforeTest(self.test) + + def addError(self, test, err): + self.assertMyTest(test) + plugins = self.plugins + plugin_handled = plugins.handleError(self.test, err) + if plugin_handled: + return + # test.passed is set in result, to account for error classes + formatted = plugins.formatError(self.test, err) + if formatted is not None: + err = formatted + plugins.addError(self.test, err) + self.result.addError(self.test, self._prepareErr(err)) + if not self.result.wasSuccessful() and self.config.stopOnError: + self.shouldStop = True + + def addFailure(self, test, err): + self.assertMyTest(test) + plugins = self.plugins + plugin_handled = plugins.handleFailure(self.test, err) + if plugin_handled: + return + self.test.passed = False + formatted = plugins.formatFailure(self.test, err) + if formatted is not None: + err = formatted + plugins.addFailure(self.test, err) + self.result.addFailure(self.test, self._prepareErr(err)) + if self.config.stopOnError: + self.shouldStop = True + + def addSkip(self, test, reason): + # 2.7 compat shim + from nose.plugins.skip import SkipTest + self.assertMyTest(test) + plugins = self.plugins + if not isinstance(reason, Exception): + # for Python 3.2+ + reason = Exception(reason) + plugins.addError(self.test, (SkipTest, reason, None)) + self.result.addSkip(self.test, reason) + + def addSuccess(self, test): + self.assertMyTest(test) + self.plugins.addSuccess(self.test) + self.result.addSuccess(self.test) + + def startTest(self, test): + self.assertMyTest(test) + self.plugins.startTest(self.test) + self.result.startTest(self.test) + + def stop(self): + self.result.stop() + + def stopTest(self, test): + self.assertMyTest(test) + self.plugins.stopTest(self.test) + self.result.stopTest(self.test) + + # proxied attributes + shouldStop = proxied_attribute('result', 'shouldStop', + """Should the test run stop?""") + errors = proxied_attribute('result', 'errors', + """Tests that raised an exception""") + failures = proxied_attribute('result', 'failures', + """Tests that failed""") + testsRun = proxied_attribute('result', 'testsRun', + """Number of tests run""") diff --git a/lib/spack/external/nose/pyversion.py b/lib/spack/external/nose/pyversion.py new file mode 100644 index 00000000000..091238da756 --- /dev/null +++ b/lib/spack/external/nose/pyversion.py @@ -0,0 +1,215 @@ +""" +This module contains fixups for using nose under different versions of Python. +""" +import sys +import os +import traceback +import types +import inspect +import nose.util + +__all__ = ['make_instancemethod', 'cmp_to_key', 'sort_list', 'ClassType', + 'TypeType', 'UNICODE_STRINGS', 'unbound_method', 'ismethod', + 'bytes_', 'is_base_exception', 'force_unicode', 'exc_to_unicode', + 'format_exception'] + +# In Python 3.x, all strings are unicode (the call to 'unicode()' in the 2.x +# source will be replaced with 'str()' when running 2to3, so this test will +# then become true) +UNICODE_STRINGS = (type(unicode()) == type(str())) + +if sys.version_info[:2] < (3, 0): + def force_unicode(s, encoding='UTF-8'): + try: + s = unicode(s) + except UnicodeDecodeError: + s = str(s).decode(encoding, 'replace') + + return s +else: + def force_unicode(s, encoding='UTF-8'): + return str(s) + +# new.instancemethod() is obsolete for new-style classes (Python 3.x) +# We need to use descriptor methods instead. +try: + import new + def make_instancemethod(function, instance): + return new.instancemethod(function.im_func, instance, + instance.__class__) +except ImportError: + def make_instancemethod(function, instance): + return function.__get__(instance, instance.__class__) + +# To be forward-compatible, we do all list sorts using keys instead of cmp +# functions. However, part of the unittest.TestLoader API involves a +# user-provideable cmp function, so we need some way to convert that. +def cmp_to_key(mycmp): + 'Convert a cmp= function into a key= function' + class Key(object): + def __init__(self, obj): + self.obj = obj + def __lt__(self, other): + return mycmp(self.obj, other.obj) < 0 + def __gt__(self, other): + return mycmp(self.obj, other.obj) > 0 + def __eq__(self, other): + return mycmp(self.obj, other.obj) == 0 + return Key + +# Python 2.3 also does not support list-sorting by key, so we need to convert +# keys to cmp functions if we're running on old Python.. +if sys.version_info < (2, 4): + def sort_list(l, key, reverse=False): + if reverse: + return l.sort(lambda a, b: cmp(key(b), key(a))) + else: + return l.sort(lambda a, b: cmp(key(a), key(b))) +else: + def sort_list(l, key, reverse=False): + return l.sort(key=key, reverse=reverse) + +# In Python 3.x, all objects are "new style" objects descended from 'type', and +# thus types.ClassType and types.TypeType don't exist anymore. For +# compatibility, we make sure they still work. +if hasattr(types, 'ClassType'): + ClassType = types.ClassType + TypeType = types.TypeType +else: + ClassType = type + TypeType = type + +# The following emulates the behavior (we need) of an 'unbound method' under +# Python 3.x (namely, the ability to have a class associated with a function +# definition so that things can do stuff based on its associated class) +class UnboundMethod: + def __init__(self, cls, func): + # Make sure we have all the same attributes as the original function, + # so that the AttributeSelector plugin will work correctly... + self.__dict__ = func.__dict__.copy() + self._func = func + self.__self__ = UnboundSelf(cls) + if sys.version_info < (3, 0): + self.im_class = cls + self.__doc__ = getattr(func, '__doc__', None) + + def address(self): + cls = self.__self__.cls + modname = cls.__module__ + module = sys.modules[modname] + filename = getattr(module, '__file__', None) + if filename is not None: + filename = os.path.abspath(filename) + return (nose.util.src(filename), modname, "%s.%s" % (cls.__name__, + self._func.__name__)) + + def __call__(self, *args, **kwargs): + return self._func(*args, **kwargs) + + def __getattr__(self, attr): + return getattr(self._func, attr) + + def __repr__(self): + return '' % (self.__self__.cls.__name__, + self._func.__name__) + +class UnboundSelf: + def __init__(self, cls): + self.cls = cls + + # We have to do this hackery because Python won't let us override the + # __class__ attribute... + def __getattribute__(self, attr): + if attr == '__class__': + return self.cls + else: + return object.__getattribute__(self, attr) + +def unbound_method(cls, func): + if inspect.ismethod(func): + return func + if not inspect.isfunction(func): + raise TypeError('%s is not a function' % (repr(func),)) + return UnboundMethod(cls, func) + +def ismethod(obj): + return inspect.ismethod(obj) or isinstance(obj, UnboundMethod) + + +# Make a pseudo-bytes function that can be called without the encoding arg: +if sys.version_info >= (3, 0): + def bytes_(s, encoding='utf8'): + if isinstance(s, bytes): + return s + return bytes(s, encoding) +else: + def bytes_(s, encoding=None): + return str(s) + + +if sys.version_info[:2] >= (2, 6): + def isgenerator(o): + if isinstance(o, UnboundMethod): + o = o._func + return inspect.isgeneratorfunction(o) or inspect.isgenerator(o) +else: + try: + from compiler.consts import CO_GENERATOR + except ImportError: + # IronPython doesn't have a complier module + CO_GENERATOR=0x20 + + def isgenerator(func): + try: + return func.func_code.co_flags & CO_GENERATOR != 0 + except AttributeError: + return False + +# Make a function to help check if an exception is derived from BaseException. +# In Python 2.4, we just use Exception instead. +if sys.version_info[:2] < (2, 5): + def is_base_exception(exc): + return isinstance(exc, Exception) +else: + def is_base_exception(exc): + return isinstance(exc, BaseException) + +if sys.version_info[:2] < (3, 0): + def exc_to_unicode(ev, encoding='utf-8'): + if is_base_exception(ev): + if not hasattr(ev, '__unicode__'): + # 2.5- + if not hasattr(ev, 'message'): + # 2.4 + msg = len(ev.args) and ev.args[0] or '' + else: + msg = ev.message + msg = force_unicode(msg, encoding=encoding) + clsname = force_unicode(ev.__class__.__name__, + encoding=encoding) + ev = u'%s: %s' % (clsname, msg) + elif not isinstance(ev, unicode): + ev = repr(ev) + + return force_unicode(ev, encoding=encoding) +else: + def exc_to_unicode(ev, encoding='utf-8'): + return str(ev) + +def format_exception(exc_info, encoding='UTF-8'): + ec, ev, tb = exc_info + + # Our exception object may have been turned into a string, and Python 3's + # traceback.format_exception() doesn't take kindly to that (it expects an + # actual exception object). So we work around it, by doing the work + # ourselves if ev is not an exception object. + if not is_base_exception(ev): + tb_data = force_unicode( + ''.join(traceback.format_tb(tb)), + encoding) + ev = exc_to_unicode(ev) + return tb_data + ev + else: + return force_unicode( + ''.join(traceback.format_exception(*exc_info)), + encoding) diff --git a/lib/spack/external/nose/result.py b/lib/spack/external/nose/result.py new file mode 100644 index 00000000000..f974a14ae24 --- /dev/null +++ b/lib/spack/external/nose/result.py @@ -0,0 +1,200 @@ +""" +Test Result +----------- + +Provides a TextTestResult that extends unittest's _TextTestResult to +provide support for error classes (such as the builtin skip and +deprecated classes), and hooks for plugins to take over or extend +reporting. +""" + +import logging +try: + # 2.7+ + from unittest.runner import _TextTestResult +except ImportError: + from unittest import _TextTestResult +from nose.config import Config +from nose.util import isclass, ln as _ln # backwards compat + +log = logging.getLogger('nose.result') + + +def _exception_detail(exc): + # this is what stdlib module traceback does + try: + return str(exc) + except: + return '' % type(exc).__name__ + + +class TextTestResult(_TextTestResult): + """Text test result that extends unittest's default test result + support for a configurable set of errorClasses (eg, Skip, + Deprecated, TODO) that extend the errors/failures/success triad. + """ + def __init__(self, stream, descriptions, verbosity, config=None, + errorClasses=None): + if errorClasses is None: + errorClasses = {} + self.errorClasses = errorClasses + if config is None: + config = Config() + self.config = config + _TextTestResult.__init__(self, stream, descriptions, verbosity) + + def addSkip(self, test, reason): + # 2.7 skip compat + from nose.plugins.skip import SkipTest + if SkipTest in self.errorClasses: + storage, label, isfail = self.errorClasses[SkipTest] + storage.append((test, reason)) + self.printLabel(label, (SkipTest, reason, None)) + + def addError(self, test, err): + """Overrides normal addError to add support for + errorClasses. If the exception is a registered class, the + error will be added to the list for that class, not errors. + """ + ec, ev, tb = err + try: + exc_info = self._exc_info_to_string(err, test) + except TypeError: + # 2.3 compat + exc_info = self._exc_info_to_string(err) + for cls, (storage, label, isfail) in self.errorClasses.items(): + #if 'Skip' in cls.__name__ or 'Skip' in ec.__name__: + # from nose.tools import set_trace + # set_trace() + if isclass(ec) and issubclass(ec, cls): + if isfail: + test.passed = False + storage.append((test, exc_info)) + self.printLabel(label, err) + return + self.errors.append((test, exc_info)) + test.passed = False + self.printLabel('ERROR') + + # override to bypass changes in 2.7 + def getDescription(self, test): + if self.descriptions: + return test.shortDescription() or str(test) + else: + return str(test) + + def printLabel(self, label, err=None): + # Might get patched into a streamless result + stream = getattr(self, 'stream', None) + if stream is not None: + if self.showAll: + message = [label] + if err: + detail = _exception_detail(err[1]) + if detail: + message.append(detail) + stream.writeln(": ".join(message)) + elif self.dots: + stream.write(label[:1]) + + def printErrors(self): + """Overrides to print all errorClasses errors as well. + """ + _TextTestResult.printErrors(self) + for cls in self.errorClasses.keys(): + storage, label, isfail = self.errorClasses[cls] + if isfail: + self.printErrorList(label, storage) + # Might get patched into a result with no config + if hasattr(self, 'config'): + self.config.plugins.report(self.stream) + + def printSummary(self, start, stop): + """Called by the test runner to print the final summary of test + run results. + """ + write = self.stream.write + writeln = self.stream.writeln + taken = float(stop - start) + run = self.testsRun + plural = run != 1 and "s" or "" + + writeln(self.separator2) + writeln("Ran %s test%s in %.3fs" % (run, plural, taken)) + writeln() + + summary = {} + eckeys = self.errorClasses.keys() + for cls in eckeys: + storage, label, isfail = self.errorClasses[cls] + count = len(storage) + if not count: + continue + summary[label] = count + if len(self.failures): + summary['failures'] = len(self.failures) + if len(self.errors): + summary['errors'] = len(self.errors) + + if not self.wasSuccessful(): + write("FAILED") + else: + write("OK") + items = summary.items() + if items: + items.sort() + write(" (") + write(", ".join(["%s=%s" % (label, count) for + label, count in items])) + writeln(")") + else: + writeln() + + def wasSuccessful(self): + """Overrides to check that there are no errors in errorClasses + lists that are marked as errors and should cause a run to + fail. + """ + if self.errors or self.failures: + return False + for cls in self.errorClasses.keys(): + storage, label, isfail = self.errorClasses[cls] + if not isfail: + continue + if storage: + return False + return True + + def _addError(self, test, err): + try: + exc_info = self._exc_info_to_string(err, test) + except TypeError: + # 2.3: does not take test arg + exc_info = self._exc_info_to_string(err) + self.errors.append((test, exc_info)) + if self.showAll: + self.stream.write('ERROR') + elif self.dots: + self.stream.write('E') + + def _exc_info_to_string(self, err, test=None): + # 2.7 skip compat + from nose.plugins.skip import SkipTest + if isclass(err[0]) and issubclass(err[0], SkipTest): + return str(err[1]) + # 2.3/2.4 -- 2.4 passes test, 2.3 does not + try: + return _TextTestResult._exc_info_to_string(self, err, test) + except TypeError: + # 2.3: does not take test arg + return _TextTestResult._exc_info_to_string(self, err) + + +def ln(*arg, **kw): + from warnings import warn + warn("ln() has moved to nose.util from nose.result and will be removed " + "from nose.result in a future release. Please update your imports ", + DeprecationWarning) + return _ln(*arg, **kw) + + diff --git a/lib/spack/external/nose/selector.py b/lib/spack/external/nose/selector.py new file mode 100644 index 00000000000..b63f7af0b15 --- /dev/null +++ b/lib/spack/external/nose/selector.py @@ -0,0 +1,251 @@ +""" +Test Selection +-------------- + +Test selection is handled by a Selector. The test loader calls the +appropriate selector method for each object it encounters that it +thinks may be a test. +""" +import logging +import os +import unittest +from nose.config import Config +from nose.util import split_test_name, src, getfilename, getpackage, ispackage, is_executable + +log = logging.getLogger(__name__) + +__all__ = ['Selector', 'defaultSelector', 'TestAddress'] + + +# for efficiency and easier mocking +op_join = os.path.join +op_basename = os.path.basename +op_exists = os.path.exists +op_splitext = os.path.splitext +op_isabs = os.path.isabs +op_abspath = os.path.abspath + + +class Selector(object): + """Core test selector. Examines test candidates and determines whether, + given the specified configuration, the test candidate should be selected + as a test. + """ + def __init__(self, config): + if config is None: + config = Config() + self.configure(config) + + def configure(self, config): + self.config = config + self.exclude = config.exclude + self.ignoreFiles = config.ignoreFiles + self.include = config.include + self.plugins = config.plugins + self.match = config.testMatch + + def matches(self, name): + """Does the name match my requirements? + + To match, a name must match config.testMatch OR config.include + and it must not match config.exclude + """ + return ((self.match.search(name) + or (self.include and + filter(None, + [inc.search(name) for inc in self.include]))) + and ((not self.exclude) + or not filter(None, + [exc.search(name) for exc in self.exclude]) + )) + + def wantClass(self, cls): + """Is the class a wanted test class? + + A class must be a unittest.TestCase subclass, or match test name + requirements. Classes that start with _ are always excluded. + """ + declared = getattr(cls, '__test__', None) + if declared is not None: + wanted = declared + else: + wanted = (not cls.__name__.startswith('_') + and (issubclass(cls, unittest.TestCase) + or self.matches(cls.__name__))) + + plug_wants = self.plugins.wantClass(cls) + if plug_wants is not None: + log.debug("Plugin setting selection of %s to %s", cls, plug_wants) + wanted = plug_wants + log.debug("wantClass %s? %s", cls, wanted) + return wanted + + def wantDirectory(self, dirname): + """Is the directory a wanted test directory? + + All package directories match, so long as they do not match exclude. + All other directories must match test requirements. + """ + tail = op_basename(dirname) + if ispackage(dirname): + wanted = (not self.exclude + or not filter(None, + [exc.search(tail) for exc in self.exclude] + )) + else: + wanted = (self.matches(tail) + or (self.config.srcDirs + and tail in self.config.srcDirs)) + plug_wants = self.plugins.wantDirectory(dirname) + if plug_wants is not None: + log.debug("Plugin setting selection of %s to %s", + dirname, plug_wants) + wanted = plug_wants + log.debug("wantDirectory %s? %s", dirname, wanted) + return wanted + + def wantFile(self, file): + """Is the file a wanted test file? + + The file must be a python source file and match testMatch or + include, and not match exclude. Files that match ignore are *never* + wanted, regardless of plugin, testMatch, include or exclude settings. + """ + # never, ever load files that match anything in ignore + # (.* _* and *setup*.py by default) + base = op_basename(file) + ignore_matches = [ ignore_this for ignore_this in self.ignoreFiles + if ignore_this.search(base) ] + if ignore_matches: + log.debug('%s matches ignoreFiles pattern; skipped', + base) + return False + if not self.config.includeExe and is_executable(file): + log.info('%s is executable; skipped', file) + return False + dummy, ext = op_splitext(base) + pysrc = ext == '.py' + + wanted = pysrc and self.matches(base) + plug_wants = self.plugins.wantFile(file) + if plug_wants is not None: + log.debug("plugin setting want %s to %s", file, plug_wants) + wanted = plug_wants + log.debug("wantFile %s? %s", file, wanted) + return wanted + + def wantFunction(self, function): + """Is the function a test function? + """ + try: + if hasattr(function, 'compat_func_name'): + funcname = function.compat_func_name + else: + funcname = function.__name__ + except AttributeError: + # not a function + return False + declared = getattr(function, '__test__', None) + if declared is not None: + wanted = declared + else: + wanted = not funcname.startswith('_') and self.matches(funcname) + plug_wants = self.plugins.wantFunction(function) + if plug_wants is not None: + wanted = plug_wants + log.debug("wantFunction %s? %s", function, wanted) + return wanted + + def wantMethod(self, method): + """Is the method a test method? + """ + try: + method_name = method.__name__ + except AttributeError: + # not a method + return False + if method_name.startswith('_'): + # never collect 'private' methods + return False + declared = getattr(method, '__test__', None) + if declared is not None: + wanted = declared + else: + wanted = self.matches(method_name) + plug_wants = self.plugins.wantMethod(method) + if plug_wants is not None: + wanted = plug_wants + log.debug("wantMethod %s? %s", method, wanted) + return wanted + + def wantModule(self, module): + """Is the module a test module? + + The tail of the module name must match test requirements. One exception: + we always want __main__. + """ + declared = getattr(module, '__test__', None) + if declared is not None: + wanted = declared + else: + wanted = self.matches(module.__name__.split('.')[-1]) \ + or module.__name__ == '__main__' + plug_wants = self.plugins.wantModule(module) + if plug_wants is not None: + wanted = plug_wants + log.debug("wantModule %s? %s", module, wanted) + return wanted + +defaultSelector = Selector + + +class TestAddress(object): + """A test address represents a user's request to run a particular + test. The user may specify a filename or module (or neither), + and/or a callable (a class, function, or method). The naming + format for test addresses is: + + filename_or_module:callable + + Filenames that are not absolute will be made absolute relative to + the working dir. + + The filename or module part will be considered a module name if it + doesn't look like a file, that is, if it doesn't exist on the file + system and it doesn't contain any directory separators and it + doesn't end in .py. + + Callables may be a class name, function name, method name, or + class.method specification. + """ + def __init__(self, name, workingDir=None): + if workingDir is None: + workingDir = os.getcwd() + self.name = name + self.workingDir = workingDir + self.filename, self.module, self.call = split_test_name(name) + log.debug('Test name %s resolved to file %s, module %s, call %s', + name, self.filename, self.module, self.call) + if self.filename is None: + if self.module is not None: + self.filename = getfilename(self.module, self.workingDir) + if self.filename: + self.filename = src(self.filename) + if not op_isabs(self.filename): + self.filename = op_abspath(op_join(workingDir, + self.filename)) + if self.module is None: + self.module = getpackage(self.filename) + log.debug( + 'Final resolution of test name %s: file %s module %s call %s', + name, self.filename, self.module, self.call) + + def totuple(self): + return (self.filename, self.module, self.call) + + def __str__(self): + return self.name + + def __repr__(self): + return "%s: (%s, %s, %s)" % (self.name, self.filename, + self.module, self.call) diff --git a/lib/spack/external/nose/sphinx/__init__.py b/lib/spack/external/nose/sphinx/__init__.py new file mode 100644 index 00000000000..2ae28399f5f --- /dev/null +++ b/lib/spack/external/nose/sphinx/__init__.py @@ -0,0 +1 @@ +pass diff --git a/lib/spack/external/nose/sphinx/pluginopts.py b/lib/spack/external/nose/sphinx/pluginopts.py new file mode 100644 index 00000000000..d2b284ab276 --- /dev/null +++ b/lib/spack/external/nose/sphinx/pluginopts.py @@ -0,0 +1,189 @@ +""" +Adds a sphinx directive that can be used to automatically document a plugin. + +this:: + + .. autoplugin :: nose.plugins.foo + :plugin: Pluggy + +produces:: + + .. automodule :: nose.plugins.foo + + Options + ------- + + .. cmdoption :: --foo=BAR, --fooble=BAR + + Do the foo thing to the new thing. + + Plugin + ------ + + .. autoclass :: nose.plugins.foo.Pluggy + :members: + + Source + ------ + + .. include :: path/to/nose/plugins/foo.py + :literal: + +""" +import os +try: + from docutils import nodes, utils + from docutils.statemachine import ViewList + from docutils.parsers.rst import directives +except ImportError: + pass # won't run anyway + +from nose.util import resolve_name +from nose.plugins.base import Plugin +from nose.plugins.manager import BuiltinPluginManager +from nose.config import Config +from nose.core import TestProgram +from inspect import isclass + + +def autoplugin_directive(dirname, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + mod_name = arguments[0] + mod = resolve_name(mod_name) + plug_name = options.get('plugin', None) + if plug_name: + obj = getattr(mod, plug_name) + else: + for entry in dir(mod): + obj = getattr(mod, entry) + if isclass(obj) and issubclass(obj, Plugin) and obj is not Plugin: + plug_name = '%s.%s' % (mod_name, entry) + break + + # mod docstring + rst = ViewList() + rst.append('.. automodule :: %s\n' % mod_name, '') + rst.append('', '') + + # options + rst.append('Options', '') + rst.append('-------', '') + rst.append('', '') + + plug = obj() + opts = OptBucket() + plug.options(opts, {}) + for opt in opts: + rst.append(opt.options(), '') + rst.append(' \n', '') + rst.append(' ' + opt.help + '\n', '') + rst.append('\n', '') + + # plugin class + rst.append('Plugin', '') + rst.append('------', '') + rst.append('', '') + + rst.append('.. autoclass :: %s\n' % plug_name, '') + rst.append(' :members:\n', '') + rst.append(' :show-inheritance:\n', '') + rst.append('', '') + + # source + rst.append('Source', '') + rst.append('------', '') + rst.append( + '.. include :: %s\n' % utils.relative_path( + state_machine.document['source'], + os.path.abspath(mod.__file__.replace('.pyc', '.py'))), + '') + rst.append(' :literal:\n', '') + rst.append('', '') + + node = nodes.section() + node.document = state.document + surrounding_title_styles = state.memo.title_styles + surrounding_section_level = state.memo.section_level + state.memo.title_styles = [] + state.memo.section_level = 0 + state.nested_parse(rst, 0, node, match_titles=1) + state.memo.title_styles = surrounding_title_styles + state.memo.section_level = surrounding_section_level + + return node.children + + +def autohelp_directive(dirname, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + """produces rst from nose help""" + config = Config(parserClass=OptBucket, + plugins=BuiltinPluginManager()) + parser = config.getParser(TestProgram.usage()) + rst = ViewList() + for line in parser.format_help().split('\n'): + rst.append(line, '') + + rst.append('Options', '') + rst.append('-------', '') + rst.append('', '') + for opt in parser: + rst.append(opt.options(), '') + rst.append(' \n', '') + rst.append(' ' + opt.help + '\n', '') + rst.append('\n', '') + node = nodes.section() + node.document = state.document + surrounding_title_styles = state.memo.title_styles + surrounding_section_level = state.memo.section_level + state.memo.title_styles = [] + state.memo.section_level = 0 + state.nested_parse(rst, 0, node, match_titles=1) + state.memo.title_styles = surrounding_title_styles + state.memo.section_level = surrounding_section_level + + return node.children + + +class OptBucket(object): + def __init__(self, doc=None, prog='nosetests'): + self.opts = [] + self.doc = doc + self.prog = prog + + def __iter__(self): + return iter(self.opts) + + def format_help(self): + return self.doc.replace('%prog', self.prog).replace(':\n', '::\n') + + def add_option(self, *arg, **kw): + self.opts.append(Opt(*arg, **kw)) + + +class Opt(object): + def __init__(self, *arg, **kw): + self.opts = arg + self.action = kw.pop('action', None) + self.default = kw.pop('default', None) + self.metavar = kw.pop('metavar', None) + self.help = kw.pop('help', None) + + def options(self): + buf = [] + for optstring in self.opts: + desc = optstring + if self.action not in ('store_true', 'store_false'): + desc += '=%s' % self.meta(optstring) + buf.append(desc) + return '.. cmdoption :: ' + ', '.join(buf) + + def meta(self, optstring): + # FIXME optparser default metavar? + return self.metavar or 'DEFAULT' + + +def setup(app): + app.add_directive('autoplugin', + autoplugin_directive, 1, (1, 0, 1), + plugin=directives.unchanged) + app.add_directive('autohelp', autohelp_directive, 0, (0, 0, 1)) diff --git a/lib/spack/external/nose/suite.py b/lib/spack/external/nose/suite.py new file mode 100644 index 00000000000..a831105e340 --- /dev/null +++ b/lib/spack/external/nose/suite.py @@ -0,0 +1,609 @@ +""" +Test Suites +----------- + +Provides a LazySuite, which is a suite whose test list is a generator +function, and ContextSuite,which can run fixtures (setup/teardown +functions or methods) for the context that contains its tests. + +""" +from __future__ import generators + +import logging +import sys +import unittest +from nose.case import Test +from nose.config import Config +from nose.proxy import ResultProxyFactory +from nose.util import isclass, resolve_name, try_run + +if sys.platform == 'cli': + if sys.version_info[:2] < (2, 6): + import clr + clr.AddReference("IronPython") + from IronPython.Runtime.Exceptions import StringException + else: + class StringException(Exception): + pass + +log = logging.getLogger(__name__) +#log.setLevel(logging.DEBUG) + +# Singleton for default value -- see ContextSuite.__init__ below +_def = object() + + +def _strclass(cls): + return "%s.%s" % (cls.__module__, cls.__name__) + +class MixedContextError(Exception): + """Error raised when a context suite sees tests from more than + one context. + """ + pass + + +class LazySuite(unittest.TestSuite): + """A suite that may use a generator as its list of tests + """ + def __init__(self, tests=()): + """Initialize the suite. tests may be an iterable or a generator + """ + super(LazySuite, self).__init__() + self._set_tests(tests) + + def __iter__(self): + return iter(self._tests) + + def __repr__(self): + return "<%s tests=generator (%s)>" % ( + _strclass(self.__class__), id(self)) + + def __hash__(self): + return object.__hash__(self) + + __str__ = __repr__ + + def addTest(self, test): + self._precache.append(test) + + # added to bypass run changes in 2.7's unittest + def run(self, result): + for test in self._tests: + if result.shouldStop: + break + test(result) + return result + + def __nonzero__(self): + log.debug("tests in %s?", id(self)) + if self._precache: + return True + if self.test_generator is None: + return False + try: + test = self.test_generator.next() + if test is not None: + self._precache.append(test) + return True + except StopIteration: + pass + return False + + def _get_tests(self): + log.debug("precache is %s", self._precache) + for test in self._precache: + yield test + if self.test_generator is None: + return + for test in self.test_generator: + yield test + + def _set_tests(self, tests): + self._precache = [] + is_suite = isinstance(tests, unittest.TestSuite) + if callable(tests) and not is_suite: + self.test_generator = tests() + elif is_suite: + # Suites need special treatment: they must be called like + # tests for their setup/teardown to run (if any) + self.addTests([tests]) + self.test_generator = None + else: + self.addTests(tests) + self.test_generator = None + + _tests = property(_get_tests, _set_tests, None, + "Access the tests in this suite. Access is through a " + "generator, so iteration may not be repeatable.") + + +class ContextSuite(LazySuite): + """A suite with context. + + A ContextSuite executes fixtures (setup and teardown functions or + methods) for the context containing its tests. + + The context may be explicitly passed. If it is not, a context (or + nested set of contexts) will be constructed by examining the tests + in the suite. + """ + failureException = unittest.TestCase.failureException + was_setup = False + was_torndown = False + classSetup = ('setup_class', 'setup_all', 'setupClass', 'setupAll', + 'setUpClass', 'setUpAll') + classTeardown = ('teardown_class', 'teardown_all', 'teardownClass', + 'teardownAll', 'tearDownClass', 'tearDownAll') + moduleSetup = ('setup_module', 'setupModule', 'setUpModule', 'setup', + 'setUp') + moduleTeardown = ('teardown_module', 'teardownModule', 'tearDownModule', + 'teardown', 'tearDown') + packageSetup = ('setup_package', 'setupPackage', 'setUpPackage') + packageTeardown = ('teardown_package', 'teardownPackage', + 'tearDownPackage') + + def __init__(self, tests=(), context=None, factory=None, + config=None, resultProxy=None, can_split=True): + log.debug("Context suite for %s (%s) (%s)", tests, context, id(self)) + self.context = context + self.factory = factory + if config is None: + config = Config() + self.config = config + self.resultProxy = resultProxy + self.has_run = False + self.can_split = can_split + self.error_context = None + super(ContextSuite, self).__init__(tests) + + def __repr__(self): + return "<%s context=%s>" % ( + _strclass(self.__class__), + getattr(self.context, '__name__', self.context)) + __str__ = __repr__ + + def id(self): + if self.error_context: + return '%s:%s' % (repr(self), self.error_context) + else: + return repr(self) + + def __hash__(self): + return object.__hash__(self) + + # 2.3 compat -- force 2.4 call sequence + def __call__(self, *arg, **kw): + return self.run(*arg, **kw) + + def exc_info(self): + """Hook for replacing error tuple output + """ + return sys.exc_info() + + def _exc_info(self): + """Bottleneck to fix up IronPython string exceptions + """ + e = self.exc_info() + if sys.platform == 'cli': + if isinstance(e[0], StringException): + # IronPython throws these StringExceptions, but + # traceback checks type(etype) == str. Make a real + # string here. + e = (str(e[0]), e[1], e[2]) + + return e + + def run(self, result): + """Run tests in suite inside of suite fixtures. + """ + # proxy the result for myself + log.debug("suite %s (%s) run called, tests: %s", id(self), self, self._tests) + #import pdb + #pdb.set_trace() + if self.resultProxy: + result, orig = self.resultProxy(result, self), result + else: + result, orig = result, result + try: + self.setUp() + except KeyboardInterrupt: + raise + except: + self.error_context = 'setup' + result.addError(self, self._exc_info()) + return + try: + for test in self._tests: + if result.shouldStop: + log.debug("stopping") + break + # each nose.case.Test will create its own result proxy + # so the cases need the original result, to avoid proxy + # chains + test(orig) + finally: + self.has_run = True + try: + self.tearDown() + except KeyboardInterrupt: + raise + except: + self.error_context = 'teardown' + result.addError(self, self._exc_info()) + + def hasFixtures(self, ctx_callback=None): + context = self.context + if context is None: + return False + if self.implementsAnyFixture(context, ctx_callback=ctx_callback): + return True + # My context doesn't have any, but its ancestors might + factory = self.factory + if factory: + ancestors = factory.context.get(self, []) + for ancestor in ancestors: + if self.implementsAnyFixture( + ancestor, ctx_callback=ctx_callback): + return True + return False + + def implementsAnyFixture(self, context, ctx_callback): + if isclass(context): + names = self.classSetup + self.classTeardown + else: + names = self.moduleSetup + self.moduleTeardown + if hasattr(context, '__path__'): + names += self.packageSetup + self.packageTeardown + # If my context has any fixture attribute, I have fixtures + fixt = False + for m in names: + if hasattr(context, m): + fixt = True + break + if ctx_callback is None: + return fixt + return ctx_callback(context, fixt) + + def setUp(self): + log.debug("suite %s setUp called, tests: %s", id(self), self._tests) + if not self: + # I have no tests + log.debug("suite %s has no tests", id(self)) + return + if self.was_setup: + log.debug("suite %s already set up", id(self)) + return + context = self.context + if context is None: + return + # before running my own context's setup, I need to + # ask the factory if my context's contexts' setups have been run + factory = self.factory + if factory: + # get a copy, since we'll be destroying it as we go + ancestors = factory.context.get(self, [])[:] + while ancestors: + ancestor = ancestors.pop() + log.debug("ancestor %s may need setup", ancestor) + if ancestor in factory.was_setup: + continue + log.debug("ancestor %s does need setup", ancestor) + self.setupContext(ancestor) + if not context in factory.was_setup: + self.setupContext(context) + else: + self.setupContext(context) + self.was_setup = True + log.debug("completed suite setup") + + def setupContext(self, context): + self.config.plugins.startContext(context) + log.debug("%s setup context %s", self, context) + if self.factory: + if context in self.factory.was_setup: + return + # note that I ran the setup for this context, so that I'll run + # the teardown in my teardown + self.factory.was_setup[context] = self + if isclass(context): + names = self.classSetup + else: + names = self.moduleSetup + if hasattr(context, '__path__'): + names = self.packageSetup + names + try_run(context, names) + + def shortDescription(self): + if self.context is None: + return "test suite" + return "test suite for %s" % self.context + + def tearDown(self): + log.debug('context teardown') + if not self.was_setup or self.was_torndown: + log.debug( + "No reason to teardown (was_setup? %s was_torndown? %s)" + % (self.was_setup, self.was_torndown)) + return + self.was_torndown = True + context = self.context + if context is None: + log.debug("No context to tear down") + return + + # for each ancestor... if the ancestor was setup + # and I did the setup, I can do teardown + factory = self.factory + if factory: + ancestors = factory.context.get(self, []) + [context] + for ancestor in ancestors: + log.debug('ancestor %s may need teardown', ancestor) + if not ancestor in factory.was_setup: + log.debug('ancestor %s was not setup', ancestor) + continue + if ancestor in factory.was_torndown: + log.debug('ancestor %s already torn down', ancestor) + continue + setup = factory.was_setup[ancestor] + log.debug("%s setup ancestor %s", setup, ancestor) + if setup is self: + self.teardownContext(ancestor) + else: + self.teardownContext(context) + + def teardownContext(self, context): + log.debug("%s teardown context %s", self, context) + if self.factory: + if context in self.factory.was_torndown: + return + self.factory.was_torndown[context] = self + if isclass(context): + names = self.classTeardown + else: + names = self.moduleTeardown + if hasattr(context, '__path__'): + names = self.packageTeardown + names + try_run(context, names) + self.config.plugins.stopContext(context) + + # FIXME the wrapping has to move to the factory? + def _get_wrapped_tests(self): + for test in self._get_tests(): + if isinstance(test, Test) or isinstance(test, unittest.TestSuite): + yield test + else: + yield Test(test, + config=self.config, + resultProxy=self.resultProxy) + + _tests = property(_get_wrapped_tests, LazySuite._set_tests, None, + "Access the tests in this suite. Tests are returned " + "inside of a context wrapper.") + + +class ContextSuiteFactory(object): + """Factory for ContextSuites. Called with a collection of tests, + the factory decides on a hierarchy of contexts by introspecting + the collection or the tests themselves to find the objects + containing the test objects. It always returns one suite, but that + suite may consist of a hierarchy of nested suites. + """ + suiteClass = ContextSuite + def __init__(self, config=None, suiteClass=None, resultProxy=_def): + if config is None: + config = Config() + self.config = config + if suiteClass is not None: + self.suiteClass = suiteClass + # Using a singleton to represent default instead of None allows + # passing resultProxy=None to turn proxying off. + if resultProxy is _def: + resultProxy = ResultProxyFactory(config=config) + self.resultProxy = resultProxy + self.suites = {} + self.context = {} + self.was_setup = {} + self.was_torndown = {} + + def __call__(self, tests, **kw): + """Return ``ContextSuite`` for tests. ``tests`` may either + be a callable (in which case the resulting ContextSuite will + have no parent context and be evaluated lazily) or an + iterable. In that case the tests will wrapped in + nose.case.Test, be examined and the context of each found and a + suite of suites returned, organized into a stack with the + outermost suites belonging to the outermost contexts. + """ + log.debug("Create suite for %s", tests) + context = kw.pop('context', getattr(tests, 'context', None)) + log.debug("tests %s context %s", tests, context) + if context is None: + tests = self.wrapTests(tests) + try: + context = self.findContext(tests) + except MixedContextError: + return self.makeSuite(self.mixedSuites(tests), None, **kw) + return self.makeSuite(tests, context, **kw) + + def ancestry(self, context): + """Return the ancestry of the context (that is, all of the + packages and modules containing the context), in order of + descent with the outermost ancestor last. + This method is a generator. + """ + log.debug("get ancestry %s", context) + if context is None: + return + # Methods include reference to module they are defined in, we + # don't want that, instead want the module the class is in now + # (classes are re-ancestored elsewhere). + if hasattr(context, 'im_class'): + context = context.im_class + elif hasattr(context, '__self__'): + context = context.__self__.__class__ + if hasattr(context, '__module__'): + ancestors = context.__module__.split('.') + elif hasattr(context, '__name__'): + ancestors = context.__name__.split('.')[:-1] + else: + raise TypeError("%s has no ancestors?" % context) + while ancestors: + log.debug(" %s ancestors %s", context, ancestors) + yield resolve_name('.'.join(ancestors)) + ancestors.pop() + + def findContext(self, tests): + if callable(tests) or isinstance(tests, unittest.TestSuite): + return None + context = None + for test in tests: + # Don't look at suites for contexts, only tests + ctx = getattr(test, 'context', None) + if ctx is None: + continue + if context is None: + context = ctx + elif context != ctx: + raise MixedContextError( + "Tests with different contexts in same suite! %s != %s" + % (context, ctx)) + return context + + def makeSuite(self, tests, context, **kw): + suite = self.suiteClass( + tests, context=context, config=self.config, factory=self, + resultProxy=self.resultProxy, **kw) + if context is not None: + self.suites.setdefault(context, []).append(suite) + self.context.setdefault(suite, []).append(context) + log.debug("suite %s has context %s", suite, + getattr(context, '__name__', None)) + for ancestor in self.ancestry(context): + self.suites.setdefault(ancestor, []).append(suite) + self.context[suite].append(ancestor) + log.debug("suite %s has ancestor %s", suite, ancestor.__name__) + return suite + + def mixedSuites(self, tests): + """The complex case where there are tests that don't all share + the same context. Groups tests into suites with common ancestors, + according to the following (essentially tail-recursive) procedure: + + Starting with the context of the first test, if it is not + None, look for tests in the remaining tests that share that + ancestor. If any are found, group into a suite with that + ancestor as the context, and replace the current suite with + that suite. Continue this process for each ancestor of the + first test, until all ancestors have been processed. At this + point if any tests remain, recurse with those tests as the + input, returning a list of the common suite (which may be the + suite or test we started with, if no common tests were found) + plus the results of recursion. + """ + if not tests: + return [] + head = tests.pop(0) + if not tests: + return [head] # short circuit when none are left to combine + suite = head # the common ancestry suite, so far + tail = tests[:] + context = getattr(head, 'context', None) + if context is not None: + ancestors = [context] + [a for a in self.ancestry(context)] + for ancestor in ancestors: + common = [suite] # tests with ancestor in common, so far + remain = [] # tests that remain to be processed + for test in tail: + found_common = False + test_ctx = getattr(test, 'context', None) + if test_ctx is None: + remain.append(test) + continue + if test_ctx is ancestor: + common.append(test) + continue + for test_ancestor in self.ancestry(test_ctx): + if test_ancestor is ancestor: + common.append(test) + found_common = True + break + if not found_common: + remain.append(test) + if common: + suite = self.makeSuite(common, ancestor) + tail = self.mixedSuites(remain) + return [suite] + tail + + def wrapTests(self, tests): + log.debug("wrap %s", tests) + if callable(tests) or isinstance(tests, unittest.TestSuite): + log.debug("I won't wrap") + return tests + wrapped = [] + for test in tests: + log.debug("wrapping %s", test) + if isinstance(test, Test) or isinstance(test, unittest.TestSuite): + wrapped.append(test) + elif isinstance(test, ContextList): + wrapped.append(self.makeSuite(test, context=test.context)) + else: + wrapped.append( + Test(test, config=self.config, resultProxy=self.resultProxy) + ) + return wrapped + + +class ContextList(object): + """Not quite a suite -- a group of tests in a context. This is used + to hint the ContextSuiteFactory about what context the tests + belong to, in cases where it may be ambiguous or missing. + """ + def __init__(self, tests, context=None): + self.tests = tests + self.context = context + + def __iter__(self): + return iter(self.tests) + + +class FinalizingSuiteWrapper(unittest.TestSuite): + """Wraps suite and calls final function after suite has + executed. Used to call final functions in cases (like running in + the standard test runner) where test running is not under nose's + control. + """ + def __init__(self, suite, finalize): + super(FinalizingSuiteWrapper, self).__init__() + self.suite = suite + self.finalize = finalize + + def __call__(self, *arg, **kw): + return self.run(*arg, **kw) + + # 2.7 compat + def __iter__(self): + return iter(self.suite) + + def run(self, *arg, **kw): + try: + return self.suite(*arg, **kw) + finally: + self.finalize(*arg, **kw) + + +# backwards compat -- sort of +class TestDir: + def __init__(*arg, **kw): + raise NotImplementedError( + "TestDir is not usable with nose 0.10. The class is present " + "in nose.suite for backwards compatibility purposes but it " + "may not be used.") + + +class TestModule: + def __init__(*arg, **kw): + raise NotImplementedError( + "TestModule is not usable with nose 0.10. The class is present " + "in nose.suite for backwards compatibility purposes but it " + "may not be used.") diff --git a/lib/spack/external/nose/tools/__init__.py b/lib/spack/external/nose/tools/__init__.py new file mode 100644 index 00000000000..74dab16a749 --- /dev/null +++ b/lib/spack/external/nose/tools/__init__.py @@ -0,0 +1,15 @@ +""" +Tools for testing +----------------- + +nose.tools provides a few convenience functions to make writing tests +easier. You don't have to use them; nothing in the rest of nose depends +on any of these methods. + +""" +from nose.tools.nontrivial import * +from nose.tools.nontrivial import __all__ as nontrivial_all +from nose.tools.trivial import * +from nose.tools.trivial import __all__ as trivial_all + +__all__ = trivial_all + nontrivial_all diff --git a/lib/spack/external/nose/tools/nontrivial.py b/lib/spack/external/nose/tools/nontrivial.py new file mode 100644 index 00000000000..283973245b4 --- /dev/null +++ b/lib/spack/external/nose/tools/nontrivial.py @@ -0,0 +1,151 @@ +"""Tools not exempt from being descended into in tracebacks""" + +import time + + +__all__ = ['make_decorator', 'raises', 'set_trace', 'timed', 'with_setup', + 'TimeExpired', 'istest', 'nottest'] + + +class TimeExpired(AssertionError): + pass + + +def make_decorator(func): + """ + Wraps a test decorator so as to properly replicate metadata + of the decorated function, including nose's additional stuff + (namely, setup and teardown). + """ + def decorate(newfunc): + if hasattr(func, 'compat_func_name'): + name = func.compat_func_name + else: + name = func.__name__ + newfunc.__dict__ = func.__dict__ + newfunc.__doc__ = func.__doc__ + newfunc.__module__ = func.__module__ + if not hasattr(newfunc, 'compat_co_firstlineno'): + newfunc.compat_co_firstlineno = func.func_code.co_firstlineno + try: + newfunc.__name__ = name + except TypeError: + # can't set func name in 2.3 + newfunc.compat_func_name = name + return newfunc + return decorate + + +def raises(*exceptions): + """Test must raise one of expected exceptions to pass. + + Example use:: + + @raises(TypeError, ValueError) + def test_raises_type_error(): + raise TypeError("This test passes") + + @raises(Exception) + def test_that_fails_by_passing(): + pass + + If you want to test many assertions about exceptions in a single test, + you may want to use `assert_raises` instead. + """ + valid = ' or '.join([e.__name__ for e in exceptions]) + def decorate(func): + name = func.__name__ + def newfunc(*arg, **kw): + try: + func(*arg, **kw) + except exceptions: + pass + except: + raise + else: + message = "%s() did not raise %s" % (name, valid) + raise AssertionError(message) + newfunc = make_decorator(func)(newfunc) + return newfunc + return decorate + + +def set_trace(): + """Call pdb.set_trace in the calling frame, first restoring + sys.stdout to the real output stream. Note that sys.stdout is NOT + reset to whatever it was before the call once pdb is done! + """ + import pdb + import sys + stdout = sys.stdout + sys.stdout = sys.__stdout__ + pdb.Pdb().set_trace(sys._getframe().f_back) + + +def timed(limit): + """Test must finish within specified time limit to pass. + + Example use:: + + @timed(.1) + def test_that_fails(): + time.sleep(.2) + """ + def decorate(func): + def newfunc(*arg, **kw): + start = time.time() + result = func(*arg, **kw) + end = time.time() + if end - start > limit: + raise TimeExpired("Time limit (%s) exceeded" % limit) + return result + newfunc = make_decorator(func)(newfunc) + return newfunc + return decorate + + +def with_setup(setup=None, teardown=None): + """Decorator to add setup and/or teardown methods to a test function:: + + @with_setup(setup, teardown) + def test_something(): + " ... " + + Note that `with_setup` is useful *only* for test functions, not for test + methods or inside of TestCase subclasses. + """ + def decorate(func, setup=setup, teardown=teardown): + if setup: + if hasattr(func, 'setup'): + _old_s = func.setup + def _s(): + setup() + _old_s() + func.setup = _s + else: + func.setup = setup + if teardown: + if hasattr(func, 'teardown'): + _old_t = func.teardown + def _t(): + _old_t() + teardown() + func.teardown = _t + else: + func.teardown = teardown + return func + return decorate + + +def istest(func): + """Decorator to mark a function or method as a test + """ + func.__test__ = True + return func + + +def nottest(func): + """Decorator to mark a function or method as *not* a test + """ + func.__test__ = False + return func diff --git a/lib/spack/external/nose/tools/trivial.py b/lib/spack/external/nose/tools/trivial.py new file mode 100644 index 00000000000..cf83efeda5b --- /dev/null +++ b/lib/spack/external/nose/tools/trivial.py @@ -0,0 +1,54 @@ +"""Tools so trivial that tracebacks should not descend into them + +We define the ``__unittest`` symbol in their module namespace so unittest will +skip them when printing tracebacks, just as it does for their corresponding +methods in ``unittest`` proper. + +""" +import re +import unittest + + +__all__ = ['ok_', 'eq_'] + +# Use the same flag as unittest itself to prevent descent into these functions: +__unittest = 1 + + +def ok_(expr, msg=None): + """Shorthand for assert. Saves 3 whole characters! + """ + if not expr: + raise AssertionError(msg) + + +def eq_(a, b, msg=None): + """Shorthand for 'assert a == b, "%r != %r" % (a, b) + """ + if not a == b: + raise AssertionError(msg or "%r != %r" % (a, b)) + + +# +# Expose assert* from unittest.TestCase +# - give them pep8 style names +# +caps = re.compile('([A-Z])') + +def pep8(name): + return caps.sub(lambda m: '_' + m.groups()[0].lower(), name) + +class Dummy(unittest.TestCase): + def nop(): + pass +_t = Dummy('nop') + +for at in [ at for at in dir(_t) + if at.startswith('assert') and not '_' in at ]: + pepd = pep8(at) + vars()[pepd] = getattr(_t, at) + __all__.append(pepd) + +del Dummy +del _t +del pep8 diff --git a/lib/spack/external/nose/twistedtools.py b/lib/spack/external/nose/twistedtools.py new file mode 100644 index 00000000000..8d9c6ffe9b3 --- /dev/null +++ b/lib/spack/external/nose/twistedtools.py @@ -0,0 +1,173 @@ +""" +Twisted integration +------------------- + +This module provides a very simple way to integrate your tests with the +Twisted_ event loop. + +You must import this module *before* importing anything from Twisted itself! + +Example:: + + from nose.twistedtools import reactor, deferred + + @deferred() + def test_resolve(): + return reactor.resolve("www.python.org") + +Or, more realistically:: + + @deferred(timeout=5.0) + def test_resolve(): + d = reactor.resolve("www.python.org") + def check_ip(ip): + assert ip == "67.15.36.43" + d.addCallback(check_ip) + return d + +.. _Twisted: http://twistedmatrix.com/trac/ +""" + +import sys +from Queue import Queue, Empty +from nose.tools import make_decorator, TimeExpired + +__all__ = [ + 'threaded_reactor', 'reactor', 'deferred', 'TimeExpired', + 'stop_reactor' +] + +_twisted_thread = None + +def threaded_reactor(): + """ + Start the Twisted reactor in a separate thread, if not already done. + Returns the reactor. + The thread will automatically be destroyed when all the tests are done. + """ + global _twisted_thread + try: + from twisted.internet import reactor + except ImportError: + return None, None + if not _twisted_thread: + from twisted.python import threadable + from threading import Thread + _twisted_thread = Thread(target=lambda: reactor.run( \ + installSignalHandlers=False)) + _twisted_thread.setDaemon(True) + _twisted_thread.start() + return reactor, _twisted_thread + +# Export global reactor variable, as Twisted does +reactor, reactor_thread = threaded_reactor() + + +def stop_reactor(): + """Stop the reactor and join the reactor thread until it stops. + Call this function in teardown at the module or package level to + reset the twisted system after your tests. You *must* do this if + you mix tests using these tools and tests using twisted.trial. + """ + global _twisted_thread + + def stop_reactor(): + '''Helper for calling stop from withing the thread.''' + reactor.stop() + + reactor.callFromThread(stop_reactor) + reactor_thread.join() + for p in reactor.getDelayedCalls(): + if p.active(): + p.cancel() + _twisted_thread = None + + +def deferred(timeout=None): + """ + By wrapping a test function with this decorator, you can return a + twisted Deferred and the test will wait for the deferred to be triggered. + The whole test function will run inside the Twisted event loop. + + The optional timeout parameter specifies the maximum duration of the test. + The difference with timed() is that timed() will still wait for the test + to end, while deferred() will stop the test when its timeout has expired. + The latter is more desireable when dealing with network tests, because + the result may actually never arrive. + + If the callback is triggered, the test has passed. + If the errback is triggered or the timeout expires, the test has failed. + + Example:: + + @deferred(timeout=5.0) + def test_resolve(): + return reactor.resolve("www.python.org") + + Attention! If you combine this decorator with other decorators (like + "raises"), deferred() must be called *first*! + + In other words, this is good:: + + @raises(DNSLookupError) + @deferred() + def test_error(): + return reactor.resolve("xxxjhjhj.biz") + + and this is bad:: + + @deferred() + @raises(DNSLookupError) + def test_error(): + return reactor.resolve("xxxjhjhj.biz") + """ + reactor, reactor_thread = threaded_reactor() + if reactor is None: + raise ImportError("twisted is not available or could not be imported") + # Check for common syntax mistake + # (otherwise, tests can be silently ignored + # if one writes "@deferred" instead of "@deferred()") + try: + timeout is None or timeout + 0 + except TypeError: + raise TypeError("'timeout' argument must be a number or None") + + def decorate(func): + def wrapper(*args, **kargs): + q = Queue() + def callback(value): + q.put(None) + def errback(failure): + # Retrieve and save full exception info + try: + failure.raiseException() + except: + q.put(sys.exc_info()) + def g(): + try: + d = func(*args, **kargs) + try: + d.addCallbacks(callback, errback) + # Check for a common mistake and display a nice error + # message + except AttributeError: + raise TypeError("you must return a twisted Deferred " + "from your test case!") + # Catch exceptions raised in the test body (from the + # Twisted thread) + except: + q.put(sys.exc_info()) + reactor.callFromThread(g) + try: + error = q.get(timeout=timeout) + except Empty: + raise TimeExpired("timeout expired before end of test (%f s.)" + % timeout) + # Re-raise all exceptions + if error is not None: + exc_type, exc_value, tb = error + raise exc_type, exc_value, tb + wrapper = make_decorator(func)(wrapper) + return wrapper + return decorate + diff --git a/lib/spack/external/nose/usage.txt b/lib/spack/external/nose/usage.txt new file mode 100644 index 00000000000..bc96894ab78 --- /dev/null +++ b/lib/spack/external/nose/usage.txt @@ -0,0 +1,115 @@ +nose collects tests automatically from python source files, +directories and packages found in its working directory (which +defaults to the current working directory). Any python source file, +directory or package that matches the testMatch regular expression +(by default: `(?:^|[\b_\.-])[Tt]est)` will be collected as a test (or +source for collection of tests). In addition, all other packages +found in the working directory will be examined for python source files +or directories that match testMatch. Package discovery descends all +the way down the tree, so package.tests and package.sub.tests and +package.sub.sub2.tests will all be collected. + +Within a test directory or package, any python source file matching +testMatch will be examined for test cases. Within a test module, +functions and classes whose names match testMatch and TestCase +subclasses with any name will be loaded and executed as tests. Tests +may use the assert keyword or raise AssertionErrors to indicate test +failure. TestCase subclasses may do the same or use the various +TestCase methods available. + +**It is important to note that the default behavior of nose is to +not include tests from files which are executable.** To include +tests from such files, remove their executable bit or use +the --exe flag (see 'Options' section below). + +Selecting Tests +--------------- + +To specify which tests to run, pass test names on the command line: + + %prog only_test_this.py + +Test names specified may be file or module names, and may optionally +indicate the test case to run by separating the module or file name +from the test case name with a colon. Filenames may be relative or +absolute. Examples: + + %prog test.module + %prog another.test:TestCase.test_method + %prog a.test:TestCase + %prog /path/to/test/file.py:test_function + +You may also change the working directory where nose looks for tests +by using the -w switch: + + %prog -w /path/to/tests + +Note, however, that support for multiple -w arguments is now deprecated +and will be removed in a future release. As of nose 0.10, you can get +the same behavior by specifying the target directories *without* +the -w switch: + + %prog /path/to/tests /another/path/to/tests + +Further customization of test selection and loading is possible +through the use of plugins. + +Test result output is identical to that of unittest, except for +the additional features (error classes, and plugin-supplied +features such as output capture and assert introspection) detailed +in the options below. + +Configuration +------------- + +In addition to passing command-line options, you may also put +configuration options in your project's *setup.cfg* file, or a .noserc +or nose.cfg file in your home directory. In any of these standard +ini-style config files, you put your nosetests configuration in a +``[nosetests]`` section. Options are the same as on the command line, +with the -- prefix removed. For options that are simple switches, you +must supply a value: + + [nosetests] + verbosity=3 + with-doctest=1 + +All configuration files that are found will be loaded and their +options combined. You can override the standard config file loading +with the ``-c`` option. + +Using Plugins +------------- + +There are numerous nose plugins available via easy_install and +elsewhere. To use a plugin, just install it. The plugin will add +command line options to nosetests. To verify that the plugin is installed, +run: + + nosetests --plugins + +You can add -v or -vv to that command to show more information +about each plugin. + +If you are running nose.main() or nose.run() from a script, you +can specify a list of plugins to use by passing a list of plugins +with the plugins keyword argument. + +0.9 plugins +----------- + +nose 1.0 can use SOME plugins that were written for nose 0.9. The +default plugin manager inserts a compatibility wrapper around 0.9 +plugins that adapts the changed plugin api calls. However, plugins +that access nose internals are likely to fail, especially if they +attempt to access test case or test suite classes. For example, +plugins that try to determine if a test passed to startTest is an +individual test or a suite will fail, partly because suites are no +longer passed to startTest and partly because it's likely that the +plugin is trying to find out if the test is an instance of a class +that no longer exists. + +0.10 and 0.11 plugins +--------------------- + +All plugins written for nose 0.10 and 0.11 should work with nose 1.0. diff --git a/lib/spack/external/nose/util.py b/lib/spack/external/nose/util.py new file mode 100644 index 00000000000..bfe16589ea1 --- /dev/null +++ b/lib/spack/external/nose/util.py @@ -0,0 +1,668 @@ +"""Utility functions and classes used by nose internally. +""" +import inspect +import itertools +import logging +import stat +import os +import re +import sys +import types +import unittest +from nose.pyversion import ClassType, TypeType, isgenerator, ismethod + + +log = logging.getLogger('nose') + +ident_re = re.compile(r'^[A-Za-z_][A-Za-z0-9_.]*$') +class_types = (ClassType, TypeType) +skip_pattern = r"(?:\.svn)|(?:[^.]+\.py[co])|(?:.*~)|(?:.*\$py\.class)|(?:__pycache__)" + +try: + set() + set = set # make from nose.util import set happy +except NameError: + try: + from sets import Set as set + except ImportError: + pass + + +def ls_tree(dir_path="", + skip_pattern=skip_pattern, + indent="|-- ", branch_indent="| ", + last_indent="`-- ", last_branch_indent=" "): + # TODO: empty directories look like non-directory files + return "\n".join(_ls_tree_lines(dir_path, skip_pattern, + indent, branch_indent, + last_indent, last_branch_indent)) + + +def _ls_tree_lines(dir_path, skip_pattern, + indent, branch_indent, last_indent, last_branch_indent): + if dir_path == "": + dir_path = os.getcwd() + + lines = [] + + names = os.listdir(dir_path) + names.sort() + dirs, nondirs = [], [] + for name in names: + if re.match(skip_pattern, name): + continue + if os.path.isdir(os.path.join(dir_path, name)): + dirs.append(name) + else: + nondirs.append(name) + + # list non-directories first + entries = list(itertools.chain([(name, False) for name in nondirs], + [(name, True) for name in dirs])) + def ls_entry(name, is_dir, ind, branch_ind): + if not is_dir: + yield ind + name + else: + path = os.path.join(dir_path, name) + if not os.path.islink(path): + yield ind + name + subtree = _ls_tree_lines(path, skip_pattern, + indent, branch_indent, + last_indent, last_branch_indent) + for x in subtree: + yield branch_ind + x + for name, is_dir in entries[:-1]: + for line in ls_entry(name, is_dir, indent, branch_indent): + yield line + if entries: + name, is_dir = entries[-1] + for line in ls_entry(name, is_dir, last_indent, last_branch_indent): + yield line + + +def absdir(path): + """Return absolute, normalized path to directory, if it exists; None + otherwise. + """ + if not os.path.isabs(path): + path = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), + path))) + if path is None or not os.path.isdir(path): + return None + return path + + +def absfile(path, where=None): + """Return absolute, normalized path to file (optionally in directory + where), or None if the file can't be found either in where or the current + working directory. + """ + orig = path + if where is None: + where = os.getcwd() + if isinstance(where, list) or isinstance(where, tuple): + for maybe_path in where: + maybe_abs = absfile(path, maybe_path) + if maybe_abs is not None: + return maybe_abs + return None + if not os.path.isabs(path): + path = os.path.normpath(os.path.abspath(os.path.join(where, path))) + if path is None or not os.path.exists(path): + if where != os.getcwd(): + # try the cwd instead + path = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), + orig))) + if path is None or not os.path.exists(path): + return None + if os.path.isdir(path): + # might want an __init__.py from pacakge + init = os.path.join(path,'__init__.py') + if os.path.isfile(init): + return init + elif os.path.isfile(path): + return path + return None + + +def anyp(predicate, iterable): + for item in iterable: + if predicate(item): + return True + return False + + +def file_like(name): + """A name is file-like if it is a path that exists, or it has a + directory part, or it ends in .py, or it isn't a legal python + identifier. + """ + return (os.path.exists(name) + or os.path.dirname(name) + or name.endswith('.py') + or not ident_re.match(os.path.splitext(name)[0])) + + +def func_lineno(func): + """Get the line number of a function. First looks for + compat_co_firstlineno, then func_code.co_first_lineno. + """ + try: + return func.compat_co_firstlineno + except AttributeError: + try: + return func.func_code.co_firstlineno + except AttributeError: + return -1 + + +def isclass(obj): + """Is obj a class? Inspect's isclass is too liberal and returns True + for objects that can't be subclasses of anything. + """ + obj_type = type(obj) + return obj_type in class_types or issubclass(obj_type, type) + + +# backwards compat (issue #64) +is_generator = isgenerator + + +def ispackage(path): + """ + Is this path a package directory? + + >>> ispackage('nose') + True + >>> ispackage('unit_tests') + False + >>> ispackage('nose/plugins') + True + >>> ispackage('nose/loader.py') + False + """ + if os.path.isdir(path): + # at least the end of the path must be a legal python identifier + # and __init__.py[co] must exist + end = os.path.basename(path) + if ident_re.match(end): + for init in ('__init__.py', '__init__.pyc', '__init__.pyo'): + if os.path.isfile(os.path.join(path, init)): + return True + if sys.platform.startswith('java') and \ + os.path.isfile(os.path.join(path, '__init__$py.class')): + return True + return False + + +def isproperty(obj): + """ + Is this a property? + + >>> class Foo: + ... def got(self): + ... return 2 + ... def get(self): + ... return 1 + ... get = property(get) + + >>> isproperty(Foo.got) + False + >>> isproperty(Foo.get) + True + """ + return type(obj) == property + + +def getfilename(package, relativeTo=None): + """Find the python source file for a package, relative to a + particular directory (defaults to current working directory if not + given). + """ + if relativeTo is None: + relativeTo = os.getcwd() + path = os.path.join(relativeTo, os.sep.join(package.split('.'))) + if os.path.exists(path + '/__init__.py'): + return path + filename = path + '.py' + if os.path.exists(filename): + return filename + return None + + +def getpackage(filename): + """ + Find the full dotted package name for a given python source file + name. Returns None if the file is not a python source file. + + >>> getpackage('foo.py') + 'foo' + >>> getpackage('biff/baf.py') + 'baf' + >>> getpackage('nose/util.py') + 'nose.util' + + Works for directories too. + + >>> getpackage('nose') + 'nose' + >>> getpackage('nose/plugins') + 'nose.plugins' + + And __init__ files stuck onto directories + + >>> getpackage('nose/plugins/__init__.py') + 'nose.plugins' + + Absolute paths also work. + + >>> path = os.path.abspath(os.path.join('nose', 'plugins')) + >>> getpackage(path) + 'nose.plugins' + """ + src_file = src(filename) + if (os.path.isdir(src_file) or not src_file.endswith('.py')) and not ispackage(src_file): + return None + base, ext = os.path.splitext(os.path.basename(src_file)) + if base == '__init__': + mod_parts = [] + else: + mod_parts = [base] + path, part = os.path.split(os.path.split(src_file)[0]) + while part: + if ispackage(os.path.join(path, part)): + mod_parts.append(part) + else: + break + path, part = os.path.split(path) + mod_parts.reverse() + return '.'.join(mod_parts) + + +def ln(label): + """Draw a 70-char-wide divider, with label in the middle. + + >>> ln('hello there') + '---------------------------- hello there -----------------------------' + """ + label_len = len(label) + 2 + chunk = (70 - label_len) // 2 + out = '%s %s %s' % ('-' * chunk, label, '-' * chunk) + pad = 70 - len(out) + if pad > 0: + out = out + ('-' * pad) + return out + + +def resolve_name(name, module=None): + """Resolve a dotted name to a module and its parts. This is stolen + wholesale from unittest.TestLoader.loadTestByName. + + >>> resolve_name('nose.util') #doctest: +ELLIPSIS + + >>> resolve_name('nose.util.resolve_name') #doctest: +ELLIPSIS + + """ + parts = name.split('.') + parts_copy = parts[:] + if module is None: + while parts_copy: + try: + log.debug("__import__ %s", name) + module = __import__('.'.join(parts_copy)) + break + except ImportError: + del parts_copy[-1] + if not parts_copy: + raise + parts = parts[1:] + obj = module + log.debug("resolve: %s, %s, %s, %s", parts, name, obj, module) + for part in parts: + obj = getattr(obj, part) + return obj + + +def split_test_name(test): + """Split a test name into a 3-tuple containing file, module, and callable + names, any of which (but not all) may be blank. + + Test names are in the form: + + file_or_module:callable + + Either side of the : may be dotted. To change the splitting behavior, you + can alter nose.util.split_test_re. + """ + norm = os.path.normpath + file_or_mod = test + fn = None + if not ':' in test: + # only a file or mod part + if file_like(test): + return (norm(test), None, None) + else: + return (None, test, None) + + # could be path|mod:callable, or a : in the file path someplace + head, tail = os.path.split(test) + if not head: + # this is a case like 'foo:bar' -- generally a module + # name followed by a callable, but also may be a windows + # drive letter followed by a path + try: + file_or_mod, fn = test.split(':') + if file_like(fn): + # must be a funny path + file_or_mod, fn = test, None + except ValueError: + # more than one : in the test + # this is a case like c:\some\path.py:a_test + parts = test.split(':') + if len(parts[0]) == 1: + file_or_mod, fn = ':'.join(parts[:-1]), parts[-1] + else: + # nonsense like foo:bar:baz + raise ValueError("Test name '%s' could not be parsed. Please " + "format test names as path:callable or " + "module:callable." % (test,)) + elif not tail: + # this is a case like 'foo:bar/' + # : must be part of the file path, so ignore it + file_or_mod = test + else: + if ':' in tail: + file_part, fn = tail.split(':') + else: + file_part = tail + file_or_mod = os.sep.join([head, file_part]) + if file_or_mod: + if file_like(file_or_mod): + return (norm(file_or_mod), None, fn) + else: + return (None, file_or_mod, fn) + else: + return (None, None, fn) +split_test_name.__test__ = False # do not collect + + +def test_address(test): + """Find the test address for a test, which may be a module, filename, + class, method or function. + """ + if hasattr(test, "address"): + return test.address() + # type-based polymorphism sucks in general, but I believe is + # appropriate here + t = type(test) + file = module = call = None + if t == types.ModuleType: + file = getattr(test, '__file__', None) + module = getattr(test, '__name__', None) + return (src(file), module, call) + if t == types.FunctionType or issubclass(t, type) or t == types.ClassType: + module = getattr(test, '__module__', None) + if module is not None: + m = sys.modules[module] + file = getattr(m, '__file__', None) + if file is not None: + file = os.path.abspath(file) + call = getattr(test, '__name__', None) + return (src(file), module, call) + if t == types.MethodType: + cls_adr = test_address(test.im_class) + return (src(cls_adr[0]), cls_adr[1], + "%s.%s" % (cls_adr[2], test.__name__)) + # handle unittest.TestCase instances + if isinstance(test, unittest.TestCase): + if (hasattr(test, '_FunctionTestCase__testFunc') # pre 2.7 + or hasattr(test, '_testFunc')): # 2.7 + # unittest FunctionTestCase + try: + return test_address(test._FunctionTestCase__testFunc) + except AttributeError: + return test_address(test._testFunc) + # regular unittest.TestCase + cls_adr = test_address(test.__class__) + # 2.5 compat: __testMethodName changed to _testMethodName + try: + method_name = test._TestCase__testMethodName + except AttributeError: + method_name = test._testMethodName + return (src(cls_adr[0]), cls_adr[1], + "%s.%s" % (cls_adr[2], method_name)) + if (hasattr(test, '__class__') and + test.__class__.__module__ not in ('__builtin__', 'builtins')): + return test_address(test.__class__) + raise TypeError("I don't know what %s is (%s)" % (test, t)) +test_address.__test__ = False # do not collect + + +def try_run(obj, names): + """Given a list of possible method names, try to run them with the + provided object. Keep going until something works. Used to run + setup/teardown methods for module, package, and function tests. + """ + for name in names: + func = getattr(obj, name, None) + if func is not None: + if type(obj) == types.ModuleType: + # py.test compatibility + if isinstance(func, types.FunctionType): + args, varargs, varkw, defaults = \ + inspect.getargspec(func) + else: + # Not a function. If it's callable, call it anyway + if hasattr(func, '__call__') and not inspect.ismethod(func): + func = func.__call__ + try: + args, varargs, varkw, defaults = \ + inspect.getargspec(func) + args.pop(0) # pop the self off + except TypeError: + raise TypeError("Attribute %s of %r is not a python " + "function. Only functions or callables" + " may be used as fixtures." % + (name, obj)) + if len(args): + log.debug("call fixture %s.%s(%s)", obj, name, obj) + return func(obj) + log.debug("call fixture %s.%s", obj, name) + return func() + + +def src(filename): + """Find the python source file for a .pyc, .pyo or $py.class file on + jython. Returns the filename provided if it is not a python source + file. + """ + if filename is None: + return filename + if sys.platform.startswith('java') and filename.endswith('$py.class'): + return '.'.join((filename[:-9], 'py')) + base, ext = os.path.splitext(filename) + if ext in ('.pyc', '.pyo', '.py'): + return '.'.join((base, 'py')) + return filename + + +def regex_last_key(regex): + """Sort key function factory that puts items that match a + regular expression last. + + >>> from nose.config import Config + >>> from nose.pyversion import sort_list + >>> c = Config() + >>> regex = c.testMatch + >>> entries = ['.', '..', 'a_test', 'src', 'lib', 'test', 'foo.py'] + >>> sort_list(entries, regex_last_key(regex)) + >>> entries + ['.', '..', 'foo.py', 'lib', 'src', 'a_test', 'test'] + """ + def k(obj): + if regex.search(obj): + return (1, obj) + return (0, obj) + return k + + +def tolist(val): + """Convert a value that may be a list or a (possibly comma-separated) + string into a list. The exception: None is returned as None, not [None]. + + >>> tolist(["one", "two"]) + ['one', 'two'] + >>> tolist("hello") + ['hello'] + >>> tolist("separate,values, with, commas, spaces , are ,ok") + ['separate', 'values', 'with', 'commas', 'spaces', 'are', 'ok'] + """ + if val is None: + return None + try: + # might already be a list + val.extend([]) + return val + except AttributeError: + pass + # might be a string + try: + return re.split(r'\s*,\s*', val) + except TypeError: + # who knows... + return list(val) + + +class odict(dict): + """Simple ordered dict implementation, based on: + + http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/107747 + """ + def __init__(self, *arg, **kw): + self._keys = [] + super(odict, self).__init__(*arg, **kw) + + def __delitem__(self, key): + super(odict, self).__delitem__(key) + self._keys.remove(key) + + def __setitem__(self, key, item): + super(odict, self).__setitem__(key, item) + if key not in self._keys: + self._keys.append(key) + + def __str__(self): + return "{%s}" % ', '.join(["%r: %r" % (k, v) for k, v in self.items()]) + + def clear(self): + super(odict, self).clear() + self._keys = [] + + def copy(self): + d = super(odict, self).copy() + d._keys = self._keys[:] + return d + + def items(self): + return zip(self._keys, self.values()) + + def keys(self): + return self._keys[:] + + def setdefault(self, key, failobj=None): + item = super(odict, self).setdefault(key, failobj) + if key not in self._keys: + self._keys.append(key) + return item + + def update(self, dict): + super(odict, self).update(dict) + for key in dict.keys(): + if key not in self._keys: + self._keys.append(key) + + def values(self): + return map(self.get, self._keys) + + +def transplant_func(func, module): + """ + Make a function imported from module A appear as if it is located + in module B. + + >>> from pprint import pprint + >>> pprint.__module__ + 'pprint' + >>> pp = transplant_func(pprint, __name__) + >>> pp.__module__ + 'nose.util' + + The original function is not modified. + + >>> pprint.__module__ + 'pprint' + + Calling the transplanted function calls the original. + + >>> pp([1, 2]) + [1, 2] + >>> pprint([1,2]) + [1, 2] + + """ + from nose.tools import make_decorator + if isgenerator(func): + def newfunc(*arg, **kw): + for v in func(*arg, **kw): + yield v + else: + def newfunc(*arg, **kw): + return func(*arg, **kw) + + newfunc = make_decorator(func)(newfunc) + newfunc.__module__ = module + return newfunc + + +def transplant_class(cls, module): + """ + Make a class appear to reside in `module`, rather than the module in which + it is actually defined. + + >>> from nose.failure import Failure + >>> Failure.__module__ + 'nose.failure' + >>> Nf = transplant_class(Failure, __name__) + >>> Nf.__module__ + 'nose.util' + >>> Nf.__name__ + 'Failure' + + """ + class C(cls): + pass + C.__module__ = module + C.__name__ = cls.__name__ + return C + + +def safe_str(val, encoding='utf-8'): + try: + return str(val) + except UnicodeEncodeError: + if isinstance(val, Exception): + return ' '.join([safe_str(arg, encoding) + for arg in val]) + return unicode(val).encode(encoding) + + +def is_executable(file): + if not os.path.exists(file): + return False + st = os.stat(file) + return bool(st.st_mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)) + + +if __name__ == '__main__': + import doctest + doctest.testmod() From 29537b99919af21f62f7ea2bea824714dd0ccd60 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 5 Dec 2015 11:21:47 -0500 Subject: [PATCH 129/910] Update gmp to 6.1.0 --- var/spack/packages/gmp/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/gmp/package.py b/var/spack/packages/gmp/package.py index d6af821b343..6cd501dbf20 100644 --- a/var/spack/packages/gmp/package.py +++ b/var/spack/packages/gmp/package.py @@ -31,6 +31,7 @@ class Gmp(Package): homepage = "https://gmplib.org" url = "https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2" + version('6.1.0' , '86ee6e54ebfc4a90b643a65e402c4048') version('6.0.0a', 'b7ff2d88cae7f8085bd5006096eed470') version('6.0.0' , '6ef5869ae735db9995619135bd856b84') From aff95ee587cf94937fe0d95cc78f7372830e3223 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 5 Dec 2015 11:23:40 -0500 Subject: [PATCH 130/910] Update mpc to 1.0.3 --- var/spack/packages/mpc/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/mpc/package.py b/var/spack/packages/mpc/package.py index 6fbfca30070..33706f043f2 100644 --- a/var/spack/packages/mpc/package.py +++ b/var/spack/packages/mpc/package.py @@ -31,6 +31,7 @@ class Mpc(Package): homepage = "http://www.multiprecision.org" url = "ftp://ftp.gnu.org/gnu/mpc/mpc-1.0.2.tar.gz" + version('1.0.3', 'd6a1d5f8ddea3abd2cc3e98f58352d26') version('1.0.2', '68fadff3358fb3e7976c7a398a0af4c3') depends_on("gmp") From 77bf133cd295c7c5521799a22cd70a873cef1b2e Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Mon, 7 Dec 2015 14:33:31 -0800 Subject: [PATCH 131/910] Update CleverLeaf package to use Boost and fix SAMRAI MPI usage --- var/spack/packages/SAMRAI/package.py | 2 ++ var/spack/packages/cleverleaf/package.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/var/spack/packages/SAMRAI/package.py b/var/spack/packages/SAMRAI/package.py index 5ee90cf400b..f8caa1185c0 100644 --- a/var/spack/packages/SAMRAI/package.py +++ b/var/spack/packages/SAMRAI/package.py @@ -38,6 +38,8 @@ def install(self, spec, prefix): configure( "--prefix=%s" % prefix, + "--with-CXX=%s" % spec[mpi].prefix.bin + "/mpic++", + "--with-CC=%s" % spec[mpi].prefix.bin + "/mpicc", "--with-hdf5=%s" % spec['hdf5'].prefix, "--with-boost=%s" % spec['boost'].prefix, "--with-zlib=%s" % spec['zlib'].prefix, diff --git a/var/spack/packages/cleverleaf/package.py b/var/spack/packages/cleverleaf/package.py index ddbe57f0194..ad843ca4125 100644 --- a/var/spack/packages/cleverleaf/package.py +++ b/var/spack/packages/cleverleaf/package.py @@ -14,6 +14,8 @@ class Cleverleaf(Package): version('develop', git='https://github.com/UK-MAC/CleverLeaf_ref.git', branch='develop') depends_on("SAMRAI@3.8.0") + depends_on("hdf5") + depends_on("boost") def install(self, spec, prefix): cmake(*std_cmake_args) From 36272c813675c98864e7b9eda50e99ea800dfe21 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Mon, 7 Dec 2015 14:58:16 -0800 Subject: [PATCH 132/910] Fix MPI selection in SAMRAI and remove narrow dependency specification --- var/spack/packages/SAMRAI/package.py | 7 ++----- var/spack/packages/cleverleaf/package.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/var/spack/packages/SAMRAI/package.py b/var/spack/packages/SAMRAI/package.py index f8caa1185c0..a17aea9c993 100644 --- a/var/spack/packages/SAMRAI/package.py +++ b/var/spack/packages/SAMRAI/package.py @@ -33,13 +33,10 @@ class Samrai(Package): # TODO: currently hard-coded to use openmpi - be careful! def install(self, spec, prefix): - mpi = next(m for m in ('openmpi', 'mpich', 'mvapich') - if m in spec) - configure( "--prefix=%s" % prefix, - "--with-CXX=%s" % spec[mpi].prefix.bin + "/mpic++", - "--with-CC=%s" % spec[mpi].prefix.bin + "/mpicc", + "--with-CXX=%s" % spec['mpi'].prefix.bin + "/mpic++", + "--with-CC=%s" % spec['mpi'].prefix.bin + "/mpicc", "--with-hdf5=%s" % spec['hdf5'].prefix, "--with-boost=%s" % spec['boost'].prefix, "--with-zlib=%s" % spec['zlib'].prefix, diff --git a/var/spack/packages/cleverleaf/package.py b/var/spack/packages/cleverleaf/package.py index ad843ca4125..4e7e6a855ab 100644 --- a/var/spack/packages/cleverleaf/package.py +++ b/var/spack/packages/cleverleaf/package.py @@ -13,7 +13,7 @@ class Cleverleaf(Package): version('develop', git='https://github.com/UK-MAC/CleverLeaf_ref.git', branch='develop') - depends_on("SAMRAI@3.8.0") + depends_on("SAMRAI@3.8.0:") depends_on("hdf5") depends_on("boost") From 1a22a507d64f45f4aa2a1dae03604ba83cfb9736 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Mon, 7 Dec 2015 15:36:36 -0800 Subject: [PATCH 133/910] mesa requires +python variant of libxml2 --- var/spack/packages/mesa/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/mesa/package.py b/var/spack/packages/mesa/package.py index 2a04a8fd511..62da8c993b3 100644 --- a/var/spack/packages/mesa/package.py +++ b/var/spack/packages/mesa/package.py @@ -14,7 +14,7 @@ class Mesa(Package): # mesa 7.x, 8.x, 9.x depends_on("libdrm@2.4.33") depends_on("llvm@3.0") - depends_on("libxml2") + depends_on("libxml2+python") # patch("llvm-fixes.patch") # using newer llvm From f8e046bb33b23890e1e3bf92866bd6ea5cefbbec Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 7 Dec 2015 15:49:47 -0800 Subject: [PATCH 134/910] Fix issue with gcc 5.x not building on RHEL6 --- var/spack/packages/gcc/package.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py index a49a1348aac..f58c1bd4169 100644 --- a/var/spack/packages/gcc/package.py +++ b/var/spack/packages/gcc/package.py @@ -79,8 +79,9 @@ def install(self, spec, prefix): "--with-gnu-as", "--with-quad"] # Binutils - binutils_options = ["--with-stage1-ldflags=%s" % self.rpath_args, - "--with-boot-ldflags=%s" % self.rpath_args, + static_bootstrap_flags = "-static-libstdc++ -static-libgcc" + binutils_options = ["--with-stage1-ldflags=%s %s" % (self.rpath_args, static_bootstrap_flags), + "--with-boot-ldflags=%s %s" % (self.rpath_args, static_bootstrap_flags), "--with-ld=%s/bin/ld" % spec['binutils'].prefix, "--with-as=%s/bin/as" % spec['binutils'].prefix] options.extend(binutils_options) @@ -89,11 +90,13 @@ def install(self, spec, prefix): isl_options = ["--with-isl=%s" % spec['isl'].prefix] options.extend(isl_options) - # Rest of install is straightforward. - configure(*options) - make() - make("install") - + with working_dir('spack-build', create=True): + # Rest of install is straightforward. + configure = Executable('../configure') + configure(*options) + make() + make("install") + self.write_rpath_specs() From c5b9732aed27e2061d2b50c02bc9933b5686c946 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 8 Dec 2015 09:57:07 +0100 Subject: [PATCH 135/910] Should fix #177 gcc : changed objdir to something out of srcdir and added '--with-sysroot=/' binutils : added '--with-sysroot=/' --- var/spack/packages/binutils/package.py | 3 ++- var/spack/packages/gcc/package.py | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py index cac0a0407f9..be5a9b714b4 100644 --- a/var/spack/packages/binutils/package.py +++ b/var/spack/packages/binutils/package.py @@ -20,7 +20,8 @@ def install(self, spec, prefix): '--enable-multilib', '--enable-shared', '--enable-64-bit-bfd', - '--enable-targets=all'] + '--enable-targets=all', + '--with-sysroot=/'] if '+libiberty' in spec: configure_args.append('--enable-install-libiberty') diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py index f58c1bd4169..b5dc585be79 100644 --- a/var/spack/packages/gcc/package.py +++ b/var/spack/packages/gcc/package.py @@ -38,6 +38,7 @@ class Gcc(Package): DEPENDS_ON_ISL_PREDICATE = '@5.0:' + version('5.3.0', 'c9616fd448f980259c31de613e575719') version('5.2.0', 'a51bcfeb3da7dd4c623e27207ed43467') version('4.9.3', '6f831b4d251872736e8e9cc09746f327') version('4.9.2', '4df8ee253b7f3863ad0b86359cd39c43') @@ -80,7 +81,8 @@ def install(self, spec, prefix): "--with-quad"] # Binutils static_bootstrap_flags = "-static-libstdc++ -static-libgcc" - binutils_options = ["--with-stage1-ldflags=%s %s" % (self.rpath_args, static_bootstrap_flags), + binutils_options = ["--with-sysroot=/", + "--with-stage1-ldflags=%s %s" % (self.rpath_args, static_bootstrap_flags), "--with-boot-ldflags=%s %s" % (self.rpath_args, static_bootstrap_flags), "--with-ld=%s/bin/ld" % spec['binutils'].prefix, "--with-as=%s/bin/as" % spec['binutils'].prefix] @@ -90,9 +92,10 @@ def install(self, spec, prefix): isl_options = ["--with-isl=%s" % spec['isl'].prefix] options.extend(isl_options) - with working_dir('spack-build', create=True): + build_dir = join_path(self.stage.path, 'spack-build') + configure = Executable( join_path(self.stage.source_path, 'configure') ) + with working_dir(build_dir, create=True): # Rest of install is straightforward. - configure = Executable('../configure') configure(*options) make() make("install") From 4d5eb4ce7daf2bb0b95067114d9ba756a4eb4f7d Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 9 Dec 2015 09:06:10 +0100 Subject: [PATCH 136/910] gdb : added package --- var/spack/packages/gdb/package.py | 48 +++++++++++++++++++++++++++ var/spack/packages/texinfo/package.py | 46 +++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 var/spack/packages/gdb/package.py create mode 100644 var/spack/packages/texinfo/package.py diff --git a/var/spack/packages/gdb/package.py b/var/spack/packages/gdb/package.py new file mode 100644 index 00000000000..fd567f346b4 --- /dev/null +++ b/var/spack/packages/gdb/package.py @@ -0,0 +1,48 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Gdb(Package): + """ + GDB, the GNU Project debugger, allows you to see what is going on `inside' another program while it executes + -- or what another program was doing at the moment it crashed. + """ + homepage = "https://www.gnu.org/software/gdb" + url = "http://ftp.gnu.org/gnu/gdb/gdb-7.10.tar.xz" + + version('7.10.1', '39e654460c9cdd80200a29ac020cfe11') + version('7.10', '2a35bac41fa8e10bf04f3a0dd7f7f363') + version('7.9.1', '35374c77a70884eb430c97061053a36e') + version('7.9', 'e6279f26559d839f0b4218a482bcb43e') + version('7.8.2', 'a80cf252ed2e775d4e4533341bbf2459') + + depends_on('texinfo') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") diff --git a/var/spack/packages/texinfo/package.py b/var/spack/packages/texinfo/package.py new file mode 100644 index 00000000000..460db65b3ec --- /dev/null +++ b/var/spack/packages/texinfo/package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Texinfo(Package): + """ + Texinfo is the official documentation format of the GNU project. It was invented by Richard Stallman and Bob + Chassell many years ago, loosely based on Brian Reid's Scribe and other formatting languages of the time. It is + used by many non-GNU projects as well.FIXME: put a proper description of your package here. + """ + homepage = "https://www.gnu.org/software/texinfo/" + url = "http://ftp.gnu.org/gnu/texinfo/texinfo-6.0.tar.xz" + + version('6.0', '02818e62a5b8ae0213a7ff572991bb50') + version('5.2', 'cb489df8a7ee9d10a236197aefdb32c5') + version('5.1', '52ee905a3b705020d2a1b6ec36d53ca6') + version('5.0', 'ef2fad34c71ddc95b20c7d6a08c0d7a6') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") From b8bb24d4bf89cd4c5d8ed9f465a260846aa86ef7 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Dec 2015 01:09:51 -0800 Subject: [PATCH 137/910] Update README.md and LICENSE with new github.com/llnl URLs --- LICENSE | 2 +- README.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/LICENSE b/LICENSE index 6ad4af58614..bb7ef91b3c9 100644 --- a/LICENSE +++ b/LICENSE @@ -5,7 +5,7 @@ This file is part of Spack. Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. LLNL-CODE-647188 -For details, see https://scalability-llnl.github.io/spack +For details, see https://github.com/llnl/spack This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License (as published by diff --git a/README.md b/README.md index 9830e31356a..ffd118ded81 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ![image](share/spack/logo/spack-logo-text-64.png "Spack") ============ -[![Build Status](https://travis-ci.org/scalability-llnl/spack.png?branch=develop)](https://travis-ci.org/scalability-llnl/spack) +[![Build Status](https://travis-ci.org/llnl/spack.png?branch=develop)](https://travis-ci.org/llnl/spack) Spack is a package management tool designed to support multiple versions and configurations of software on a wide variety of platforms @@ -19,19 +19,19 @@ written in pure Python, and specs allow package authors to write a single build script for many different builds of the same package. See the -[Feature Overview](http://scalability-llnl.github.io/spack/features.html) +[Feature Overview](http://llnl.github.io/spack/features.html) for examples and highlights. To install spack and install your first package: - $ git clone https://github.com/scalability-llnl/spack.git + $ git clone https://github.com/llnl/spack.git $ cd spack/bin $ ./spack install libelf Documentation ---------------- -[**Full documentation**](http://scalability-llnl.github.io/spack) for Spack is +[**Full documentation**](http://llnl.github.io/spack) for Spack is the first place to look. See also: @@ -70,7 +70,7 @@ latest stable release. Authors ---------------- -Many thanks go to Spack's [contributors](https://github.com/scalability-llnl/spack/graphs/contributors). +Many thanks go to Spack's [contributors](https://github.com/llnl/spack/graphs/contributors). Spack was originally written by Todd Gamblin, tgamblin@llnl.gov. From c41b9b7ddc05271e46a86e21e682291420e6e488 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Dec 2015 01:24:15 -0800 Subject: [PATCH 138/910] Change github.com/scalability-llnl to github.com/llnl everywhere. --- bin/spack | 2 +- bin/spack-python | 2 +- lib/spack/docs/conf.py | 2 +- lib/spack/docs/exts/sphinxcontrib/__init__.py | 2 +- lib/spack/docs/exts/sphinxcontrib/programoutput.py | 2 +- lib/spack/docs/getting_started.rst | 6 +++--- lib/spack/docs/index.rst | 4 ++-- lib/spack/env/cc | 2 +- lib/spack/external/__init__.py | 2 +- lib/spack/llnl/util/filesystem.py | 2 +- lib/spack/llnl/util/lang.py | 2 +- lib/spack/llnl/util/link_tree.py | 2 +- lib/spack/llnl/util/lock.py | 2 +- lib/spack/llnl/util/tty/__init__.py | 2 +- lib/spack/llnl/util/tty/colify.py | 2 +- lib/spack/llnl/util/tty/color.py | 2 +- lib/spack/llnl/util/tty/log.py | 2 +- lib/spack/spack/__init__.py | 2 +- lib/spack/spack/architecture.py | 2 +- lib/spack/spack/cmd/__init__.py | 2 +- lib/spack/spack/cmd/activate.py | 2 +- lib/spack/spack/cmd/arch.py | 2 +- lib/spack/spack/cmd/bootstrap.py | 2 +- lib/spack/spack/cmd/cd.py | 2 +- lib/spack/spack/cmd/checksum.py | 2 +- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/cmd/compiler.py | 2 +- lib/spack/spack/cmd/compilers.py | 2 +- lib/spack/spack/cmd/config.py | 2 +- lib/spack/spack/cmd/create.py | 2 +- lib/spack/spack/cmd/deactivate.py | 2 +- lib/spack/spack/cmd/dependents.py | 2 +- lib/spack/spack/cmd/diy.py | 2 +- lib/spack/spack/cmd/doc.py | 2 +- lib/spack/spack/cmd/edit.py | 2 +- lib/spack/spack/cmd/env.py | 2 +- lib/spack/spack/cmd/extensions.py | 2 +- lib/spack/spack/cmd/fetch.py | 2 +- lib/spack/spack/cmd/find.py | 2 +- lib/spack/spack/cmd/graph.py | 2 +- lib/spack/spack/cmd/help.py | 2 +- lib/spack/spack/cmd/info.py | 2 +- lib/spack/spack/cmd/install.py | 2 +- lib/spack/spack/cmd/list.py | 2 +- lib/spack/spack/cmd/load.py | 2 +- lib/spack/spack/cmd/location.py | 2 +- lib/spack/spack/cmd/md5.py | 2 +- lib/spack/spack/cmd/mirror.py | 2 +- lib/spack/spack/cmd/module.py | 2 +- lib/spack/spack/cmd/package-list.py | 4 ++-- lib/spack/spack/cmd/patch.py | 2 +- lib/spack/spack/cmd/pkg.py | 2 +- lib/spack/spack/cmd/providers.py | 2 +- lib/spack/spack/cmd/purge.py | 2 +- lib/spack/spack/cmd/python.py | 2 +- lib/spack/spack/cmd/reindex.py | 2 +- lib/spack/spack/cmd/restage.py | 2 +- lib/spack/spack/cmd/spec.py | 2 +- lib/spack/spack/cmd/stage.py | 2 +- lib/spack/spack/cmd/test-install.py | 2 +- lib/spack/spack/cmd/test.py | 2 +- lib/spack/spack/cmd/uninstall.py | 2 +- lib/spack/spack/cmd/unload.py | 2 +- lib/spack/spack/cmd/unuse.py | 2 +- lib/spack/spack/cmd/urls.py | 2 +- lib/spack/spack/cmd/use.py | 2 +- lib/spack/spack/cmd/versions.py | 2 +- lib/spack/spack/compiler.py | 2 +- lib/spack/spack/compilers/__init__.py | 2 +- lib/spack/spack/compilers/clang.py | 2 +- lib/spack/spack/compilers/gcc.py | 2 +- lib/spack/spack/compilers/intel.py | 2 +- lib/spack/spack/compilers/pgi.py | 2 +- lib/spack/spack/compilers/xl.py | 2 +- lib/spack/spack/concretize.py | 2 +- lib/spack/spack/config.py | 2 +- lib/spack/spack/database.py | 2 +- lib/spack/spack/directives.py | 2 +- lib/spack/spack/directory_layout.py | 2 +- lib/spack/spack/error.py | 2 +- lib/spack/spack/fetch_strategy.py | 2 +- lib/spack/spack/graph.py | 2 +- lib/spack/spack/hooks/__init__.py | 2 +- lib/spack/spack/hooks/dotkit.py | 2 +- lib/spack/spack/hooks/extensions.py | 2 +- lib/spack/spack/hooks/tclmodule.py | 2 +- lib/spack/spack/mirror.py | 2 +- lib/spack/spack/modules.py | 2 +- lib/spack/spack/multimethod.py | 2 +- lib/spack/spack/package.py | 2 +- lib/spack/spack/packages.py | 2 +- lib/spack/spack/parse.py | 2 +- lib/spack/spack/patch.py | 2 +- lib/spack/spack/spec.py | 2 +- lib/spack/spack/stage.py | 2 +- lib/spack/spack/test/__init__.py | 2 +- lib/spack/spack/test/cc.py | 2 +- lib/spack/spack/test/concretize.py | 2 +- lib/spack/spack/test/config.py | 2 +- lib/spack/spack/test/configure_guess.py | 2 +- lib/spack/spack/test/database.py | 2 +- lib/spack/spack/test/directory_layout.py | 2 +- lib/spack/spack/test/git_fetch.py | 2 +- lib/spack/spack/test/hg_fetch.py | 2 +- lib/spack/spack/test/install.py | 2 +- lib/spack/spack/test/link_tree.py | 2 +- lib/spack/spack/test/lock.py | 2 +- lib/spack/spack/test/make_executable.py | 2 +- lib/spack/spack/test/mirror.py | 2 +- lib/spack/spack/test/mock_packages_test.py | 2 +- lib/spack/spack/test/mock_repo.py | 2 +- lib/spack/spack/test/multimethod.py | 2 +- lib/spack/spack/test/optional_deps.py | 2 +- lib/spack/spack/test/package_sanity.py | 2 +- lib/spack/spack/test/packages.py | 2 +- lib/spack/spack/test/python_version.py | 2 +- lib/spack/spack/test/spec_dag.py | 2 +- lib/spack/spack/test/spec_semantics.py | 2 +- lib/spack/spack/test/spec_syntax.py | 2 +- lib/spack/spack/test/spec_yaml.py | 2 +- lib/spack/spack/test/stage.py | 2 +- lib/spack/spack/test/svn_fetch.py | 2 +- lib/spack/spack/test/unit_install.py | 2 +- lib/spack/spack/test/url_extrapolate.py | 2 +- lib/spack/spack/test/url_parse.py | 2 +- lib/spack/spack/test/url_substitution.py | 2 +- lib/spack/spack/test/versions.py | 2 +- lib/spack/spack/url.py | 4 ++-- lib/spack/spack/util/__init__.py | 2 +- lib/spack/spack/util/compression.py | 2 +- lib/spack/spack/util/crypto.py | 2 +- lib/spack/spack/util/debug.py | 2 +- lib/spack/spack/util/environment.py | 2 +- lib/spack/spack/util/executable.py | 2 +- lib/spack/spack/util/multiproc.py | 2 +- lib/spack/spack/util/prefix.py | 2 +- lib/spack/spack/util/string.py | 2 +- lib/spack/spack/util/web.py | 2 +- lib/spack/spack/variant.py | 2 +- lib/spack/spack/version.py | 2 +- lib/spack/spack/virtual.py | 2 +- share/spack/setup-env.csh | 2 +- share/spack/setup-env.sh | 2 +- var/spack/mock_packages/callpath/package.py | 2 +- var/spack/mock_packages/direct_mpich/package.py | 2 +- var/spack/mock_packages/dyninst/package.py | 2 +- var/spack/mock_packages/fake/package.py | 2 +- var/spack/mock_packages/indirect_mpich/package.py | 2 +- var/spack/mock_packages/libdwarf/package.py | 2 +- var/spack/mock_packages/libelf/package.py | 2 +- var/spack/mock_packages/mpich/package.py | 2 +- var/spack/mock_packages/mpich2/package.py | 2 +- var/spack/mock_packages/mpileaks/package.py | 2 +- var/spack/mock_packages/multimethod/package.py | 2 +- var/spack/mock_packages/netlib-blas/package.py | 2 +- var/spack/mock_packages/netlib-lapack/package.py | 2 +- var/spack/mock_packages/openblas/package.py | 2 +- .../mock_packages/trivial_install_test_package/package.py | 2 +- var/spack/mock_packages/zmpi/package.py | 2 +- var/spack/packages/Mitos/package.py | 8 ++++---- var/spack/packages/adept-utils/package.py | 6 +++--- var/spack/packages/automaded/package.py | 6 +++--- var/spack/packages/callpath/package.py | 6 +++--- var/spack/packages/clang/package.py | 2 +- var/spack/packages/cmake/package.py | 2 +- var/spack/packages/cram/package.py | 4 ++-- var/spack/packages/damselfly/package.py | 6 +++--- var/spack/packages/dyninst/package.py | 2 +- var/spack/packages/gcc/package.py | 2 +- var/spack/packages/gmp/package.py | 2 +- var/spack/packages/gperftools/package.py | 2 +- var/spack/packages/launchmon/package.py | 4 ++-- var/spack/packages/libNBC/package.py | 2 +- var/spack/packages/libdwarf/package.py | 2 +- var/spack/packages/libelf/package.py | 2 +- var/spack/packages/libmonitor/package.py | 2 +- var/spack/packages/libunwind/package.py | 2 +- var/spack/packages/llvm-lld/package.py | 2 +- var/spack/packages/llvm/package.py | 2 +- var/spack/packages/memaxes/package.py | 5 ++--- var/spack/packages/mpc/package.py | 2 +- var/spack/packages/mpfr/package.py | 2 +- var/spack/packages/mpich/package.py | 2 +- var/spack/packages/mpileaks/package.py | 2 +- var/spack/packages/muster/package.py | 4 ++-- var/spack/packages/netgauge/package.py | 2 +- var/spack/packages/pmgr_collective/package.py | 2 +- var/spack/packages/ravel/package.py | 4 ++-- var/spack/packages/scr/package.py | 2 +- var/spack/packages/spindle/package.py | 2 +- var/spack/packages/sqlite/package.py | 2 +- var/spack/packages/sundials/package.py | 2 +- var/spack/packages/swig/package.py | 2 +- 193 files changed, 214 insertions(+), 215 deletions(-) diff --git a/bin/spack b/bin/spack index e92d7cc273f..ec6a80ff024 100755 --- a/bin/spack +++ b/bin/spack @@ -7,7 +7,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/bin/spack-python b/bin/spack-python index 8a4b9c175d7..e0745e8c585 100755 --- a/bin/spack-python +++ b/bin/spack-python @@ -7,7 +7,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index bce9ef0e94f..e8e170d0a61 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/docs/exts/sphinxcontrib/__init__.py b/lib/spack/docs/exts/sphinxcontrib/__init__.py index 838d616eb46..298856746cb 100644 --- a/lib/spack/docs/exts/sphinxcontrib/__init__.py +++ b/lib/spack/docs/exts/sphinxcontrib/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/docs/exts/sphinxcontrib/programoutput.py b/lib/spack/docs/exts/sphinxcontrib/programoutput.py index ff006acf72c..f0fa045c865 100644 --- a/lib/spack/docs/exts/sphinxcontrib/programoutput.py +++ b/lib/spack/docs/exts/sphinxcontrib/programoutput.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index d958d9e74a9..67ca18e71ae 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -5,11 +5,11 @@ Download -------------------- Getting spack is easy. You can clone it from the `github repository -`_ using this command: +`_ using this command: .. code-block:: sh - $ git clone https://github.com/scalability-llnl/spack.git + $ git clone https://github.com/llnl/spack.git This will create a directory called ``spack``. We'll assume that the full path to this directory is in the ``SPACK_ROOT`` environment @@ -22,7 +22,7 @@ go: $ spack install libelf For a richer experience, use Spack's `shell support -`_: +`_: .. code-block:: sh diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 97c83614213..79757208c9a 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -24,12 +24,12 @@ maintain a single file for many different builds of the same package. See the :doc:`features` for examples and highlights. Get spack from the `github repository -`_ and install your first +`_ and install your first package: .. code-block:: sh - $ git clone https://github.com/scalability-llnl/spack.git + $ git clone https://github.com/llnl/spack.git $ cd spack/bin $ ./spack install libelf diff --git a/lib/spack/env/cc b/lib/spack/env/cc index fa85bb595ee..75a63f6fac3 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -7,7 +7,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/external/__init__.py b/lib/spack/external/__init__.py index 05780222102..7a89a1ac673 100644 --- a/lib/spack/external/__init__.py +++ b/lib/spack/external/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 03f25d3dff7..24cfbfde719 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 156ee34c9ef..cc87b7eac2b 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py index 583f077b794..6ae8aff75c9 100644 --- a/lib/spack/llnl/util/link_tree.py +++ b/lib/spack/llnl/util/link_tree.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index ac3684bd557..a7a9bf6b197 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index 48368543ff3..203f429a481 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index db928444c74..5545cf0311c 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/tty/color.py b/lib/spack/llnl/util/tty/color.py index 0d09303da09..167a99d3c23 100644 --- a/lib/spack/llnl/util/tty/color.py +++ b/lib/spack/llnl/util/tty/color.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index 5a52d45bc77..2819cd40df3 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 1ecf662178c..02eeed01fa1 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 6c874e30be0..2701fab90c9 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 6ce6fa0960c..926e7ac14ab 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/activate.py b/lib/spack/spack/cmd/activate.py index 1e44948d247..bcd01f2a286 100644 --- a/lib/spack/spack/cmd/activate.py +++ b/lib/spack/spack/cmd/activate.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/arch.py b/lib/spack/spack/cmd/arch.py index f0e88d1849b..db27544ffd1 100644 --- a/lib/spack/spack/cmd/arch.py +++ b/lib/spack/spack/cmd/arch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index f75b68b00ac..e4ec7da35db 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/cd.py b/lib/spack/spack/cmd/cd.py index 24d56db7d0b..16cbe6555a7 100644 --- a/lib/spack/spack/cmd/cd.py +++ b/lib/spack/spack/cmd/cd.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index c940d577811..6b7022a7a11 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index e303b3d6342..c3409887fb0 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 3173d110709..589ca87fb57 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/compilers.py b/lib/spack/spack/cmd/compilers.py index 8d046bfd7c5..c485a910ebd 100644 --- a/lib/spack/spack/cmd/compilers.py +++ b/lib/spack/spack/cmd/compilers.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index 78972a8be0c..a6e914131ee 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 05cf170e392..5e42860f3e7 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index 5a2b353fa2d..a0c78bf755b 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/dependents.py b/lib/spack/spack/cmd/dependents.py index 129a4eeb237..de76098d2fa 100644 --- a/lib/spack/spack/cmd/dependents.py +++ b/lib/spack/spack/cmd/dependents.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index d6bd1fbb79a..ebe8424f09e 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/doc.py b/lib/spack/spack/cmd/doc.py index 601ae26e5ed..29cadec94fb 100644 --- a/lib/spack/spack/cmd/doc.py +++ b/lib/spack/spack/cmd/doc.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index b8764ba391e..b168d967b93 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index ae8e95491ed..525e955a00c 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index a3db2da3945..c2cf288877c 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 1dd8703daf2..6f9e7ab5e21 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 05fa620c592..c7a376fd8da 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/graph.py b/lib/spack/spack/cmd/graph.py index 586a02c53b3..586a8523517 100644 --- a/lib/spack/spack/cmd/graph.py +++ b/lib/spack/spack/cmd/graph.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/help.py b/lib/spack/spack/cmd/help.py index eae3aabd972..841a0d5bcbe 100644 --- a/lib/spack/spack/cmd/help.py +++ b/lib/spack/spack/cmd/help.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index 085e4db44d8..8040e239368 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index e4338e222f6..5ee7bc01b71 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 0b55d9fb7bc..b51d5b429a6 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py index 5bc6b15784c..30d86c3b014 100644 --- a/lib/spack/spack/cmd/load.py +++ b/lib/spack/spack/cmd/load.py @@ -6,7 +6,7 @@ # Written by David Beckingsale, david@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 6fbab9782fa..e805cc40126 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index 8be0a7ad4c8..ef1e4f34756 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 4599944f1c2..4a1ce00b75d 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index c3daed64022..a5a9570eb52 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/package-list.py b/lib/spack/spack/cmd/package-list.py index f0484828455..eca9f918f10 100644 --- a/lib/spack/spack/cmd/package-list.py +++ b/lib/spack/spack/cmd/package-list.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -34,7 +34,7 @@ def github_url(pkg): """Link to a package file on github.""" - return ("https://github.com/scalability-llnl/spack/blob/master/var/spack/packages/%s/package.py" % + return ("https://github.com/llnl/spack/blob/master/var/spack/packages/%s/package.py" % pkg.name) diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index 2356583b074..8fc6f1383ed 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index ae5efd9d9c4..1ebc5ce3ee5 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/providers.py b/lib/spack/spack/cmd/providers.py index 1a652c82d17..0472f9bbe4c 100644 --- a/lib/spack/spack/cmd/providers.py +++ b/lib/spack/spack/cmd/providers.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/purge.py b/lib/spack/spack/cmd/purge.py index 9b969371495..d5d7513c466 100644 --- a/lib/spack/spack/cmd/purge.py +++ b/lib/spack/spack/cmd/purge.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/python.py b/lib/spack/spack/cmd/python.py index 15c45654c2f..e26b8d3e797 100644 --- a/lib/spack/spack/cmd/python.py +++ b/lib/spack/spack/cmd/python.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/reindex.py b/lib/spack/spack/cmd/reindex.py index c0008930c4b..2b30ef8814c 100644 --- a/lib/spack/spack/cmd/reindex.py +++ b/lib/spack/spack/cmd/reindex.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/restage.py b/lib/spack/spack/cmd/restage.py index 9230cf5a1a0..703ae30a04b 100644 --- a/lib/spack/spack/cmd/restage.py +++ b/lib/spack/spack/cmd/restage.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/spec.py b/lib/spack/spack/cmd/spec.py index af7ec1b36c9..43a106ea378 100644 --- a/lib/spack/spack/cmd/spec.py +++ b/lib/spack/spack/cmd/spec.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index 09cf0e1a1c2..7638cf31c47 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index ba0cb39baf2..e37554155f3 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index b1418ac2f1b..1669ec4cc98 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 93575e005dc..191d9d88e8d 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/unload.py b/lib/spack/spack/cmd/unload.py index 24e49b3f24a..cfb640ee6f5 100644 --- a/lib/spack/spack/cmd/unload.py +++ b/lib/spack/spack/cmd/unload.py @@ -6,7 +6,7 @@ # Written by David Beckingsale, david@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/unuse.py b/lib/spack/spack/cmd/unuse.py index 7f0b384ea05..06176a976bd 100644 --- a/lib/spack/spack/cmd/unuse.py +++ b/lib/spack/spack/cmd/unuse.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/urls.py b/lib/spack/spack/cmd/urls.py index 417ce3ab68b..a544b6153b6 100644 --- a/lib/spack/spack/cmd/urls.py +++ b/lib/spack/spack/cmd/urls.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/use.py b/lib/spack/spack/cmd/use.py index 4990fea2f82..c09695cfd3d 100644 --- a/lib/spack/spack/cmd/use.py +++ b/lib/spack/spack/cmd/use.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/versions.py b/lib/spack/spack/cmd/versions.py index ed16728261b..494f13d36d5 100644 --- a/lib/spack/spack/cmd/versions.py +++ b/lib/spack/spack/cmd/versions.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 1e800a89795..b9abf943e80 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index b7b021a1ac8..66e608cf79f 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 790901c86e4..b34d1f2f9c7 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index f0d27d590e8..2886888d578 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 2a72c4eaea0..12984299746 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index d97f24c12e8..6999eb50dea 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index 562186b865f..308f811eb4a 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -7,7 +7,7 @@ # Written by François Bissey, francois.bissey@canterbury.ac.nz, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 66002492cb8..d9419da7840 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 41afe8b2327..c127f6a28f8 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 908ffc7fa4c..d62a47547d2 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 78039ac6f95..2a818e8d0c7 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 918405cab69..056606b429a 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py index b3b24e6105b..0c2e7eb53ce 100644 --- a/lib/spack/spack/error.py +++ b/lib/spack/spack/error.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 5e6850d14b6..8c4bd4ed8a9 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/graph.py b/lib/spack/spack/graph.py index 5fb6a9cd234..e8c5cfb0808 100644 --- a/lib/spack/spack/graph.py +++ b/lib/spack/spack/graph.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/hooks/__init__.py b/lib/spack/spack/hooks/__init__.py index 1c44e8abaaa..2765f7be391 100644 --- a/lib/spack/spack/hooks/__init__.py +++ b/lib/spack/spack/hooks/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/hooks/dotkit.py b/lib/spack/spack/hooks/dotkit.py index 4e748ff80a4..91236373561 100644 --- a/lib/spack/spack/hooks/dotkit.py +++ b/lib/spack/spack/hooks/dotkit.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py index cf87a78c8c8..b4847d697f5 100644 --- a/lib/spack/spack/hooks/extensions.py +++ b/lib/spack/spack/hooks/extensions.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/hooks/tclmodule.py b/lib/spack/spack/hooks/tclmodule.py index 0b9fd5a67c8..8b315f27a27 100644 --- a/lib/spack/spack/hooks/tclmodule.py +++ b/lib/spack/spack/hooks/tclmodule.py @@ -6,7 +6,7 @@ # Written by David Beckingsale, david@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 306c8085aa5..ee0bf6de11f 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 56a61adefb9..59ed9f893ec 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index 892619c6ac0..df9b9b2ab14 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c631a35bf30..daba5cd352d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 2e3e95ca407..f6f4cbf0254 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/parse.py b/lib/spack/spack/parse.py index bc12ec258c2..e9467aa685b 100644 --- a/lib/spack/spack/parse.py +++ b/lib/spack/spack/parse.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index b1b6e077385..42f2105f52c 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index a10edcc6fce..f62182ce76b 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 78930ecb5b2..754344fc016 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 0f776bfea46..13cb1d2b789 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index aa16f9b3516..4188b8d5505 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index b3a77d076a1..2f8e0c7ec0e 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 790b22f3b0a..ed11e34c691 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index 766dd51d52b..a4e8565b62b 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 8416143f2d2..c07d32686e4 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index b3ad8efec4b..703ac1b8672 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 9700bd75338..244680b5d05 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 531dfabaa18..f8c6571bda9 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 5659e97a4d4..1ef4171fb2f 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index 9e887ecc49e..886b7ef4c5d 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py index 5664e71b037..bc68df01dbe 100644 --- a/lib/spack/spack/test/lock.py +++ b/lib/spack/spack/test/lock.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index c4bfeb2a034..09efec85805 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 89ab14359ef..189a85fb1a8 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 00f81114af3..e4e1b21b53e 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index fd184e64bc9..c454b1f106c 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index cd5d9e625e3..d8d61d14c8c 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index fbee0cfa8f0..ebd72819992 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index 6222e7b5f8b..370cf676ef3 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index a8183cf6a65..b2daea7b7b2 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index ba7bab6f4b8..ba570b416f8 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 549f829d3eb..94438b6a0c9 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 64220e58933..1381556aad5 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 404f38906e8..1daaa4be8fc 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/spec_yaml.py b/lib/spack/spack/test/spec_yaml.py index 869befc02a6..11987ea1b3d 100644 --- a/lib/spack/spack/test/spec_yaml.py +++ b/lib/spack/spack/test/spec_yaml.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index 8cff8f79607..c1b2a2a5730 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 184fe8faa10..9229af76d49 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index c4b9092f051..41c76a6dfae 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py index 00d8216020a..87adf894017 100644 --- a/lib/spack/spack/test/url_extrapolate.py +++ b/lib/spack/spack/test/url_extrapolate.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index ae1d559f7c0..efde7c0c73b 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index db7ddd251de..8b90ee086ae 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 20e946e90ef..108450e098e 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 6adbfe156dd..02c0b83e260 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -70,7 +70,7 @@ def find_list_url(url): """ url_types = [ - # e.g. https://github.com/scalability-llnl/callpath/archive/v1.0.1.tar.gz + # e.g. https://github.com/llnl/callpath/archive/v1.0.1.tar.gz (r'^(https://github.com/[^/]+/[^/]+)/archive/', lambda m: m.group(1) + '/releases') ] diff --git a/lib/spack/spack/util/__init__.py b/lib/spack/spack/util/__init__.py index 1c388f31c5b..b54691b67cc 100644 --- a/lib/spack/spack/util/__init__.py +++ b/lib/spack/spack/util/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py index fd17785ad0b..ea1f233bce8 100644 --- a/lib/spack/spack/util/compression.py +++ b/lib/spack/spack/util/compression.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/crypto.py b/lib/spack/spack/util/crypto.py index 8a8574cd3d4..52692602849 100644 --- a/lib/spack/spack/util/crypto.py +++ b/lib/spack/spack/util/crypto.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/debug.py b/lib/spack/spack/util/debug.py index 37985eccddb..7930753f6f4 100644 --- a/lib/spack/spack/util/debug.py +++ b/lib/spack/spack/util/debug.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index 7a4ff919ad6..cd413dcfbc8 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index d1dfb62ffb6..4f9958062b1 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/multiproc.py b/lib/spack/spack/util/multiproc.py index 21cd6f543d0..8ca82df0110 100644 --- a/lib/spack/spack/util/multiproc.py +++ b/lib/spack/spack/util/multiproc.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/prefix.py b/lib/spack/spack/util/prefix.py index 7bd63c16ca9..c613ca51821 100644 --- a/lib/spack/spack/util/prefix.py +++ b/lib/spack/spack/util/prefix.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/string.py b/lib/spack/spack/util/string.py index 234163bf52c..1556ce6d29c 100644 --- a/lib/spack/spack/util/string.py +++ b/lib/spack/spack/util/string.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 1420d62a77b..94384e9c86f 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/variant.py b/lib/spack/spack/variant.py index 3d3e2b0f6d5..8959e76684b 100644 --- a/lib/spack/spack/variant.py +++ b/lib/spack/spack/variant.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index ffce2d1ff89..e8a0a261c96 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/virtual.py b/lib/spack/spack/virtual.py index f92cc4509c5..d16aea8642b 100644 --- a/lib/spack/spack/virtual.py +++ b/lib/spack/spack/virtual.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/share/spack/setup-env.csh b/share/spack/setup-env.csh index 5f91670a60d..42d8c42726d 100755 --- a/share/spack/setup-env.csh +++ b/share/spack/setup-env.csh @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index b90846d28fc..c96a94195d1 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/callpath/package.py b/var/spack/mock_packages/callpath/package.py index 5b6b70ba2a5..abc576f78fe 100644 --- a/var/spack/mock_packages/callpath/package.py +++ b/var/spack/mock_packages/callpath/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/direct_mpich/package.py b/var/spack/mock_packages/direct_mpich/package.py index 2ced82521b3..efe7fc2afc5 100644 --- a/var/spack/mock_packages/direct_mpich/package.py +++ b/var/spack/mock_packages/direct_mpich/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/dyninst/package.py b/var/spack/mock_packages/dyninst/package.py index 7998578da1c..ea57950865f 100644 --- a/var/spack/mock_packages/dyninst/package.py +++ b/var/spack/mock_packages/dyninst/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/fake/package.py b/var/spack/mock_packages/fake/package.py index fb3c2bdd2e7..5f81ef20fc7 100644 --- a/var/spack/mock_packages/fake/package.py +++ b/var/spack/mock_packages/fake/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/indirect_mpich/package.py b/var/spack/mock_packages/indirect_mpich/package.py index daf8b4b166e..0b1773a27be 100644 --- a/var/spack/mock_packages/indirect_mpich/package.py +++ b/var/spack/mock_packages/indirect_mpich/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/libdwarf/package.py b/var/spack/mock_packages/libdwarf/package.py index 0b8df04cfb2..e486a5de03e 100644 --- a/var/spack/mock_packages/libdwarf/package.py +++ b/var/spack/mock_packages/libdwarf/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/libelf/package.py b/var/spack/mock_packages/libelf/package.py index 94c8f942cdb..5e5b0b71437 100644 --- a/var/spack/mock_packages/libelf/package.py +++ b/var/spack/mock_packages/libelf/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/mpich/package.py b/var/spack/mock_packages/mpich/package.py index f77d3efc5d1..55bf97f2cfc 100644 --- a/var/spack/mock_packages/mpich/package.py +++ b/var/spack/mock_packages/mpich/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/mpich2/package.py b/var/spack/mock_packages/mpich2/package.py index 827b94c8a4a..90f969b8986 100644 --- a/var/spack/mock_packages/mpich2/package.py +++ b/var/spack/mock_packages/mpich2/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/mpileaks/package.py b/var/spack/mock_packages/mpileaks/package.py index 3989f1b452c..9a18c5e1f2d 100644 --- a/var/spack/mock_packages/mpileaks/package.py +++ b/var/spack/mock_packages/mpileaks/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index 75b1606ffc6..ea103fe1754 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/netlib-blas/package.py b/var/spack/mock_packages/netlib-blas/package.py index 199327812e1..39f2c92ae54 100644 --- a/var/spack/mock_packages/netlib-blas/package.py +++ b/var/spack/mock_packages/netlib-blas/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/netlib-lapack/package.py b/var/spack/mock_packages/netlib-lapack/package.py index 8f7f236f1b4..331844e544e 100644 --- a/var/spack/mock_packages/netlib-lapack/package.py +++ b/var/spack/mock_packages/netlib-lapack/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/openblas/package.py b/var/spack/mock_packages/openblas/package.py index 9c2fb305735..c7771b92a30 100644 --- a/var/spack/mock_packages/openblas/package.py +++ b/var/spack/mock_packages/openblas/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/trivial_install_test_package/package.py b/var/spack/mock_packages/trivial_install_test_package/package.py index c4db9f5f071..fec5849e672 100644 --- a/var/spack/mock_packages/trivial_install_test_package/package.py +++ b/var/spack/mock_packages/trivial_install_test_package/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/zmpi/package.py b/var/spack/mock_packages/zmpi/package.py index 8c6ceda6d3f..201fac2fbf7 100644 --- a/var/spack/mock_packages/zmpi/package.py +++ b/var/spack/mock_packages/zmpi/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/Mitos/package.py b/var/spack/packages/Mitos/package.py index 0377887943e..ea131872ddb 100644 --- a/var/spack/packages/Mitos/package.py +++ b/var/spack/packages/Mitos/package.py @@ -4,15 +4,15 @@ class Mitos(Package): """Mitos is a library and a tool for collecting sampled memory performance data to view with MemAxes""" - homepage = "https://github.com/scalability-llnl/Mitos" - url = "https://github.com/scalability-llnl/Mitos" + homepage = "https://github.com/llnl/Mitos" + url = "https://github.com/llnl/Mitos" version('0.9.2', - git='https://github.com/scalability-llnl/Mitos.git', + git='https://github.com/llnl/Mitos.git', commit='8cb143a2e8c00353ff531a781a9ca0992b0aaa3d') version('0.9.1', - git='https://github.com/scalability-llnl/Mitos.git', + git='https://github.com/llnl/Mitos.git', tag='v0.9.1') depends_on('dyninst@8.2.1:') diff --git a/var/spack/packages/adept-utils/package.py b/var/spack/packages/adept-utils/package.py index e4a2e1523fe..fb59576c21b 100644 --- a/var/spack/packages/adept-utils/package.py +++ b/var/spack/packages/adept-utils/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -27,8 +27,8 @@ class AdeptUtils(Package): """Utility libraries for LLNL performance tools.""" - homepage = "https://github.com/scalability-llnl/adept-utils" - url = "https://github.com/scalability-llnl/adept-utils/archive/v1.0.tar.gz" + homepage = "https://github.com/llnl/adept-utils" + url = "https://github.com/llnl/adept-utils/archive/v1.0.tar.gz" version('1.0.1', '731a310717adcb004d9d195130efee7d') version('1.0', '5c6cd9badce56c945ac8551e34804397') diff --git a/var/spack/packages/automaded/package.py b/var/spack/packages/automaded/package.py index 9fbd93e3b32..e0bc7efb8bb 100644 --- a/var/spack/packages/automaded/package.py +++ b/var/spack/packages/automaded/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -36,8 +36,8 @@ class Automaded(Package): finding the process (or group of processes) that caused the hang. """ - homepage = "https://github.com/scalability-llnl/AutomaDeD" - url = "https://github.com/scalability-llnl/AutomaDeD/archive/v1.0.tar.gz" + homepage = "https://github.com/llnl/AutomaDeD" + url = "https://github.com/llnl/AutomaDeD/archive/v1.0.tar.gz" version('1.0', '16a3d4def2c4c77d0bc4b21de8b3ab03') diff --git a/var/spack/packages/callpath/package.py b/var/spack/packages/callpath/package.py index f8a1eab9f7b..3d2d96249ee 100644 --- a/var/spack/packages/callpath/package.py +++ b/var/spack/packages/callpath/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -28,8 +28,8 @@ class Callpath(Package): """Library for representing callpaths consistently in distributed-memory performance tools.""" - homepage = "https://github.com/scalability-llnl/callpath" - url = "https://github.com/scalability-llnl/callpath/archive/v1.0.1.tar.gz" + homepage = "https://github.com/llnl/callpath" + url = "https://github.com/llnl/callpath/archive/v1.0.1.tar.gz" version('1.0.2', 'b1994d5ee7c7db9d27586fc2dcf8f373') version('1.0.1', '0047983d2a52c5c335f8ba7f5bab2325') diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py index 4f977bf9a44..20a5ac2c940 100644 --- a/var/spack/packages/clang/package.py +++ b/var/spack/packages/clang/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/cmake/package.py b/var/spack/packages/cmake/package.py index fcfbca07056..c24a80748c2 100644 --- a/var/spack/packages/cmake/package.py +++ b/var/spack/packages/cmake/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/cram/package.py b/var/spack/packages/cram/package.py index 1f1722993f9..b19422b8a89 100644 --- a/var/spack/packages/cram/package.py +++ b/var/spack/packages/cram/package.py @@ -2,8 +2,8 @@ class Cram(Package): """Cram runs many small MPI jobs inside one large MPI job.""" - homepage = "https://github.com/scalability-llnl/cram" - url = "http://github.com/scalability-llnl/cram/archive/v1.0.1.tar.gz" + homepage = "https://github.com/llnl/cram" + url = "http://github.com/llnl/cram/archive/v1.0.1.tar.gz" version('1.0.1', 'c73711e945cf5dc603e44395f6647f5e') diff --git a/var/spack/packages/damselfly/package.py b/var/spack/packages/damselfly/package.py index 54df528c4a7..96666d1abe7 100644 --- a/var/spack/packages/damselfly/package.py +++ b/var/spack/packages/damselfly/package.py @@ -2,10 +2,10 @@ class Damselfly(Package): """Damselfly is a model-based parallel network simulator.""" - homepage = "https://github.com/scalability-llnl/damselfly" - url = "https://github.com/scalability-llnl/damselfly" + homepage = "https://github.com/llnl/damselfly" + url = "https://github.com/llnl/damselfly" - version('1.0', '05cf7e2d8ece4408c0f2abb7ab63fd74c0d62895', git='https://github.com/scalability-llnl/damselfly.git', tag='v1.0') + version('1.0', '05cf7e2d8ece4408c0f2abb7ab63fd74c0d62895', git='https://github.com/llnl/damselfly.git', tag='v1.0') def install(self, spec, prefix): with working_dir('spack-build', create=True): diff --git a/var/spack/packages/dyninst/package.py b/var/spack/packages/dyninst/package.py index 41ec57dd2f0..7f8598e0e5f 100644 --- a/var/spack/packages/dyninst/package.py +++ b/var/spack/packages/dyninst/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py index b5dc585be79..7ec160d5957 100644 --- a/var/spack/packages/gcc/package.py +++ b/var/spack/packages/gcc/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/gmp/package.py b/var/spack/packages/gmp/package.py index 6cd501dbf20..fe13de3b95e 100644 --- a/var/spack/packages/gmp/package.py +++ b/var/spack/packages/gmp/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/gperftools/package.py b/var/spack/packages/gperftools/package.py index 89004623244..0ba44c93296 100644 --- a/var/spack/packages/gperftools/package.py +++ b/var/spack/packages/gperftools/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/launchmon/package.py b/var/spack/packages/launchmon/package.py index 6fbe6a68d0e..f97384a249f 100644 --- a/var/spack/packages/launchmon/package.py +++ b/var/spack/packages/launchmon/package.py @@ -6,7 +6,7 @@ # Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -31,7 +31,7 @@ class Launchmon(Package): url = "http://downloads.sourceforge.net/project/launchmon/launchmon/1.0.1%20release/launchmon-1.0.1.tar.gz" version('1.0.1', '2f12465803409fd07f91174a4389eb2b') - version('1.0.1-2', git='https://github.com/scalability-llnl/launchmon.git', commit='ff7e22424b8f375318951eb1c9282fcbbfa8aadf') + version('1.0.1-2', git='https://github.com/llnl/launchmon.git', commit='ff7e22424b8f375318951eb1c9282fcbbfa8aadf') depends_on('autoconf') depends_on('automake') diff --git a/var/spack/packages/libNBC/package.py b/var/spack/packages/libNBC/package.py index 6d08f3219c2..550568e97da 100644 --- a/var/spack/packages/libNBC/package.py +++ b/var/spack/packages/libNBC/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/libdwarf/package.py b/var/spack/packages/libdwarf/package.py index 099a974e934..addb557519b 100644 --- a/var/spack/packages/libdwarf/package.py +++ b/var/spack/packages/libdwarf/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/libelf/package.py b/var/spack/packages/libelf/package.py index 9338b8f393e..29bc21b65c4 100644 --- a/var/spack/packages/libelf/package.py +++ b/var/spack/packages/libelf/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/libmonitor/package.py b/var/spack/packages/libmonitor/package.py index 3b95b86ddf5..c75e1a79470 100644 --- a/var/spack/packages/libmonitor/package.py +++ b/var/spack/packages/libmonitor/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/libunwind/package.py b/var/spack/packages/libunwind/package.py index 239fcbcfd55..6f162f7b088 100644 --- a/var/spack/packages/libunwind/package.py +++ b/var/spack/packages/libunwind/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/llvm-lld/package.py b/var/spack/packages/llvm-lld/package.py index f2292113962..cb91aa22a51 100644 --- a/var/spack/packages/llvm-lld/package.py +++ b/var/spack/packages/llvm-lld/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index 80c44e4d5fa..b3ca488809a 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -6,7 +6,7 @@ # Written by David Beckingsale, david@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/memaxes/package.py b/var/spack/packages/memaxes/package.py index 76d5d3f8310..4b1da558a2b 100644 --- a/var/spack/packages/memaxes/package.py +++ b/var/spack/packages/memaxes/package.py @@ -3,10 +3,10 @@ class Memaxes(Package): """MemAxes is a visualizer for sampled memory trace data.""" - homepage = "https://github.com/scalability-llnl/MemAxes" + homepage = "https://github.com/llnl/MemAxes" version('0.5', '5874f3fda9fd2d313c0ff9684f915ab5', - url='https://github.com/scalability-llnl/MemAxes/archive/v0.5.tar.gz') + url='https://github.com/llnl/MemAxes/archive/v0.5.tar.gz') depends_on("cmake@2.8.9:") depends_on("qt@5:") @@ -16,4 +16,3 @@ def install(self, spec, prefix): cmake('..', *std_cmake_args) make() make("install") - diff --git a/var/spack/packages/mpc/package.py b/var/spack/packages/mpc/package.py index 33706f043f2..50477a0ccba 100644 --- a/var/spack/packages/mpc/package.py +++ b/var/spack/packages/mpc/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/mpfr/package.py b/var/spack/packages/mpfr/package.py index 9c744a22df4..0f2baac0041 100644 --- a/var/spack/packages/mpfr/package.py +++ b/var/spack/packages/mpfr/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py index d48bf878f65..e018a082011 100644 --- a/var/spack/packages/mpich/package.py +++ b/var/spack/packages/mpich/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/mpileaks/package.py b/var/spack/packages/mpileaks/package.py index 4ef866588ce..661d9d66bf3 100644 --- a/var/spack/packages/mpileaks/package.py +++ b/var/spack/packages/mpileaks/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/muster/package.py b/var/spack/packages/muster/package.py index 722daf3d7f2..0dc2e5e0865 100644 --- a/var/spack/packages/muster/package.py +++ b/var/spack/packages/muster/package.py @@ -7,8 +7,8 @@ class Muster(Package): for performance data analysis on systems with very large numbers of processes. """ - homepage = "https://github.com/scalability-llnl/muster" - url = "https://github.com/scalability-llnl/muster/archive/v1.0.tar.gz" + homepage = "https://github.com/llnl/muster" + url = "https://github.com/llnl/muster/archive/v1.0.tar.gz" version('1.0.1', 'd709787db7e080447afb6571ac17723c') version('1.0', '2eec6979a4a36d3a65a792d12969be16') diff --git a/var/spack/packages/netgauge/package.py b/var/spack/packages/netgauge/package.py index c2378b07181..0ea42175c69 100644 --- a/var/spack/packages/netgauge/package.py +++ b/var/spack/packages/netgauge/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/pmgr_collective/package.py b/var/spack/packages/pmgr_collective/package.py index 5d9b02acc31..1fc47c658f6 100644 --- a/var/spack/packages/pmgr_collective/package.py +++ b/var/spack/packages/pmgr_collective/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/ravel/package.py b/var/spack/packages/ravel/package.py index 01fa941cfef..d774a0ab869 100644 --- a/var/spack/packages/ravel/package.py +++ b/var/spack/packages/ravel/package.py @@ -4,8 +4,8 @@ class Ravel(Package): """Ravel is a parallel communication trace visualization tool that orders events according to logical time.""" - homepage = "https://github.com/scalability-llnl/ravel" - url = 'https://github.com/scalability-llnl/ravel/archive/v1.0.0.tar.gz' + homepage = "https://github.com/llnl/ravel" + url = 'https://github.com/llnl/ravel/archive/v1.0.0.tar.gz' version('1.0.0', 'b25fece58331c2adfcce76c5036485c2') diff --git a/var/spack/packages/scr/package.py b/var/spack/packages/scr/package.py index 9fb758f0722..1408dce6780 100644 --- a/var/spack/packages/scr/package.py +++ b/var/spack/packages/scr/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/spindle/package.py b/var/spack/packages/spindle/package.py index 06a1e142847..a20753458af 100644 --- a/var/spack/packages/spindle/package.py +++ b/var/spack/packages/spindle/package.py @@ -6,7 +6,7 @@ # Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/sqlite/package.py b/var/spack/packages/sqlite/package.py index 734b0b6cb62..1cf2d302397 100644 --- a/var/spack/packages/sqlite/package.py +++ b/var/spack/packages/sqlite/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/sundials/package.py b/var/spack/packages/sundials/package.py index 8b784c8c3c1..7e025a8244d 100644 --- a/var/spack/packages/sundials/package.py +++ b/var/spack/packages/sundials/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/swig/package.py b/var/spack/packages/swig/package.py index ee536d7063d..8d46c4fe465 100644 --- a/var/spack/packages/swig/package.py +++ b/var/spack/packages/swig/package.py @@ -6,7 +6,7 @@ # Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify From e009a910ccbd111ae210e0f65aef2db964ed3916 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Dec 2015 01:28:54 -0800 Subject: [PATCH 139/910] Fix travis badge URL. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ffd118ded81..bdce345764f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ![image](share/spack/logo/spack-logo-text-64.png "Spack") ============ -[![Build Status](https://travis-ci.org/llnl/spack.png?branch=develop)](https://travis-ci.org/llnl/spack) +[![Build Status](https://travis-ci.org/LLNL/spack.png?branch=develop)](https://travis-ci.org/LLNL/spack) Spack is a package management tool designed to support multiple versions and configurations of software on a wide variety of platforms From 537451f0b1ea057e723daa9bb81bc950730583cb Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Dec 2015 01:32:12 -0800 Subject: [PATCH 140/910] bugfix: doc build needed additional sys.path due to externals change. --- lib/spack/docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index e8e170d0a61..3d2a8251aa7 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -43,6 +43,7 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('exts')) +sys.path.insert(0, os.path.abspath('../external')) # Add the Spack bin directory to the path so that we can use its output in docs. spack_root = '../../..' From 31863c68d8d3010dc28945f35e323d4541962788 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Tue, 8 Dec 2015 15:18:10 +0100 Subject: [PATCH 141/910] Adding curl and expat support to git for https repos --- var/spack/packages/curl/package.py | 25 +++++++++++++++++ var/spack/packages/expat/package.py | 17 ++++++++++++ var/spack/packages/git/package.py | 42 +++++++++++++++++++++++++---- 3 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 var/spack/packages/curl/package.py create mode 100644 var/spack/packages/expat/package.py diff --git a/var/spack/packages/curl/package.py b/var/spack/packages/curl/package.py new file mode 100644 index 00000000000..9e684445c71 --- /dev/null +++ b/var/spack/packages/curl/package.py @@ -0,0 +1,25 @@ +from spack import * + +class Curl(Package): + """cURL is an open source command line tool and library for + transferring data with URL syntax""" + + homepage = "http://curl.haxx.se" + url = "http://curl.haxx.se/download/curl-7.46.0.tar.bz2" + + version('7.46.0', '9979f989a2a9930d10f1b3deeabc2148') + version('7.45.0', '62c1a352b28558f25ba6209214beadc8') + version('7.44.0', '6b952ca00e5473b16a11f05f06aa8dae') + version('7.43.0', '11bddbb452a8b766b932f859aaeeed39') + version('7.42.1', '296945012ce647b94083ed427c1877a8') + + depends_on("openssl") + depends_on("zlib") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, + '--with-zlib=%s' % spec['zlib'].prefix, + '--with-ssl=%s' % spec['openssl'].prefix) + + make() + make("install") diff --git a/var/spack/packages/expat/package.py b/var/spack/packages/expat/package.py new file mode 100644 index 00000000000..082da5bf0be --- /dev/null +++ b/var/spack/packages/expat/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Expat(Package): + """ is an XML parser library written in C""" + homepage = "http://expat.sourceforge.net/" + url = "http://downloads.sourceforge.net/project/expat/expat/2.1.0/expat-2.1.0.tar.gz" + + version('2.1.0', 'dd7dab7a5fea97d2a6a43f511449b7cd') + + + def install(self, spec, prefix): + + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + make() + make('install') + diff --git a/var/spack/packages/git/package.py b/var/spack/packages/git/package.py index 0f1a3ba05bf..28c7aa8161f 100644 --- a/var/spack/packages/git/package.py +++ b/var/spack/packages/git/package.py @@ -7,10 +7,25 @@ class Git(Package): homepage = "http://git-scm.com" url = "https://www.kernel.org/pub/software/scm/git/git-2.2.1.tar.xz" + version('2.6.3', '5a6375349c3f13c8dbbabfc327bae429') + version('2.6.2', '32ae5ad29763fc927bfcaeab55385fd9') + version('2.6.1', 'dd4a3a7fe96598c553edd39d40c9c290') + version('2.6.0', '6b7d43d615fb3f0dfecf4d131e23f438') + version('2.5.4', 'ec118fcd1cf984edc17eb6588b78e81b') version('2.2.1', '43e01f9d96ba8c11611e0eef0d9f9f28') - # Use system openssl. - # depends_on("openssl") + + # Git compiles with curl support by default on but if your system + # does not have it you will not be able to clone https repos + variant("curl", default=False, description="Add the internal support of curl for https clone") + + # Git compiles with expat support by default on but if your system + # does not have it you will not be able to push https repos + variant("expat", default=False, description="Add the internal support of expat for https push") + + depends_on("openssl") + depends_on("curl", when="+curl") + depends_on("expat", when="+expat") # Use system perl for now. # depends_on("perl") @@ -19,9 +34,26 @@ class Git(Package): depends_on("zlib") def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--without-pcre", - "--without-python") + configure_args = [ + "--prefix=%s" % prefix, + "--without-pcre", + "--with-openssl=%s" % spec['openssl'].prefix, + "--with-zlib=%s" % spec['zlib'].prefix + ] + if '+curl' in spec: + configure_args.append("--with-curl=%s" % spec['curl'].prefix) + + if '+expat' in spec: + configure_args.append("--with-expat=%s" % spec['expat'].prefix) + + configure(*configure_args) make() make("install") + + + + + + + From 20e67bc5e6c4a4ac759bb068ff78c13bfc17fb0f Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 9 Dec 2015 13:06:39 +0100 Subject: [PATCH 142/910] clang : solve the issue with missing default include paths for OpenMP and libc++ resource : support for finer grained linking of resources --- lib/spack/spack/directives.py | 17 ++++++++-------- lib/spack/spack/package.py | 16 ++++++++++----- lib/spack/spack/resource.py | 6 +++--- var/spack/packages/clang/package.py | 31 +++++++++++++++++++++++++++++ var/spack/packages/llvm/package.py | 4 ++-- 5 files changed, 56 insertions(+), 18 deletions(-) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 07ef32294b0..22b262b57c4 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -269,19 +269,20 @@ def variant(pkg, name, default=False, description=""): def resource(pkg, **kwargs): """ Define an external resource to be fetched and staged when building the package. Based on the keywords present in the - dictionary the appropriate FetchStrategy will be used for the resource. + dictionary the appropriate FetchStrategy will be used for the resource. Resources are fetched and staged in their + own folder inside spack stage area, and then linked into the stage area of the package that needs them. List of recognized keywords: - * 'when' : represents the condition upon which the resource is needed (optional) - * 'destination' : path where to extract / checkout the resource (optional). This path must be a relative path, - and it must fall inside the stage area of the main package. - * 'basename' : basename of the resource source folder within destination (optional). - + * 'when' : (optional) represents the condition upon which the resource is needed + * 'destination' : (optional) path where to link the resource. This path must be relative to the main package stage + area. + * 'placement' : (optional) gives the possibility to fine tune how the resource is linked into the main package stage + area. """ when = kwargs.get('when', pkg.name) destination = kwargs.get('destination', "") - basename = kwargs.get('basename', None) + placement = kwargs.get('placement', None) # Check if the path is relative if os.path.isabs(destination): message = "The destination keyword of a resource directive can't be an absolute path.\n" @@ -298,7 +299,7 @@ def resource(pkg, **kwargs): resources = pkg.resources.setdefault(when_spec, []) fetcher = from_kwargs(**kwargs) name = kwargs.get('name') - resources.append(Resource(name, fetcher, destination, basename)) + resources.append(Resource(name, fetcher, destination, placement)) class DirectiveError(spack.error.SpackError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index dcb514af007..b386f8f6a87 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -683,11 +683,17 @@ def _expand_archive(stage, name=self.name): for resource in resources: stage = resource.fetcher.stage _expand_archive(stage, resource.name) - basename = os.path.basename(stage.source_path) if resource.basename is None else resource.basename - link_path = join_path(self.stage.source_path, resource.destination, basename) - if not os.path.exists(link_path): - # Create a symlink - os.symlink(stage.source_path, link_path) + # Turn placement into a dict with relative paths + placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement + if not isinstance(placement, dict): + placement = {'': placement} + # Make the paths in the dictionary absolute and link + for key, value in placement.iteritems(): + link_path = join_path(self.stage.source_path, resource.destination, value) + source_path = join_path(stage.source_path, key) + if not os.path.exists(link_path): + # Create a symlink + os.symlink(source_path, link_path) ########## self.stage.chdir_to_source() diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py index b20612686d9..8d081b45c96 100644 --- a/lib/spack/spack/resource.py +++ b/lib/spack/spack/resource.py @@ -30,10 +30,10 @@ class Resource(object): """ - Represents an optional resource. Aggregates a name, a fetcher and a destination. + Represents an optional resource. Aggregates a name, a fetcher, a destination and a placement """ - def __init__(self, name, fetcher, destination, basename): + def __init__(self, name, fetcher, destination, placement): self.name = name self.fetcher = fetcher self.destination = destination - self.basename = basename + self.placement = placement diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py index eac1863b975..ca368b3074d 100644 --- a/var/spack/packages/clang/package.py +++ b/var/spack/packages/clang/package.py @@ -22,8 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## + + from spack import * +import os +import os.path + class Clang(Package): """The goal of the Clang project is to create a new C, C++, Objective C and Objective C++ front-end for the LLVM compiler. @@ -62,3 +67,29 @@ def install(self, spec, prefix): *options) make() make("install") + # CLang doesn't look in llvm folders for system headers... + self.link_llvm_directories(spec) + + def link_llvm_directories(self, spec): + + def clang_include_dir_at(root): + return join_path(root, 'include') + + def clang_lib_dir_at(root): + return join_path(root, 'lib/clang/', str(self.version), 'include') + + def do_link(source_dir, destination_dir): + if os.path.exists(source_dir): + for name in os.listdir(source_dir): + source = join_path(source_dir, name) + link = join_path(destination_dir, name) + os.symlink(source, link) + + # Link folder and files in include + llvm_dir = clang_include_dir_at(spec['llvm'].prefix) + clang_dir = clang_include_dir_at(self.prefix) + do_link(llvm_dir, clang_dir) + # Link folder and files in lib + llvm_dir = clang_lib_dir_at(spec['llvm'].prefix) + clang_dir = clang_lib_dir_at(self.prefix) + do_link(llvm_dir, clang_dir) \ No newline at end of file diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index 872a6c082b5..d7ae3390be9 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -53,10 +53,10 @@ class Llvm(Package): destination='projects', when='@3.7.0') resource(name='libcxx', url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd', - destination='projects', basename='libcxx', when='+libcxx@3.7.0') + destination='projects', placement='libcxx', when='+libcxx@3.7.0') resource(name='libcxxabi', url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772', - destination='projects', basename='libcxxabi', when='+libcxx@3.7.0') + destination='projects', placement='libcxxabi', when='+libcxx@3.7.0') ########## def install(self, spec, prefix): From 0daa08416a0209a71b131fb9c8b4d58fa222474f Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 9 Dec 2015 20:00:54 +0100 Subject: [PATCH 143/910] corrected filter_compilers + added it for mvapich2 --- var/spack/packages/mpich/package.py | 13 +++++++---- var/spack/packages/mvapich2/package.py | 30 +++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py index e018a082011..7cfa0a3b619 100644 --- a/var/spack/packages/mpich/package.py +++ b/var/spack/packages/mpich/package.py @@ -85,8 +85,13 @@ def filter_compilers(self): mpif77 = os.path.join(bin, 'mpif77') mpif90 = os.path.join(bin, 'mpif90') + spack_cc = os.environ['CC'] + spack_cxx = os.environ['CXX'] + spack_f77 = os.environ['F77'] + spack_fc = os.environ['FC'] + kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : True } - filter_file('CC="cc"', 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) - filter_file('CXX="c++"', 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) - filter_file('F77="f77"', 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) - filter_file('FC="f90"', 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + filter_file('CC="%s"' % spack_cc , 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) + filter_file('CXX="%s"'% spack_cxx, 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) + filter_file('F77="%s"'% spack_f77, 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) + filter_file('FC="%s"' % spack_fc , 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py index dc2b2cb23f7..23a11b3171a 100644 --- a/var/spack/packages/mvapich2/package.py +++ b/var/spack/packages/mvapich2/package.py @@ -1,5 +1,5 @@ from spack import * - +import os class Mvapich2(Package): """MVAPICH2 is an MPI implementation for Infiniband networks.""" @@ -154,3 +154,31 @@ def install(self, spec, prefix): configure(*configure_args) make() make("install") + + self.filter_compilers() + + + def filter_compilers(self): + """Run after install to make the MPI compilers use the + compilers that Spack built the package with. + + If this isn't done, they'll have CC, CXX, F77, and FC set + to Spack's generic cc, c++, f77, and f90. We want them to + be bound to whatever compiler they were built with. + """ + bin = self.prefix.bin + mpicc = os.path.join(bin, 'mpicc') + mpicxx = os.path.join(bin, 'mpicxx') + mpif77 = os.path.join(bin, 'mpif77') + mpif90 = os.path.join(bin, 'mpif90') + + spack_cc = os.environ['CC'] + spack_cxx = os.environ['CXX'] + spack_f77 = os.environ['F77'] + spack_fc = os.environ['FC'] + + kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : True } + filter_file('CC="%s"' % spack_cc , 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) + filter_file('CXX="%s"'% spack_cxx, 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) + filter_file('F77="%s"'% spack_f77, 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) + filter_file('FC="%s"' % spack_fc , 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) From 3163d016db3849c3c9e801c1cdb9e6e907afa313 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Wed, 9 Dec 2015 14:10:05 -0800 Subject: [PATCH 144/910] install python files to libxml2 prefix instead of python prefix and ignore non-python files when activating --- var/spack/packages/libxml2/package.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/var/spack/packages/libxml2/package.py b/var/spack/packages/libxml2/package.py index 5d84fad22d2..134e5969634 100644 --- a/var/spack/packages/libxml2/package.py +++ b/var/spack/packages/libxml2/package.py @@ -1,4 +1,5 @@ from spack import * +import os class Libxml2(Package): """Libxml2 is the XML C parser and toolkit developed for the Gnome @@ -11,18 +12,19 @@ class Libxml2(Package): variant('python', default=False, description='Enable Python support') - extends('python', when='+python') + extends('python', when='+python', ignore=r'(bin.*$)|(include.*$)|(share.*$)|(lib/libxml2.*$)|(lib/xml2.*$)|(lib/cmake.*$)') depends_on('zlib') depends_on('xz') def install(self, spec, prefix): if '+python' in spec: - python_arg = "--with-python=%s" % spec['python'].prefix + site_packages_dir = os.path.join(prefix, 'lib/python%s.%s/site-packages' %(spec['python'].version[:2])) + python_args = ["--with-python=%s" % spec['python'].prefix, "--with-python-install-dir=%s" % site_packages_dir] else: - python_arg = "--without-python" + python_args = ["--without-python"] configure("--prefix=%s" % prefix, - python_arg) + *python_args) make() make("install") From 25f2b01a3cbf74fa86f349287e8026ea422fbeb3 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Thu, 10 Dec 2015 07:27:06 -0800 Subject: [PATCH 145/910] fetch: add options to fetch missing or all deps Small additions to fetch to make it easier to fetch all files necessary for a build on a system without network connectivity. --- lib/spack/spack/cmd/fetch.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 6f9e7ab5e21..04cad80bd2f 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -33,10 +33,13 @@ def setup_parser(subparser): subparser.add_argument( '-n', '--no-checksum', action='store_true', dest='no_checksum', help="Do not check packages against checksum") + subparser.add_argument( + '-m', '--missing', action='store_true', help="Also fetch all missing dependencies") + subparser.add_argument( + '-d', '--dependencies', action='store_true', help="Also fetch all dependencies") subparser.add_argument( 'packages', nargs=argparse.REMAINDER, help="specs of packages to fetch") - def fetch(parser, args): if not args.packages: tty.die("fetch requires at least one package argument") @@ -46,5 +49,13 @@ def fetch(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: + if args.missing or args.dependencies: + to_fetch = set() + for s in spec.traverse(): + package = spack.db.get(s) + if args.missing and package.installed: + continue + package.do_fetch() + package = spack.db.get(spec) package.do_fetch() From 786f4cd2c2cc492f25f8886223efaa397ac97185 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Thu, 10 Dec 2015 08:32:51 -0800 Subject: [PATCH 146/910] add dependency fetching to mirror creation --- lib/spack/spack/cmd/fetch.py | 2 +- lib/spack/spack/cmd/mirror.py | 11 ++++++++++- var/spack/packages/py-cffi/package.py | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 04cad80bd2f..57d6f6b63b3 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -36,7 +36,7 @@ def setup_parser(subparser): subparser.add_argument( '-m', '--missing', action='store_true', help="Also fetch all missing dependencies") subparser.add_argument( - '-d', '--dependencies', action='store_true', help="Also fetch all dependencies") + '-D', '--dependencies', action='store_true', help="Also fetch all dependencies") subparser.add_argument( 'packages', nargs=argparse.REMAINDER, help="specs of packages to fetch") diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 4a1ce00b75d..10348e5b99e 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -54,6 +54,8 @@ def setup_parser(subparser): 'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror") create_parser.add_argument( '-f', '--file', help="File with specs of packages to put in mirror.") + create_parser.add_argument( + '-D', '--dependencies', action='store_true', help="Also fetch all dependencies") create_parser.add_argument( '-o', '--one-version-per-spec', action='store_const', const=1, default=0, help="Only fetch one 'preferred' version per spec, not all known versions.") @@ -118,7 +120,7 @@ def mirror_create(args): """Create a directory to be used as a spack mirror, and fill it with package archives.""" # try to parse specs from the command line first. - specs = spack.cmd.parse_specs(args.specs) + specs = spack.cmd.parse_specs(args.specs, concretize=True) # If there is a file, parse each line as a spec and add it to the list. if args.file: @@ -131,6 +133,13 @@ def mirror_create(args): specs = [Spec(n) for n in spack.db.all_package_names()] specs.sort(key=lambda s: s.format("$_$@").lower()) + if args.dependencies: + new_specs = set() + for spec in specs: + for s in spec.traverse(): + new_specs.add(s) + specs = list(new_specs) + # Default name for directory is spack-mirror- directory = args.directory if not directory: diff --git a/var/spack/packages/py-cffi/package.py b/var/spack/packages/py-cffi/package.py index a4d37483fe4..909049a67c9 100644 --- a/var/spack/packages/py-cffi/package.py +++ b/var/spack/packages/py-cffi/package.py @@ -4,7 +4,7 @@ class PyCffi(Package): """Foreign Function Interface for Python calling C code""" homepage = "http://cffi.readthedocs.org/en/latest/" # base https://pypi.python.org/pypi/cffi - url = "https://pypi.python.org/packages/source/c/cffi/cffi-1.1.2.tar.gz#md5=" + url = "https://pypi.python.org/packages/source/c/cffi/cffi-1.1.2.tar.gz" version('1.1.2', 'ca6e6c45b45caa87aee9adc7c796eaea') From c5b7eba45743ebd131233c035101a0e64bb0c1c7 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 16 Nov 2015 17:00:11 -0500 Subject: [PATCH 147/910] hdf5: make mpi support optional --- var/spack/packages/hdf5/package.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/hdf5/package.py b/var/spack/packages/hdf5/package.py index 48997425cdb..44d4ede2787 100644 --- a/var/spack/packages/hdf5/package.py +++ b/var/spack/packages/hdf5/package.py @@ -15,19 +15,28 @@ class Hdf5(Package): version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') version('1.8.13', 'c03426e9e77d7766944654280b467289') - depends_on("mpi") + variant('mpi', default=False, description='Enable MPI support') + + depends_on("mpi", when='+mpi') depends_on("zlib") # TODO: currently hard-coded to use OpenMPI def install(self, spec, prefix): + extra_args = [] + if '+mpi' in spec: + extra_args.extend([ + "--enable-parallel", + "CC=%s" % spec['mpich'].prefix.bin + "/mpicc", + "CXX=%s" % spec['mpich'].prefix.bin + "/mpic++", + ]) configure( "--prefix=%s" % prefix, "--with-zlib=%s" % spec['zlib'].prefix, - "--enable-parallel", "--enable-shared", "CC=%s" % spec['mpi'].prefix.bin + "/mpicc", - "CXX=%s" % spec['mpi'].prefix.bin + "/mpic++") + "CXX=%s" % spec['mpi'].prefix.bin + "/mpic++", + *extra_args) make() make("install") From 79a641ccf16ee0bef0c32730c99b8513a88aa918 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 16 Nov 2015 17:01:01 -0500 Subject: [PATCH 148/910] paraview: update hdf5 comment --- var/spack/packages/paraview/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/paraview/package.py b/var/spack/packages/paraview/package.py index b0893237e9a..3bee5a10415 100644 --- a/var/spack/packages/paraview/package.py +++ b/var/spack/packages/paraview/package.py @@ -24,7 +24,7 @@ class Paraview(Package): depends_on('bzip2') depends_on('freetype') - depends_on('hdf5') # drags in mpi + depends_on('hdf5') depends_on('jpeg') depends_on('libpng') depends_on('libtiff') From 6f978af77b7b98ca51e47bc8d3c097fe236e1491 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 10 Dec 2015 11:53:40 -0500 Subject: [PATCH 149/910] hdf5: update dependents to declare MPI support --- var/spack/packages/SAMRAI/package.py | 2 +- var/spack/packages/cleverleaf/package.py | 2 +- var/spack/packages/paraview/package.py | 1 + var/spack/packages/petsc/package.py | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/SAMRAI/package.py b/var/spack/packages/SAMRAI/package.py index a17aea9c993..2c3b9180afe 100644 --- a/var/spack/packages/SAMRAI/package.py +++ b/var/spack/packages/SAMRAI/package.py @@ -25,7 +25,7 @@ class Samrai(Package): depends_on("mpi") depends_on("zlib") - depends_on("hdf5") + depends_on("hdf5+mpi") depends_on("boost") # don't build tools with gcc diff --git a/var/spack/packages/cleverleaf/package.py b/var/spack/packages/cleverleaf/package.py index 4e7e6a855ab..fb400b25c3e 100644 --- a/var/spack/packages/cleverleaf/package.py +++ b/var/spack/packages/cleverleaf/package.py @@ -14,7 +14,7 @@ class Cleverleaf(Package): version('develop', git='https://github.com/UK-MAC/CleverLeaf_ref.git', branch='develop') depends_on("SAMRAI@3.8.0:") - depends_on("hdf5") + depends_on("hdf5+mpi") depends_on("boost") def install(self, spec, prefix): diff --git a/var/spack/packages/paraview/package.py b/var/spack/packages/paraview/package.py index 3bee5a10415..1d99b348997 100644 --- a/var/spack/packages/paraview/package.py +++ b/var/spack/packages/paraview/package.py @@ -25,6 +25,7 @@ class Paraview(Package): depends_on('bzip2') depends_on('freetype') depends_on('hdf5') + depends_on('hdf5+mpi', when='+mpi') depends_on('jpeg') depends_on('libpng') depends_on('libtiff') diff --git a/var/spack/packages/petsc/package.py b/var/spack/packages/petsc/package.py index 4864e39bf18..f3ed3d72ec5 100644 --- a/var/spack/packages/petsc/package.py +++ b/var/spack/packages/petsc/package.py @@ -18,7 +18,7 @@ class Petsc(Package): depends_on("hypre") depends_on("parmetis") depends_on("metis") - depends_on("hdf5") + depends_on("hdf5+mpi") depends_on("mpi") def install(self, spec, prefix): From 281a869ef699c354f31bbb069ab28012b65c9b4c Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Thu, 10 Dec 2015 09:24:01 -0800 Subject: [PATCH 150/910] fix path resolution for mirror packages, especially with dependency fetching --- lib/spack/spack/cmd/mirror.py | 1 + lib/spack/spack/mirror.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 10348e5b99e..89d51bbe046 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -136,6 +136,7 @@ def mirror_create(args): if args.dependencies: new_specs = set() for spec in specs: + spec.concretize() for s in spec.traverse(): new_specs.add(s) specs = list(new_specs) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index ee0bf6de11f..6fbf82de143 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -146,7 +146,7 @@ def create(path, specs, **kwargs): stage = None try: # create a subdirectory for the current package@version - archive_path = os.path.abspath(join_path(path, mirror_archive_path(spec))) + archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec))) subdir = os.path.dirname(archive_path) mkdirp(subdir) From 7526a89463c6f4c61cd4ae725fc096e14fc063f3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 11 Dec 2015 02:16:24 -0800 Subject: [PATCH 151/910] Fix #217: Make package cache use DAG hash instead of sorted deps. - Gets rid of last vestige of old-style specs. - Uses new hashing for lookup --- lib/spack/spack/packages.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index f6f4cbf0254..9d87c3a94b5 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -67,27 +67,28 @@ def get(self, spec, **kwargs): if spec.virtual: raise UnknownPackageError(spec.name) + dhash = spec.dag_hash() if kwargs.get('new', False): - if spec in self.instances: - del self.instances[spec] + if dhash in self.instances: + del self.instances[dhash] - if not spec in self.instances: + if not dhash in self.instances: package_class = self.get_class_for_package_name(spec.name) try: - copy = spec.copy() - self.instances[copy] = package_class(copy) + copy = spec.copy() # defensive copy. Package owns its spec. + self.instances[dhash] = package_class(copy) except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) raise FailedConstructorError(spec.name, e) - return self.instances[spec] + return self.instances[dhash] @_autospec def delete(self, spec): """Force a package to be recreated.""" - del self.instances[spec] + del self.instances[spec.dag_hash()] def purge(self): From 6ee2eb21dd2c3c32d115ebf28becc735311bb1a9 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 11 Dec 2015 12:29:32 -0800 Subject: [PATCH 152/910] Fix #217: Use MUCH faster hashing, reduce number of DAG copies. This changes the hash algorithm so that it does much less object allocation and copying, and so that it is correct. The old version of `_cmp_key()` would call `sorted_deps`, which would call `flat_dependencies` to get a list of dependencies so that it could sort them in alphabetical order. This isn't necessary in the `_cmp_key()`, and in fact we want more DAG structure than that to be included in the `_cmp_key()`. The new version constructs a tuple without copying the Spec DAG, and the tuple contains hashes of sub-DAGs that are computed recursively in-place. This is way faster than the previous algorithm and reduces the numebr of copies significantly. It is also a correct DAG hash. Example timing and copy counts for the different hashing algorithms we've tried: Original (wrong) Spec hash: ``` 106,170 copies real 0m5.024s user 0m4.949s sys 0m0.104s ``` Spec hash using YAML `dag_hash()`: ``` 3,794 copies real 0m5.024s user 0m4.949s sys 0m0.104s New no-copy, no-YAML hash: ``` 3,594 copies real 0m2.543s user 0m2.435s sys 0m0.104s ``` So now we have a hash that is correct AND faster. The remaining ~3k copies happen mostly during concretization, and as all packages are initially loaded. I believe this is because Spack currently has to load all packages to figure out virtual dependency information; it could also be becasue there ar a lot of lookups of partial specs in concretize. I can investigate this further. --- lib/spack/spack/packages.py | 12 ++++++------ lib/spack/spack/spec.py | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 9d87c3a94b5..080644fb900 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -67,22 +67,22 @@ def get(self, spec, **kwargs): if spec.virtual: raise UnknownPackageError(spec.name) - dhash = spec.dag_hash() + key = hash(spec) if kwargs.get('new', False): - if dhash in self.instances: - del self.instances[dhash] + if key in self.instances: + del self.instances[key] - if not dhash in self.instances: + if not key in self.instances: package_class = self.get_class_for_package_name(spec.name) try: copy = spec.copy() # defensive copy. Package owns its spec. - self.instances[dhash] = package_class(copy) + self.instances[key] = package_class(copy) except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) raise FailedConstructorError(spec.name, e) - return self.instances[dhash] + return self.instances[key] @_autospec diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f62182ce76b..483da03c47d 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1481,8 +1481,11 @@ def ne_dag(self, other): def _cmp_node(self): """Comparison key for just *this node* and not its deps.""" - return (self.name, self.versions, self.variants, - self.architecture, self.compiler) + return (self.name, + self.versions, + self.variants, + self.architecture, + self.compiler) def eq_node(self, other): @@ -1496,11 +1499,14 @@ def ne_node(self, other): def _cmp_key(self): - """Comparison key for this node and all dependencies *without* - considering structure. This is the default, as - normalization will restore structure. + """This returns a key for the spec *including* DAG structure. + + The key is the concatenation of: + 1. A tuple describing this node in the DAG. + 2. The hash of each of this node's dependencies' cmp_keys. """ - return self._cmp_node() + (self.sorted_deps(),) + return self._cmp_node() + ( + tuple(sorted(hash(d) for d in self.dependencies.values())),) def colorized(self): From 3dd6cbc556458ff5776988d12927b9d93c326b64 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 11 Dec 2015 13:07:20 -0800 Subject: [PATCH 153/910] Fix #217: update spec_dag test for new `_cmp_key`. --- lib/spack/spack/test/spec_dag.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 94438b6a0c9..d3a4d77b32d 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -340,16 +340,18 @@ def test_normalize_mpileaks(self): self.assertEqual(spec, expected_flat) self.assertTrue(spec.eq_dag(expected_flat)) - self.assertEqual(spec, expected_normalized) + # Normalized has different DAG structure, so NOT equal. + self.assertNotEqual(spec, expected_normalized) self.assertFalse(spec.eq_dag(expected_normalized)) - self.assertEqual(spec, non_unique_nodes) + # Again, different DAG structure so not equal. + self.assertNotEqual(spec, non_unique_nodes) self.assertFalse(spec.eq_dag(non_unique_nodes)) spec.normalize() # After normalizing, spec_dag_equal should match the normalized spec. - self.assertEqual(spec, expected_flat) + self.assertNotEqual(spec, expected_flat) self.assertFalse(spec.eq_dag(expected_flat)) self.assertEqual(spec, expected_normalized) From afbd0e77d0754227b799ae3872d8775ceec0ccd0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 11 Dec 2015 16:47:34 -0800 Subject: [PATCH 154/910] Make internal hash dep sort order match external one. --- lib/spack/spack/spec.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 483da03c47d..fb5ea5d02b4 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1506,7 +1506,8 @@ def _cmp_key(self): 2. The hash of each of this node's dependencies' cmp_keys. """ return self._cmp_node() + ( - tuple(sorted(hash(d) for d in self.dependencies.values())),) + tuple(hash(self.dependencies[name]) + for name in sorted(self.dependencies)),) def colorized(self): From 8df4fd9e942cb4e632e309ae758356c29a72ec02 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Sun, 13 Dec 2015 17:13:34 -0800 Subject: [PATCH 155/910] Fix build of krell tools to use spack with minimal impact on rest of spack packages. --- var/spack/packages/binutils/package.py | 6 ---- var/spack/packages/boost/package.py | 3 +- var/spack/packages/cbtf-krell/package.py | 38 ++++++++++----------- var/spack/packages/cbtf-lanl/package.py | 2 +- var/spack/packages/dyninst/package.py | 33 +++++------------- var/spack/packages/openspeedshop/package.py | 33 ++++++++++-------- var/spack/packages/papi/package.py | 1 + 7 files changed, 51 insertions(+), 65 deletions(-) diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py index 47234367d70..123f4598f69 100644 --- a/var/spack/packages/binutils/package.py +++ b/var/spack/packages/binutils/package.py @@ -15,12 +15,6 @@ class Binutils(Package): variant('libiberty', default=False, description='Also install libiberty.') - variant('libiberty', default=False, description='Also install libiberty.') - - variant('libiberty', default=False, description='Also install libiberty.') - - variant('libiberty', default=False, description='Also install libiberty.') - def install(self, spec, prefix): configure_args = [ '--prefix=%s' % prefix, diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index 81dadbbf611..fc721753c70 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -58,7 +58,8 @@ def url_for_version(self, version): parts = [str(p) for p in Version(version)] dots = ".".join(parts) underscores = "_".join(parts) - return "http://downloads.sourceforge.net/project/boost/boost/%s/boost_%s.tar.bz2" % ( + #return "http://downloads.sourceforge.net/project/boost/boost/%s/boost_%s.tar.bz2" % ( + return "http://downloads.sourceforge.net/project/boost/boost/%s.0/boost_%s_0.tar.bz2" % ( dots, underscores) def determine_toolset(self): diff --git a/var/spack/packages/cbtf-krell/package.py b/var/spack/packages/cbtf-krell/package.py index 3f486b86ced..d9a2b7f8b53 100644 --- a/var/spack/packages/cbtf-krell/package.py +++ b/var/spack/packages/cbtf-krell/package.py @@ -38,8 +38,8 @@ class CbtfKrell(Package): depends_on("binutils@2.24+krellpatch") # collectionTool - depends_on("boost@1.42:") - depends_on("dyninst@8.2.1:+krelloptions") + depends_on("boost@1.50") + depends_on("dyninst@8.2.1") depends_on("mrnet@4.1.0:+krelloptions") depends_on("xerces-c@3.1.1:") depends_on("cbtf") @@ -64,8 +64,8 @@ def install(self, spec, prefix): # FIXME - hard code path until external package support is available # Need to change this path and/or add additional paths for MPI experiment support on different platforms - openmpi_prefix_path = "/opt/openmpi-1.8.2" - mvapich_prefix_path = "/usr/local/tools/mvapich-gnu" + #openmpi_prefix_path = "/opt/openmpi-1.8.2" + #mvapich_prefix_path = "/usr/local/tools/mvapich-gnu" # Other possibilities, they will need a -DMVAPICH_DIR=, etc clause in the cmake command to be recognized # mvapich_prefix_path = "" @@ -76,18 +76,22 @@ def install(self, spec, prefix): # Add in paths for cuda if requested via the cuda variant # FIXME - hard code path until external package support is available - if '+cuda' in spec: - cuda_prefix_path = "/usr/local/cuda-6.0" - cupti_prefix_path = "/usr/local/cuda-6.0/extras/CUPTI" - else: - cuda_prefix_path = "" - cupti_prefix_path = "" + #if '+cuda' in spec: + # cuda_prefix_path = "/usr/local/cuda-6.0" + # cupti_prefix_path = "/usr/local/cuda-6.0/extras/CUPTI" + #else: + # cuda_prefix_path = "" + # cupti_prefix_path = "" - #'-DMVAPICH2_DIR=%s' % spec['mvapich2'].prefix, - #'-DOPENMPI_DIR=%s' % spec['openmpi'].prefix, - #'-DMPICH_DIR=%s' % spec['mpich'].prefix, - #'-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, - #'-DLIB_SUFFIX=64', + #'-DMVAPICH2_DIR=%s' % spec['mvapich2'].prefix, + #'-DOPENMPI_DIR=%s' % spec['openmpi'].prefix, + #'-DMPICH_DIR=%s' % spec['mpich'].prefix, + #'-DCMAKE_LIBRARY_PATH=%s' % prefix.lib64, + #'-DOPENMPI_DIR=%s' % openmpi_prefix_path, + #'-DMVAPICH_DIR=%s' % mvapich_prefix_path, + #'-DLIB_SUFFIX=64', + #'-DCUDA_DIR=%s' % cuda_prefix_path, + #'-DCUPTI_DIR=%s' % cupti_prefix_path, # Build cbtf-krell with cmake with working_dir('build_cbtf_krell', create=True): @@ -104,10 +108,6 @@ def install(self, spec, prefix): '-DDYNINST_DIR=%s' % spec['dyninst'].prefix, '-DXERCESC_DIR=%s' % spec['xerces-c'].prefix, '-DCMAKE_PREFIX_PATH=%s' % cmake_prefix_path, - '-DCUDA_DIR=%s' % cuda_prefix_path, - '-DCUPTI_DIR=%s' % cupti_prefix_path, - '-DOPENMPI_DIR=%s' % openmpi_prefix_path, - '-DMVAPICH_DIR=%s' % mvapich_prefix_path, *std_cmake_args) make("clean") diff --git a/var/spack/packages/cbtf-lanl/package.py b/var/spack/packages/cbtf-lanl/package.py index 25f91d8c2f5..14651723cd3 100644 --- a/var/spack/packages/cbtf-lanl/package.py +++ b/var/spack/packages/cbtf-lanl/package.py @@ -31,7 +31,7 @@ class CbtfLanl(Package): # Dependencies for cbtf-krell - depends_on("boost@1.42:") + depends_on("boost@1.50") depends_on("mrnet@4.1.0+krelloptions") depends_on("xerces-c@3.1.1:") depends_on("cbtf") diff --git a/var/spack/packages/dyninst/package.py b/var/spack/packages/dyninst/package.py index 81ea7b7e483..0111dcbe08d 100644 --- a/var/spack/packages/dyninst/package.py +++ b/var/spack/packages/dyninst/package.py @@ -38,8 +38,6 @@ class Dyninst(Package): version('8.1.1', 'd1a04e995b7aa70960cd1d1fac8bd6ac', url="http://www.paradyn.org/release8.1/DyninstAPI-8.1.1.tgz") - variant('krelloptions', default=False, description="build dyninst includes into include/dyninst.") - depends_on("libelf") depends_on("libdwarf") depends_on("boost@1.42:") @@ -50,28 +48,15 @@ def install(self, spec, prefix): libdwarf = spec['libdwarf'].prefix with working_dir('spack-build', create=True): - # cbtf-krell and openspeedshop expect to reference dyninst/include, so adding include specification to that end - if '+krelloptions' in spec: - cmake('..', - '-DINSTALL_INCLUDE_DIR=%s' % join_path(self.prefix.include, 'dyninst'), - '-DBoost_INCLUDE_DIR=%s' % spec['boost'].prefix.include, - '-DBoost_LIBRARY_DIR=%s' % spec['boost'].prefix.lib, - '-DBoost_NO_SYSTEM_PATHS=TRUE', - '-DLIBELF_INCLUDE_DIR=%s' % join_path(libelf.include, 'libelf'), - '-DLIBELF_LIBRARIES=%s' % join_path(libelf.lib, 'libelf.so'), - '-DLIBDWARF_INCLUDE_DIR=%s' % libdwarf.include, - '-DLIBDWARF_LIBRARIES=%s' % join_path(libdwarf.lib, 'libdwarf.so'), - *std_cmake_args) - else: - cmake('..', - '-DBoost_INCLUDE_DIR=%s' % spec['boost'].prefix.include, - '-DBoost_LIBRARY_DIR=%s' % spec['boost'].prefix.lib, - '-DBoost_NO_SYSTEM_PATHS=TRUE', - '-DLIBELF_INCLUDE_DIR=%s' % join_path(libelf.include, 'libelf'), - '-DLIBELF_LIBRARIES=%s' % join_path(libelf.lib, 'libelf.so'), - '-DLIBDWARF_INCLUDE_DIR=%s' % libdwarf.include, - '-DLIBDWARF_LIBRARIES=%s' % join_path(libdwarf.lib, 'libdwarf.so'), - *std_cmake_args) + cmake('..', + '-DBoost_INCLUDE_DIR=%s' % spec['boost'].prefix.include, + '-DBoost_LIBRARY_DIR=%s' % spec['boost'].prefix.lib, + '-DBoost_NO_SYSTEM_PATHS=TRUE', + '-DLIBELF_INCLUDE_DIR=%s' % join_path(libelf.include, 'libelf'), + '-DLIBELF_LIBRARIES=%s' % join_path(libelf.lib, 'libelf.so'), + '-DLIBDWARF_INCLUDE_DIR=%s' % libdwarf.include, + '-DLIBDWARF_LIBRARIES=%s' % join_path(libdwarf.lib, 'libdwarf.so'), + *std_cmake_args) make() make("install") diff --git a/var/spack/packages/openspeedshop/package.py b/var/spack/packages/openspeedshop/package.py index 93f356fb924..d439fba734c 100644 --- a/var/spack/packages/openspeedshop/package.py +++ b/var/spack/packages/openspeedshop/package.py @@ -28,9 +28,14 @@ class Openspeedshop(Package): as open source code primarily under LGPL. """ + homepage = "http://www.openspeedshop.org" - url = "http://sourceforge.net/projects/openss/files/openss/openspeedshop-2.1/openspeedshop-2.1.tar.gz/download" - version('2.1', 'bdaa57c1a0db9d0c3e0303fd8496c507') + url = "http://sourceforge.net/projects/openss/files/openss/openspeedshop-2.2/openspeedshop-2.2.tar.gz/download" + version('2.2', '16cb051179c2038de4e8a845edf1d573') + + #homepage = "http://www.openspeedshop.org" + #url = "http://sourceforge.net/projects/openss/files/openss/openspeedshop-2.1/openspeedshop-2.1.tar.gz/download" + #version('2.1', 'bdaa57c1a0db9d0c3e0303fd8496c507') # optional mirror template #url = "file:/g/g24/jeg/openspeedshop-2.1.tar.gz" @@ -40,7 +45,7 @@ class Openspeedshop(Package): parallel = False - variant('offline', default=False, description="build with offline instrumentor enabled.") + variant('offline', default=True, description="build with offline instrumentor enabled.") variant('cbtf', default=False, description="build with cbtf instrumentor enabled.") variant('runtime', default=False, description="build only the runtime libraries and collectors.") variant('frontend', default=False, description="build only the front-end tool using the runtime_dir to point to the target build.") @@ -58,8 +63,8 @@ class Openspeedshop(Package): depends_on("libelf") depends_on("libdwarf") depends_on("sqlite") - depends_on("boost@1.42:") - depends_on("dyninst@8.2.1:+krelloptions") + depends_on("boost@1.50") + depends_on("dyninst@8.2.1") depends_on("python") depends_on("qt@3.3.8b+krellpatch") @@ -79,15 +84,17 @@ class Openspeedshop(Package): def install(self, spec, prefix): - openmpi_prefix_path = "/opt/openmpi-1.8.2" - mvapich_prefix_path = "/usr/local/tools/mvapich-gnu" + #openmpi_prefix_path = "/opt/openmpi-1.8.2" + #mvapich_prefix_path = "/usr/local/tools/mvapich-gnu" #'-DOPENMPI_DIR=%s' % spec['openmpi'].prefix, + #'-DOPENMPI_DIR=%s' % openmpi_prefix_path, + #'-DMVAPICH_DIR=%s' % mvapich_prefix_path, # FIXME: How do we make this dynamic in spack? That is, can we specify the paths to cuda dynamically? # WAITING for external package support. - if '+cuda' in spec: - cuda_prefix_path = "/usr/local/cuda-6.0" - cupti_prefix_path = "/usr/local/cuda-6.0/extras/CUPTI" + #if '+cuda' in spec: + # cuda_prefix_path = "/usr/local/cuda-6.0" + # cupti_prefix_path = "/usr/local/cuda-6.0/extras/CUPTI" if '+offline' in spec: instrumentor_setting = "offline" @@ -100,8 +107,6 @@ def install(self, spec, prefix): '-DLIBMONITOR_DIR=%s' % spec['libmonitor'].prefix, '-DLIBUNWIND_DIR=%s' % spec['libunwind'].prefix, '-DPAPI_DIR=%s' % spec['papi'].prefix, - '-DOPENMPI_DIR=%s' % openmpi_prefix_path, - '-DMVAPICH_DIR=%s' % mvapich_prefix_path, *std_cmake_args) make("clean") make() @@ -110,6 +115,8 @@ def install(self, spec, prefix): cmake_prefix_path = join_path(spec['dyninst'].prefix) with working_dir('build', create=True): #python_vers=join_path(spec['python'].version[:2]) + #'-DOPENMPI_DIR=%s' % openmpi_prefix_path, + #'-DMVAPICH_DIR=%s' % mvapich_prefix_path, python_vers='%d.%d' % spec['python'].version[:2] cmake('..', '-DCMAKE_INSTALL_PREFIX=%s' % prefix, @@ -130,8 +137,6 @@ def install(self, spec, prefix): '-DBoost_NO_SYSTEM_PATHS=TRUE', '-DBOOST_ROOT=%s' % spec['boost'].prefix, '-DDYNINST_DIR=%s' % spec['dyninst'].prefix, - '-DOPENMPI_DIR=%s' % openmpi_prefix_path, - '-DMVAPICH_DIR=%s' % mvapich_prefix_path, *std_cmake_args) make("clean") make() diff --git a/var/spack/packages/papi/package.py b/var/spack/packages/papi/package.py index f7c087c69b3..910e0aa9f9a 100644 --- a/var/spack/packages/papi/package.py +++ b/var/spack/packages/papi/package.py @@ -12,6 +12,7 @@ class Papi(Package): across the hardware and software stack.""" homepage = "http://icl.cs.utk.edu/papi/index.html" + url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.4.1.tar.gz" version('5.4.1', '9134a99219c79767a11463a76b0b01a2') version('5.3.0', '367961dd0ab426e5ae367c2713924ffb') From 1bfb16592b961cc8f1e95afaf12e608b55ff4b60 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 14 Dec 2015 16:31:02 +0100 Subject: [PATCH 156/910] fftw : added package --- var/spack/packages/fftw/package.py | 96 ++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 var/spack/packages/fftw/package.py diff --git a/var/spack/packages/fftw/package.py b/var/spack/packages/fftw/package.py new file mode 100644 index 00000000000..5f71762c4ff --- /dev/null +++ b/var/spack/packages/fftw/package.py @@ -0,0 +1,96 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + + +from spack import * + + +class Fftw(Package): + """ + FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) in one or more dimensions, of + arbitrary input size, and of both real and complex data (as well as of even/odd data, i.e. the discrete cosine/sine + transforms or DCT/DST). We believe that FFTW, which is free software, should become the FFT library of choice for + most applications. + """ + homepage = "http://www.fftw.org" + url = "http://www.fftw.org/fftw-3.3.4.tar.gz" + + version('3.3.4', '2edab8c06b24feeb3b82bbb3ebf3e7b3') + + ########## + # Floating point precision + FLOAT = 'float' + LONG_DOUBLE = 'long_double' + QUAD_PRECISION = 'quad' + PRECISION_OPTIONS = { + FLOAT: '--enable-float', + LONG_DOUBLE: '--enable--long-double', + QUAD_PRECISION: '--enable-quad-precision' + } + variant(FLOAT, default=False, description='Produces a single precision version of the library') + variant(LONG_DOUBLE, default=False, description='Produces a long double precision version of the library') + variant(QUAD_PRECISION, default=False, description='Produces a quad precision version of the library (works only with GCC and libquadmath)') + ########## + + variant('mpi', default=False, description='Activate MPI support') + + depends_on('mpi', when='+mpi') + + @staticmethod + def enabled(x): + """ + Given a variant name returns the string that means the variant is enabled + + :param x: variant name + """ + # FIXME : duplicated from MVAPICH2 + return '+' + x + + def check_fortran_availability(self, options): + if not self.compiler.f77 or not self.compiler.fc: + options.append("--disable-fortran") + + def set_floating_point_precision(self, spec, options): + l = [option for variant, option in Fftw.PRECISION_OPTIONS.iteritems() if self.enabled(variant) in spec] + if len(l) > 1: + raise RuntimeError('At most one floating point precision variant may activated per build.') + options.extend(l) + + def install(self, spec, prefix): + + options = ['--prefix=%s' % prefix, + '--enable-shared', + '--enable-threads', + '--enable-openmp'] + self.check_fortran_availability(options) + self.set_floating_point_precision(spec, options) + + if '+mpi' in spec: + options.append('--enable-mpi') + + configure(*options) + make() + make("install") + From ed7db8e48999de4e6bb67efce7a97718ad117a74 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Mon, 14 Dec 2015 21:59:40 +0100 Subject: [PATCH 157/910] adding external dependencies + variants for mpi, shared and compression --- var/spack/packages/scotch/package.py | 117 +++++++++++++++++++++++---- 1 file changed, 102 insertions(+), 15 deletions(-) diff --git a/var/spack/packages/scotch/package.py b/var/spack/packages/scotch/package.py index 79289ff2ade..7e54b4fd8dc 100644 --- a/var/spack/packages/scotch/package.py +++ b/var/spack/packages/scotch/package.py @@ -1,5 +1,4 @@ from spack import * -import glob import os class Scotch(Package): @@ -11,28 +10,116 @@ class Scotch(Package): version('6.0.3', '10b0cc0f184de2de99859eafaca83cfc') - depends_on('mpi') + variant('mpi', default=False, description='Activate the compilation of PT-Scotch') + variant('compression', default=True, description='Activate the posibility to use compressed files') + variant('esmumps', default=False, description='Activate the compilation of the lib esmumps needed by mumps') + variant('shared', default=True, description='Build shared libraries') + + depends_on('mpi', when='+mpi') + depends_on('zlib', when='+compression') + depends_on('flex') + depends_on('bison') + + def compiler_specifics(self, makefile_inc, defines): + if self.compiler.name == 'gcc': + defines.append('-Drestrict=__restrict') + elif self.compiler.name == 'intel': + defines.append('-restrict') + + makefile_inc.append('CCS = $(CC)') + + if '+mpi' in self.spec: + makefile_inc.extend([ + 'CCP = %s' % os.path.join(self.spec['mpi'].prefix.bin, 'mpicc'), + 'CCD = $(CCP)' + ]) + else: + makefile_inc.extend([ + 'CCP = mpicc', # It is set but not used + 'CCD = $(CCS)' + ]) - def patch(self): - with working_dir('src/Make.inc'): - makefiles = glob.glob('Makefile.inc.x86-64_pc_linux2*') - filter_file(r'^CCS\s*=.*$', 'CCS = cc', *makefiles) - filter_file(r'^CCD\s*=.*$', 'CCD = cc', *makefiles) + def library_build_type(self, makefile_inc, defines): + makefile_inc.extend([ + 'LIB = .a', + 'CLIBFLAGS = ', + 'RANLIB = ranlib', + 'AR = ar', + 'ARFLAGS = -ruv ' + ]) + + @when('+shared') + def library_build_type(self, makefile_inc, defines): + makefile_inc.extend([ + 'LIB = .so', + 'CLIBFLAGS = -shared -fPIC', + 'RANLIB = echo', + 'AR = $(CC)', + 'ARFLAGS = -shared $(LDFLAGS) -o' + ]) + + def extra_features(self, makefile_inc, defines): + ldflags = [] + + if '+compression' in self.spec: + defines.append('-DCOMMON_FILE_COMPRESS_GZ') + ldflags.append('-L%s -lz' % (self.spec['zlib'].prefix.lib)) + + defines.append('-DCOMMON_PTHREAD') + ldflags.append('-lm -lrt -pthread') + + makefile_inc.append('LDFLAGS = %s' % ' '.join(ldflags)) + + + def write_make_inc(self): + makefile_inc = [] + defines = [ + '-DCOMMON_RANDOM_FIXED_SEED', + '-DSCOTCH_DETERMINISTIC', + '-DSCOTCH_RENAME', + '-DIDXSIZE64' ] + + self.library_build_type(makefile_inc, defines) + self.compiler_specifics(makefile_inc, defines) + self.extra_features(makefile_inc, defines) + + makefile_inc.extend([ + 'EXE =', + 'OBJ = .o', + 'MAKE = make', + 'CAT = cat', + 'LN = ln', + 'MKDIR = mkdir', + 'MV = mv', + 'CP = cp', + 'CFLAGS = -O3 %s' % (' '.join(defines)), + 'LEX = %s -Pscotchyy -olex.yy.c' % os.path.join(self.spec['flex'].prefix.bin , 'flex'), + 'YACC = %s -pscotchyy -y -b y' % os.path.join(self.spec['bison'].prefix.bin, 'bison'), + 'prefix = %s' % self.prefix, + '' + ]) + + with open('Makefile.inc', 'w') as fh: + fh.write('\n'.join(makefile_inc)) def install(self, spec, prefix): - # Currently support gcc and icc on x86_64 (maybe others with - # vanilla makefile) - makefile = 'Make.inc/Makefile.inc.x86-64_pc_linux2' - if spec.satisfies('%icc'): - makefile += '.icc' + targets = ['scotch'] + if '+mpi' in self.spec: + targets.append('ptscotch') + + if '+esmumps' in self.spec: + targets.append('esmumps') + if '+mpi' in self.spec: + targets.append('ptesmumps') with working_dir('src'): - force_symlink(makefile, 'Makefile.inc') - for app in ('scotch', 'ptscotch'): - make(app) + self.write_make_inc() + for app in targets: + make(app, parallel=(not app=='ptesmumps')) + install_tree('bin', prefix.bin) install_tree('lib', prefix.lib) install_tree('include', prefix.include) From 6e074a196a25f5737e50c57bf64b1d452245ef50 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 14 Dec 2015 21:06:32 -0800 Subject: [PATCH 158/910] Fix #206: need to make deps AND root concrete when read in. --- lib/spack/spack/directory_layout.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 056606b429a..a434dad5c49 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -212,8 +212,10 @@ def read_spec(self, path): spec = Spec.from_yaml(f) # Specs read from actual installations are always concrete - spec._normal = True - spec._concrete = True + for s in spec.traverse(): + s._normal = True + s._concrete = True + return spec From 0a0cd4b9d3c86e7e8438261b94be05caac5ea178 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Dec 2015 16:29:36 +0100 Subject: [PATCH 159/910] gnuplot : added package --- var/spack/packages/gnuplot/package.py | 59 ++++++++++++++++++++++ var/spack/packages/libcerf/package.py | 42 +++++++++++++++ var/spack/packages/libgd/package.py | 51 +++++++++++++++++++ var/spack/packages/ncurses/package.py | 2 + var/spack/packages/ncurses/patch_gcc_5.txt | 12 +++++ var/spack/packages/wx/package.py | 2 + 6 files changed, 168 insertions(+) create mode 100644 var/spack/packages/gnuplot/package.py create mode 100644 var/spack/packages/libcerf/package.py create mode 100644 var/spack/packages/libgd/package.py create mode 100644 var/spack/packages/ncurses/patch_gcc_5.txt diff --git a/var/spack/packages/gnuplot/package.py b/var/spack/packages/gnuplot/package.py new file mode 100644 index 00000000000..e034cb09951 --- /dev/null +++ b/var/spack/packages/gnuplot/package.py @@ -0,0 +1,59 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + +import os + +class Gnuplot(Package): + """ + Gnuplot is a portable command-line driven graphing utility for Linux, OS/2, MS Windows, OSX, VMS, and many other + platforms. The source code is copyrighted but freely distributed (i.e., you don't have to pay for it). It was + originally created to allow scientists and students to visualize mathematical functions and data interactively, + but has grown to support many non-interactive uses such as web scripting. It is also used as a plotting engine by + third-party applications like Octave. Gnuplot has been supported and under active development since 1986 + """ + homepage = "http://www.gnuplot.info" + url = "http://downloads.sourceforge.net/project/gnuplot/gnuplot/5.0.1/gnuplot-5.0.1.tar.gz" + + version('5.0.1', '79b4f9e203728f76b60b28bcd402d3c7') + + depends_on('readline') + depends_on('libcerf') + depends_on('libgd') + depends_on('cairo') + depends_on('pango') + depends_on('wx') + + def install(self, spec, prefix): + # It seems there's an open bug for wxWidgets support + # See : http://sourceforge.net/p/gnuplot/bugs/1694/ + os.environ['TERMLIBS'] = '-lX11' + + options = ['--prefix=%s' % prefix] + + configure(*options) + make() + make("install") diff --git a/var/spack/packages/libcerf/package.py b/var/spack/packages/libcerf/package.py new file mode 100644 index 00000000000..15e87ce4fe6 --- /dev/null +++ b/var/spack/packages/libcerf/package.py @@ -0,0 +1,42 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Libcerf(Package): + """ + A self-contained C library providing complex error functions, based on Faddeeva's plasma dispersion function + w(z). Also provides Dawson's integral and Voigt's convolution of a Gaussian and a Lorentzian + """ + homepage = "http://sourceforge.net/projects/libcerf" + url = "http://downloads.sourceforge.net/project/libcerf/libcerf-1.3.tgz" + + version('1.3', 'b3504c467204df71e62aeccf73a25612') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") diff --git a/var/spack/packages/libgd/package.py b/var/spack/packages/libgd/package.py new file mode 100644 index 00000000000..b066b43878f --- /dev/null +++ b/var/spack/packages/libgd/package.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Libgd(Package): + """ + GD is an open source code library for the dynamic creation of images by programmers. GD is written in C, and + "wrappers" are available for Perl, PHP and other languages. GD creates PNG, JPEG, GIF, WebP, XPM, BMP images, + among other formats. GD is commonly used to generate charts, graphics, thumbnails, and most anything else, on the + fly. While not restricted to use on the web, the most common applications of GD involve website development. + """ + + homepage = "https://github.com/libgd/libgd" + url = "https://github.com/libgd/libgd/archive/gd-2.1.1.tar.gz" + + version('2.1.1', 'e91a1a99903e460e7ba00a794e72cc1e') + + def install(self, spec, prefix): + + with working_dir('spack-build', create=True): + cmake('..', + '-DENABLE_JPEG:BOOL=ON', + '-DENABLE_PNG:BOOL=ON', + '-DENABLE_TIFF:BOOL=ON', + *std_cmake_args) + make() + make("install") diff --git a/var/spack/packages/ncurses/package.py b/var/spack/packages/ncurses/package.py index cc180bbae1b..31f53b6c43e 100644 --- a/var/spack/packages/ncurses/package.py +++ b/var/spack/packages/ncurses/package.py @@ -14,6 +14,8 @@ class Ncurses(Package): version('6.0', 'ee13d052e1ead260d7c28071f46eefb1', url='http://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.0.tar.gz') + patch('patch_gcc_5.txt', when='%gcc@5.0:') + def install(self, spec, prefix): configure("--prefix=%s" % prefix, "--with-shared", diff --git a/var/spack/packages/ncurses/patch_gcc_5.txt b/var/spack/packages/ncurses/patch_gcc_5.txt new file mode 100644 index 00000000000..f85e07cb8d1 --- /dev/null +++ b/var/spack/packages/ncurses/patch_gcc_5.txt @@ -0,0 +1,12 @@ +diff -Naur ncurses-6.0/ncurses/Makefile.in ncurses-6.0-patched/ncurses/Makefile.in +--- ncurses-6.0/ncurses/Makefile.in 2015-08-06 01:15:41.000000000 +0200 ++++ ncurses-6.0-patched/ncurses/Makefile.in 2015-12-15 14:58:52.710199407 +0100 +@@ -219,7 +219,7 @@ + $(SHELL) -e $(tinfo)/MKfallback.sh $(TERMINFO) $(TERMINFO_SRC) $(TIC_PATH) $(FALLBACK_LIST) >$@ + + ./lib_gen.c : $(base)/MKlib_gen.sh ../include/curses.h +- $(SHELL) -e $(base)/MKlib_gen.sh "$(CPP) $(CPPFLAGS)" "$(AWK)" generated <../include/curses.h >$@ ++ $(SHELL) -e $(base)/MKlib_gen.sh "$(CPP) $(CPPFLAGS) -P" "$(AWK)" generated <../include/curses.h >$@ + + init_keytry.h: make_keys$(BUILD_EXEEXT) keys.list + ./make_keys$(BUILD_EXEEXT) keys.list > $@ diff --git a/var/spack/packages/wx/package.py b/var/spack/packages/wx/package.py index 1813a8c8a5a..206fde77751 100644 --- a/var/spack/packages/wx/package.py +++ b/var/spack/packages/wx/package.py @@ -16,6 +16,8 @@ class Wx(Package): version('3.0.1', 'dad1f1cd9d4c370cbc22700dc492da31', url="https://sourceforge.net/projects/wxwindows/files/3.0.1/wxWidgets-3.0.1.tar.bz2") + depends_on('gtkplus') + def install(self, spec, prefix): configure("--prefix=%s" % prefix, "--enable-unicode", "--disable-precomp-headers") From 917fb328bb5d992b71b28c7fb86e2a0a82e06d02 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Tue, 15 Dec 2015 12:25:31 -0500 Subject: [PATCH 160/910] cmake: enable ssl, for https downloads By default cmake builds its own curl, without SSL support. This patch enables SSL when building cmake, fixing the following error: error: downloading 'https://...' failed status_code: 1 status_string: "Unsupported protocol" log: Protocol "https" not supported or disabled in libcurl --- var/spack/packages/cmake/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/cmake/package.py b/var/spack/packages/cmake/package.py index c24a80748c2..cb54c92d692 100644 --- a/var/spack/packages/cmake/package.py +++ b/var/spack/packages/cmake/package.py @@ -43,6 +43,7 @@ class Cmake(Package): def install(self, spec, prefix): configure('--prefix=' + prefix, - '--parallel=' + str(make_jobs)) + '--parallel=' + str(make_jobs), + '--', '-DCMAKE_USE_OPENSSL=ON') make() make('install') From 2f9ee884087005d90a9d63d29667cca1634fafca Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 15 Dec 2015 17:15:37 -0800 Subject: [PATCH 161/910] Fix #238: Database specs need to be concrete when read in as well. --- lib/spack/spack/database.py | 4 ++++ lib/spack/spack/directory_layout.py | 4 +--- lib/spack/spack/spec.py | 13 ++++++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index d62a47547d2..bf54055a241 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -211,6 +211,10 @@ def _read_spec_from_yaml(self, hash_key, installs, parent_key=None): child = self._read_spec_from_yaml(dep_hash, installs, hash_key) spec._add_dependency(child) + # Specs from the database need to be marked concrete because + # they represent actual installations. + spec._mark_concrete() + return spec diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index a434dad5c49..d91fbe9f4e9 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -212,9 +212,7 @@ def read_spec(self, path): spec = Spec.from_yaml(f) # Specs read from actual installations are always concrete - for s in spec.traverse(): - s._normal = True - s._concrete = True + spec._mark_concrete() return spec diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index fb5ea5d02b4..037ec97a5e8 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -833,7 +833,18 @@ def concretize(self): changed = any(changes) force=True - self._concrete = True + self._mark_concrete() + + + def _mark_concrete(self): + """Mark this spec and its dependencies as concrete. + + Only for internal use -- client code should use "concretize" + unless there is a need to force a spec to be concrete. + """ + for s in self.traverse(): + s._normal = True + s._concrete = True def concretized(self): From d5018e77a39bc25d654b4bc9bd057b7798889fb7 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 16 Dec 2015 09:32:48 +0100 Subject: [PATCH 162/910] gnuplot : modifications to build on RHLE 6 --- var/spack/packages/fontconfig/package.py | 3 ++- var/spack/packages/gnuplot/package.py | 4 +++- var/spack/packages/libgd/package.py | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/fontconfig/package.py b/var/spack/packages/fontconfig/package.py index 89b13604e89..517c9d18133 100644 --- a/var/spack/packages/fontconfig/package.py +++ b/var/spack/packages/fontconfig/package.py @@ -8,9 +8,10 @@ class Fontconfig(Package): version('2.11.1' , 'e75e303b4f7756c2b16203a57ac87eba') depends_on('freetype') + depends_on('libxml2') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + configure("--prefix=%s" % prefix, "--enable-libxml2") make() make("install") diff --git a/var/spack/packages/gnuplot/package.py b/var/spack/packages/gnuplot/package.py index e034cb09951..71c09bd43d2 100644 --- a/var/spack/packages/gnuplot/package.py +++ b/var/spack/packages/gnuplot/package.py @@ -45,7 +45,9 @@ class Gnuplot(Package): depends_on('libgd') depends_on('cairo') depends_on('pango') - depends_on('wx') + depends_on('wx', when='+wx') + + variant('wx', default=False, description='Activates wxWidgets terminal') def install(self, spec, prefix): # It seems there's an open bug for wxWidgets support diff --git a/var/spack/packages/libgd/package.py b/var/spack/packages/libgd/package.py index b066b43878f..d920957ef1c 100644 --- a/var/spack/packages/libgd/package.py +++ b/var/spack/packages/libgd/package.py @@ -39,6 +39,8 @@ class Libgd(Package): version('2.1.1', 'e91a1a99903e460e7ba00a794e72cc1e') + depends_on('libpng') + def install(self, spec, prefix): with working_dir('spack-build', create=True): From cc9266e56c3b72e707dc5b1370c1eca8b3014223 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 16 Dec 2015 12:45:58 +0100 Subject: [PATCH 163/910] gsl : added package --- var/spack/packages/gsl/package.py | 46 +++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 var/spack/packages/gsl/package.py diff --git a/var/spack/packages/gsl/package.py b/var/spack/packages/gsl/package.py new file mode 100644 index 00000000000..789eb49d854 --- /dev/null +++ b/var/spack/packages/gsl/package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Gsl(Package): + """ + The GNU Scientific Library (GSL) is a numerical library for C and C++ programmers. It is free software under the + GNU General Public License. The library provides a wide range of mathematical routines such as random number + generators, special functions and least-squares fitting. There are over 1000 functions in total with an extensive + test suite. + """ + homepage = "http://www.gnu.org/software/gsl" + url = "http://mirror.switch.ch/ftp/mirror/gnu/gsl/gsl-2.1.tar.gz" + + version('2.1' , 'd8f70abafd3e9f0bae03c52d1f4e8de5') + version('2.0' , 'ae44cdfed78ece40e73411b63a78c375') + version('1.16', 'e49a664db13d81c968415cd53f62bc8b') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") From 82aa366adfd5b8e0acd6d320c620bf33dd8e6a68 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Wed, 16 Dec 2015 11:43:07 -0500 Subject: [PATCH 164/910] minor comment tweak in modules.py --- lib/spack/spack/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 59ed9f893ec..7036626e29e 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -29,11 +29,11 @@ removed after an uninstall by post-uninstall hooks. This class consolidates the logic for creating an abstract description of the information that module systems need. Currently that includes a -number directories to be appended to paths in the user's environment: +number of directories to be appended to paths in the user's environment: * /bin directories to be appended to PATH * /lib* directories for LD_LIBRARY_PATH - * /man* and /share/man* directories for LD_LIBRARY_PATH + * /man* and /share/man* directories for MANPATH * the package prefix for CMAKE_PREFIX_PATH This module also includes logic for coming up with unique names for From 107c0dd1a8f56c92e02a78b258bdf5ccabf6a277 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Wed, 16 Dec 2015 15:54:15 -0800 Subject: [PATCH 165/910] added option to enable pdb debug --- bin/spack | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/spack b/bin/spack index ec6a80ff024..a0b260ccaf3 100755 --- a/bin/spack +++ b/bin/spack @@ -89,6 +89,8 @@ spec expressions: parser.add_argument('-d', '--debug', action='store_true', help="Write out debug logs during compile") +parser.add_argument('-D', '--pdb', action='store_true', + help="Run spack under the pdb debugger") parser.add_argument('-k', '--insecure', action='store_true', help="Do not check ssl certificates when downloading.") parser.add_argument('-m', '--mock', action='store_true', @@ -159,5 +161,8 @@ def main(): if args.profile: import cProfile cProfile.run('main()', sort='tottime') +elif args.pdb: + import pdb + pdb.run('main()') else: main() From 5d618886570e98e6745593701246b877998a5367 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 23 Nov 2015 16:51:16 -0500 Subject: [PATCH 166/910] Correct Spack cc script --- lib/spack/env/cc | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 75a63f6fac3..f3fadfcf45b 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -126,6 +126,11 @@ if [ -z "$mode" ]; then elif [ "$arg" = -c ]; then mode=cc break + elif [ "$arg" = -S ]; then + mode=as + echo "spac cc mode as" "$@" + exit 1 + break fi done fi @@ -155,6 +160,7 @@ libraries=() libs=() rpaths=() other_args=() +all_args=("$@") while [ -n "$1" ]; do case "$1" in @@ -178,6 +184,8 @@ while [ -n "$1" ]; do if [ -z "$arg" ]; then shift; arg="$1"; fi if [[ "$arg" = -rpath=* ]]; then rpaths+=("${arg#-rpath=}") + elif [[ "$arg" = -rpath,* ]]; then + rpaths+=("${arg#-rpath,}") elif [[ "$arg" = -rpath ]]; then shift; arg="$1" if [[ "$arg" != -Wl,* ]]; then @@ -193,6 +201,8 @@ while [ -n "$1" ]; do if [ -z "$arg" ]; then shift; arg="$1"; fi if [[ "$arg" = -rpath=* ]]; then rpaths+=("${arg#-rpath=}") + elif [[ "$arg" = -rpath,* ]]; then + rpaths+=("${arg#-rpath,}") elif [[ "$arg" = -rpath ]]; then shift; arg="$1" if [[ "$arg" != -Xlinker,* ]]; then @@ -247,25 +257,46 @@ IFS=':' read -ra deps <<< "$SPACK_DEPENDENCIES" for dep in "${deps[@]}"; do if [ -d "$dep/include" ]; then includes+=("$dep/include") + all_args=("-I$dep/include" ${all_args[@]}) fi if [ -d "$dep/lib" ]; then libraries+=("$dep/lib") rpaths+=("$dep/lib") + if [ "$mode" = ccld ]; then + all_args=("-L$dep/lib" "-Wl,-rpath,$dep/lib" ${all_args[@]}) + elif [ "$mode" = ld ]; then + all_args=("-L$dep/lib" "-rpath" "$dep/lib" ${all_args[@]}) + fi fi if [ -d "$dep/lib64" ]; then libraries+=("$dep/lib64") rpaths+=("$dep/lib64") + if [ "$mode" = ccld ]; then + all_args=("-L$dep/lib" "-Wl,-rpath,$dep/lib" ${all_args[@]}) + elif [ "$mode" = ld ]; then + all_args=("-L$dep/lib" "-rpath" "$dep/lib" ${all_args[@]}) + fi fi done # Include all -L's and prefix/whatever dirs in rpath for dir in "${libraries[@]}"; do [[ dir = $SPACK_INSTALL* ]] && rpaths+=("$dir") + if [ "$mode" = ccld ]; then + [[ dir = $SPACK_INSTALL* ]] && all_args=("-Wl,-rpath,$dir" ${all_args[@]}) + elif [ "$mode" = ld ]; then + [[ dir = $SPACK_INSTALL* ]] && all_args=("-rpath" "$dir" ${all_args[@]}) + fi done rpaths+=("$SPACK_PREFIX/lib") rpaths+=("$SPACK_PREFIX/lib64") +if [ "$mode" = ccld ]; then + all_args=("-Wl,-rpath,$SPACK_PREFIX/lib" "-Wl,-rpath,$SPACK_PREFIX/lib64" ${all_args[@]}) +elif [ "$mode" = ld ]; then + all_args=("-rpath" "$SPACK_PREFIX/lib" "-rpath" "$SPACK_PREFIX/lib64" ${all_args[@]}) +fi # Put the arguments together args=() @@ -317,7 +348,8 @@ done export PATH full_command=("$command") -full_command+=("${args[@]}") +# full_command+=("${args[@]}") +full_command+=("${all_args[@]}") # # Write the input and output commands to debug logs if it's asked for. From c02cbaecd8cad7154b5507c0122a5163544fd349 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Thu, 17 Dec 2015 07:41:49 -0800 Subject: [PATCH 167/910] Change back the previous boost change and change the mrnet variant name to lwthreads. --- var/spack/packages/boost/package.py | 3 +-- var/spack/packages/cbtf-argonavis/package.py | 7 ++++--- var/spack/packages/cbtf-krell/package.py | 4 ++-- var/spack/packages/cbtf-lanl/package.py | 2 +- var/spack/packages/cbtf/package.py | 7 +++---- var/spack/packages/mrnet/package.py | 4 ++-- var/spack/packages/openspeedshop/package.py | 6 +++--- 7 files changed, 16 insertions(+), 17 deletions(-) diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index fc721753c70..81dadbbf611 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -58,8 +58,7 @@ def url_for_version(self, version): parts = [str(p) for p in Version(version)] dots = ".".join(parts) underscores = "_".join(parts) - #return "http://downloads.sourceforge.net/project/boost/boost/%s/boost_%s.tar.bz2" % ( - return "http://downloads.sourceforge.net/project/boost/boost/%s.0/boost_%s_0.tar.bz2" % ( + return "http://downloads.sourceforge.net/project/boost/boost/%s/boost_%s.tar.bz2" % ( dots, underscores) def determine_toolset(self): diff --git a/var/spack/packages/cbtf-argonavis/package.py b/var/spack/packages/cbtf-argonavis/package.py index 2dccb0e0c6e..7b07933911a 100644 --- a/var/spack/packages/cbtf-argonavis/package.py +++ b/var/spack/packages/cbtf-argonavis/package.py @@ -33,7 +33,7 @@ class CbtfArgonavis(Package): depends_on("papi") depends_on("cbtf") depends_on("cbtf-krell") - #depends_on("cuda") + depends_on("cuda") parallel = False @@ -43,8 +43,8 @@ def install(self, spec, prefix): cmake_prefix_path = join_path(spec['cbtf'].prefix) + ':' + join_path(spec['cbtf-krell'].prefix) # FIXME, hard coded for testing purposes, we will alter when the external package feature is available - cuda_prefix_path = "/opt/cudatoolkit-6.5" - cupti_prefix_path = "/opt/cudatoolkit-6.5/extras/CUPTI" + cuda_prefix_path = "/usr/local/cudatoolkit-6.0" + cupti_prefix_path = "/usr/local/cudatoolkit-6.0/extras/CUPTI" with working_dir('CUDA'): @@ -56,6 +56,7 @@ def install(self, spec, prefix): '-DCUDA_INSTALL_PATH=%s' % cuda_prefix_path, '-DCUDA_ROOT=%s' % cuda_prefix_path, '-DCUPTI_ROOT=%s' % cupti_prefix_path, + '-DCUDA_DIR=%s' % cuda_prefix_path, '-DPAPI_ROOT=%s' % spec['papi'].prefix, '-DCBTF_PREFIX=%s' % spec['cbtf'].prefix, *std_cmake_args) diff --git a/var/spack/packages/cbtf-krell/package.py b/var/spack/packages/cbtf-krell/package.py index d9a2b7f8b53..9458ac113c9 100644 --- a/var/spack/packages/cbtf-krell/package.py +++ b/var/spack/packages/cbtf-krell/package.py @@ -38,9 +38,9 @@ class CbtfKrell(Package): depends_on("binutils@2.24+krellpatch") # collectionTool - depends_on("boost@1.50") + depends_on("boost@1.50.0") depends_on("dyninst@8.2.1") - depends_on("mrnet@4.1.0:+krelloptions") + depends_on("mrnet@4.1.0:+lwthreads") depends_on("xerces-c@3.1.1:") depends_on("cbtf") diff --git a/var/spack/packages/cbtf-lanl/package.py b/var/spack/packages/cbtf-lanl/package.py index 14651723cd3..2da9e8a1f74 100644 --- a/var/spack/packages/cbtf-lanl/package.py +++ b/var/spack/packages/cbtf-lanl/package.py @@ -32,7 +32,7 @@ class CbtfLanl(Package): # Dependencies for cbtf-krell depends_on("boost@1.50") - depends_on("mrnet@4.1.0+krelloptions") + depends_on("mrnet@4.1.0:+lwthreads") depends_on("xerces-c@3.1.1:") depends_on("cbtf") depends_on("cbtf-krell") diff --git a/var/spack/packages/cbtf/package.py b/var/spack/packages/cbtf/package.py index 6f1e49deed2..52e6a07020a 100644 --- a/var/spack/packages/cbtf/package.py +++ b/var/spack/packages/cbtf/package.py @@ -32,9 +32,9 @@ class Cbtf(Package): version('1.6', branch='master', git='http://git.code.sf.net/p/cbtf/cbtf') depends_on("cmake") - #depends_on("boost@1.42:") - depends_on("boost@1.50") - depends_on("mrnet@4.1.0+krelloptions") + #depends_on("boost@1.42.0:") + depends_on("boost@1.50.0") + depends_on("mrnet@4.1.0+lwthreads") depends_on("xerces-c@3.1.1:") depends_on("libxml2") @@ -43,7 +43,6 @@ class Cbtf(Package): def install(self, spec, prefix): with working_dir('build', create=True): - # Boost_NO_SYSTEM_PATHS Set to TRUE to suppress searching # in system paths (or other locations outside of BOOST_ROOT # or BOOST_INCLUDEDIR). Useful when specifying BOOST_ROOT. diff --git a/var/spack/packages/mrnet/package.py b/var/spack/packages/mrnet/package.py index 08c57d74369..3eb33d07140 100644 --- a/var/spack/packages/mrnet/package.py +++ b/var/spack/packages/mrnet/package.py @@ -8,14 +8,14 @@ class Mrnet(Package): version('4.0.0', 'd00301c078cba57ef68613be32ceea2f') version('4.1.0', '5a248298b395b329e2371bf25366115c') - variant('krelloptions', default=False, description="Also build the MRNet LW threadsafe libraries") + variant('lwthreads', default=False, description="Also build the MRNet LW threadsafe libraries") parallel = False depends_on("boost") def install(self, spec, prefix): # Build the MRNet LW thread safe libraries when the krelloptions variant is present - if '+krelloptions' in spec: + if '+lwthreads' in spec: configure("--prefix=%s" %prefix, "--enable-shared", "--enable-ltwt-threadsafe") else: configure("--prefix=%s" %prefix, "--enable-shared") diff --git a/var/spack/packages/openspeedshop/package.py b/var/spack/packages/openspeedshop/package.py index d439fba734c..8c71bcb7c3d 100644 --- a/var/spack/packages/openspeedshop/package.py +++ b/var/spack/packages/openspeedshop/package.py @@ -63,7 +63,7 @@ class Openspeedshop(Package): depends_on("libelf") depends_on("libdwarf") depends_on("sqlite") - depends_on("boost@1.50") + depends_on("boost@1.50.0") depends_on("dyninst@8.2.1") depends_on("python") depends_on("qt@3.3.8b+krellpatch") @@ -79,8 +79,8 @@ class Openspeedshop(Package): # Dependencies only for the openspeedshop cbtf package. depends_on("cbtf", when='+cbtf') depends_on("cbtf-krell", when='+cbtf') - #depends_on("cbtf-argonavis", when='+cbtf') - depends_on("mrnet@4.1.0:+krelloptions", when='+cbtf') + depends_on("cbtf-argonavis", when='+cbtf') + depends_on("mrnet@4.1.0:+lwthreads", when='+cbtf') def install(self, spec, prefix): From f91e1ae83fd60f575eb9e316125ca9a2a63be52e Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Thu, 17 Dec 2015 15:21:26 -0800 Subject: [PATCH 168/910] Added Lmod package --- var/spack/packages/lmod/package.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 var/spack/packages/lmod/package.py diff --git a/var/spack/packages/lmod/package.py b/var/spack/packages/lmod/package.py new file mode 100644 index 00000000000..d642594f928 --- /dev/null +++ b/var/spack/packages/lmod/package.py @@ -0,0 +1,26 @@ +from spack import * +import os + +class Lmod(Package): + """ + Lmod is a Lua based module system that easily handles the MODULEPATH + Hierarchical problem. Environment Modules provide a convenient way to + dynamically change the users' environment through modulefiles. This + includes easily adding or removing directories to the PATH environment + variable. Modulefiles for Library packages provide environment variables + that specify where the library and header files can be found. + """ + homepage = "https://www.tacc.utexas.edu/research-development/tacc-projects/lmod" + url = "http://sourceforge.net/projects/lmod/files/Lmod-6.0.1.tar.bz2/download" + + version('6.0.1', '91abf52fe5033bd419ffe2842ebe7af9') + + depends_on("lua@5.2:") + + def install(self, spec, prefix): + # Add our lua to PATH + os.environ['PATH'] = spec['lua'].prefix.bin + ';' + os.environ['PATH'] + + configure('--prefix=%s' % prefix) + make() + make("install") From 8dc32aeedb569400dffdd352ae7dcbeaa6c43635 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Dec 2015 12:58:29 +0100 Subject: [PATCH 169/910] Improved tau build and added optional dependencies --- var/spack/packages/cube/package.py | 3 + var/spack/packages/opari2/package.py | 2 + var/spack/packages/otf2/package.py | 4 + var/spack/packages/pdt/package.py | 45 ++++++++++ var/spack/packages/scorep/package.py | 124 +++++++++++++++------------ var/spack/packages/tau/package.py | 122 ++++++++++++++++++++++++-- 6 files changed, 236 insertions(+), 64 deletions(-) create mode 100644 var/spack/packages/pdt/package.py diff --git a/var/spack/packages/cube/package.py b/var/spack/packages/cube/package.py index d97cd25636c..cc1c684594f 100644 --- a/var/spack/packages/cube/package.py +++ b/var/spack/packages/cube/package.py @@ -12,6 +12,9 @@ class Cube(Package): homepage = "http://www.scalasca.org/software/cube-4.x/download.html" url = "http://apps.fz-juelich.de/scalasca/releases/cube/4.2/dist/cube-4.2.3.tar.gz" + version('4.3.3', '07e109248ed8ffc7bdcce614264a2909', + url='http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3.3.tar.gz') + version('4.2.3', '8f95b9531f5a8f8134f279c2767c9b20') version('4.3TP1', 'a2090fbc7b2ba394bd5c09ba971e237f', diff --git a/var/spack/packages/opari2/package.py b/var/spack/packages/opari2/package.py index daaee61e3a3..3f8c65377df 100644 --- a/var/spack/packages/opari2/package.py +++ b/var/spack/packages/opari2/package.py @@ -17,6 +17,8 @@ class Opari2(Package): homepage = "http://www.vi-hps.org/projects/score-p" url = "http://www.vi-hps.org/upload/packages/opari2/opari2-1.1.2.tar.gz" + version('1.1.4', '245d3d11147a06de77909b0805f530c0', + url='http://www.vi-hps.org/upload/packages/opari2/opari2-1.1.4.tar.gz') version('1.1.2', '9a262c7ca05ff0ab5f7775ae96f3539e') backend_user_provided = """\ diff --git a/var/spack/packages/otf2/package.py b/var/spack/packages/otf2/package.py index fa0a5898b6d..b3d3a5b6633 100644 --- a/var/spack/packages/otf2/package.py +++ b/var/spack/packages/otf2/package.py @@ -11,6 +11,10 @@ class Otf2(Package): homepage = "http://www.vi-hps.org/score-p" url = "http://www.vi-hps.org/upload/packages/otf2/otf2-1.4.tar.gz" + version('2.0', '5b546188b25bc1c4e285e06dddf75dfc', + url="http://www.vi-hps.org/upload/packages/otf2/otf2-2.0.tar.gz") + version('1.5.1', '16a9df46e0da78e374f5d12c8cdc1109', + url='http://www.vi-hps.org/upload/packages/otf2/otf2-1.5.1.tar.gz') version('1.4', 'a23c42e936eb9209c4e08b61c3cf5092', url="http://www.vi-hps.org/upload/packages/otf2/otf2-1.4.tar.gz") version('1.3.1', 'd0ffc4e858455ace4f596f910e68c9f2', diff --git a/var/spack/packages/pdt/package.py b/var/spack/packages/pdt/package.py new file mode 100644 index 00000000000..ce3b793e307 --- /dev/null +++ b/var/spack/packages/pdt/package.py @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Pdt(Package): + """ + Program Database Toolkit (PDT) is a framework for analyzing source code written in several programming languages + and for making rich program knowledge accessible to developers of static and dynamic analysis tools. PDT implements + a standard program representation, the program database (PDB), that can be accessed in a uniform way through a + class library supporting common PDB operations. + """ + homepage = "https://www.cs.uoregon.edu/research/pdt/home.php" + url = "https://www.cs.uoregon.edu/research/tau/pdt_releases/pdt-3.21.tar.gz" + + version('3.21', '8df94298b71703decf680709a4ddf68f') + version('3.19', 'ba5591994998771fdab216699e362228') + + def install(self, spec, prefix): + configure('-prefix=%s' % prefix) + make() + make("install") diff --git a/var/spack/packages/scorep/package.py b/var/spack/packages/scorep/package.py index f013bd1cbb7..fa9e339a490 100644 --- a/var/spack/packages/scorep/package.py +++ b/var/spack/packages/scorep/package.py @@ -1,74 +1,84 @@ -# FIXME: Add copyright statement +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## from spack import * + class Scorep(Package): - """The Score-P measurement infrastructure is a highly scalable and + """ + The Score-P measurement infrastructure is a highly scalable and easy-to-use tool suite for profiling, event tracing, and online analysis of HPC applications.""" - # FIXME: add a proper url for your package's homepage here. homepage = "http://www.vi-hps.org/projects/score-p" url = "http://www.vi-hps.org/upload/packages/scorep/scorep-1.2.3.tar.gz" - version('1.3', '9db6f957b7f51fa01377a9537867a55c', - url = 'http://www.vi-hps.org/upload/packages/scorep/scorep-1.3.tar.gz') + version('1.4.2', '3b9a042b13bdd5836452354e6567f71e', + url='http://www.vi-hps.org/upload/packages/scorep/scorep-1.4.2.tar.gz') + #version('1.3', '9db6f957b7f51fa01377a9537867a55c', + # url='http://www.vi-hps.org/upload/packages/scorep/scorep-1.3.tar.gz') + #version('1.2.3', '4978084e7cbd05b94517aa8beaea0817') - version('1.2.3', '4978084e7cbd05b94517aa8beaea0817') + ########## + # Dependencies for SCORE-P are quite tight. See the homepage for more information. + # SCOREP 1.4.2 + depends_on('otf2@1.5:1.6', when='@1.4.2') + depends_on('opari2@1.1.4', when='@1.4.2') + depends_on('cube@4.3:4.4', when='@1.4.2') + # SCOREP 1.3 + + ########## + + # FIXME : check version for 1.3 + #depends_on("otf2") + #depends_on("opari2") + #depends_on("cube") depends_on("mpi") depends_on("papi") - # depends_on("otf2@1.2:1.2.1") # only Score-P 1.2.x - depends_on("otf2") - depends_on("opari2") - depends_on("cube@4.2:4.2.3") - backend_user_provided = """\ -CC=cc -CXX=c++ -F77=f77 -FC=f90 -CFLAGS=-fPIC -CXXFLAGS=-fPIC -""" - frontend_user_provided = """\ -CC_FOR_BUILD=cc -CXX_FOR_BUILD=c++ -F77_FOR_BUILD=f70 -FC_FOR_BUILD=f90 -CFLAGS_FOR_BUILD=-fPIC -CXXFLAGS_FOR_BUILD=-fPIC -""" - mpi_user_provided = """\ -MPICC=mpicc -MPICXX=mpicxx -MPIF77=mpif77 -MPIFC=mpif90 -MPI_CFLAGS=-fPIC -MPI_CXXFLAGS=-fPIC -""" + def get_compiler_config_line(self): + backend_user_provided = ['CC=%s' % self.compiler.cc_names[0], + 'CXX=%s' % self.compiler.cxx_names[0], + 'F77=%s' % self.compiler.f77_names[0] if len(self.compiler.f77_names) else "", + 'FC=%s' % self.compiler.fc_names[0] if len(self.compiler.fc_names) else "", + 'CFLAGS=-fPIC %s' % self.rpath_args, + 'CXXFLAGS=-fPIC %s'% self.rpath_args] + return backend_user_provided def install(self, spec, prefix): - # Use a custom compiler configuration, otherwise the score-p - # build system messes with spack's compiler settings. - # Create these three files in the build directory - with open("platform-backend-user-provided", "w") as backend_file: - backend_file.write(self.backend_user_provided) - with open("platform-frontend-user-provided", "w") as frontend_file: - frontend_file.write(self.frontend_user_provided) - with open("platform-mpi-user-provided", "w") as mpi_file: - mpi_file.write(self.mpi_user_provided) - - configure_args = ["--prefix=%s" % prefix, - "--with-custom-compilers", - "--with-otf2=%s" % spec['otf2'].prefix.bin, - "--with-opari2=%s" % spec['opari2'].prefix.bin, - "--with-cube=%s" % spec['cube'].prefix.bin, - "--with-papi-header=%s" % spec['papi'].prefix.include, - "--with-papi-lib=%s" % spec['papi'].prefix.lib, - "--enable-shared"] - - configure(*configure_args) - - make() - make("install") + configure = Executable( join_path(self.stage.source_path, 'configure') ) + with working_dir('spack-build', create=True): + configure_args = ["--prefix=%s" % prefix, + "--with-otf2=%s" % spec['otf2'].prefix.bin, + "--with-opari2=%s" % spec['opari2'].prefix.bin, + "--with-cube=%s" % spec['cube'].prefix.bin, + "--with-papi-header=%s" % spec['papi'].prefix.include, + "--with-papi-lib=%s" % spec['papi'].prefix.lib, + "--enable-shared"] + configure_args.extend(self.get_compiler_config_line()) + configure(*configure_args) + make() + make("install") diff --git a/var/spack/packages/tau/package.py b/var/spack/packages/tau/package.py index 048fac80aa5..69f00ae4ea8 100644 --- a/var/spack/packages/tau/package.py +++ b/var/spack/packages/tau/package.py @@ -1,32 +1,140 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + from spack import * import os +import os.path + from llnl.util.filesystem import join_path class Tau(Package): - """A portable profiling and tracing toolkit for performance - analysis of parallel programs written in Fortran, C, C++, UPC, - Java, Python.""" + """ + A portable profiling and tracing toolkit for performance + analysis of parallel programs written in Fortran, C, C++, UPC, + Java, Python. + """ homepage = "http://www.cs.uoregon.edu/research/tau" - url = "http://www.cs.uoregon.edu/research/paracomp/tau/tauprofile/dist/tau-2.23.1.tar.gz" + url = "https://www.cs.uoregon.edu/research/tau/tau_releases/tau-2.25.tar.gz" + version('2.25', '46cd48fa3f3c4ce0197017b3158a2b43') + version('2.24.1', '6635ece6d1f08215b02f5d0b3c1e971b') + version('2.24', '57ce33539c187f2e5ec68f0367c76db4') version('2.23.1', '6593b47ae1e7a838e632652f0426fe72') + # FIXME : shmem variant missing + variant('download', default=True, description='Downloads and builds various dependencies') + variant('openmp', default=True, description='Use OpenMP threads') + variant('mpi', default=True, description='Specify use of TAU MPI wrapper library') + variant('phase', default=True, description='Generate phase based profiles') + variant('comm', default=True, description=' Generate profiles with MPI communicator info') + + depends_on('pdt') # Required for TAU instrumentation + depends_on('scorep') + #depends_on('otf2', when='%gcc') # FIXME : still missing logic for intel and other compilers + depends_on('binutils', when='~download') + depends_on('mpi', when='+mpi') + + def set_compiler_options(self): + + name = self.compiler.name # Name of the current toolchain + useropt = ["-O2", self.rpath_args] + + ########## + # Selecting a compiler with TAU configure is quite tricky: + # 1 - compilers are mapped to a given set of strings (and spack cc, cxx, etc. wrappers are not among them) + # 2 - absolute paths are not allowed + # 3 - the usual environment variables seems not to be checked ('CC', 'CXX' and 'FC') + # 4 - if no -cc= -cxx= is passed tau is built with system compiler silently + # (regardless of what % is used in the spec) + # + # In the following we give TAU what he expects and put compilers into PATH + compiler_path = os.path.dirname(self.compiler.cc) + os.environ['PATH'] = ':'.join([compiler_path, os.environ['PATH']]) + compiler_options = ['-c++=%s' % self.compiler.cxx_names[0], + '-cc=%s' % self.compiler.cc_names[0]] + if self.compiler.fc: + compiler_options.append('-fortran=%s' % self.compiler.fc_names[0]) + # It turns out that some parts of the OTF support library in TAU are non-conformant, so if we want + # to compile them we must suppress some warnings. With Clang it seems impossible to do so, so OTF + # support is not built + #if name == 'gcc': + # useropt.append("-fpermissive") + ########## + + # Construct the string of custom compiler flags and append it to compiler related options + useropt = ' '.join(useropt) + useropt = "-useropt=%s" % useropt + compiler_options.append(useropt) + return compiler_options + def install(self, spec, prefix): # TAU isn't happy with directories that have '@' in the path. Sigh. change_sed_delimiter('@', ';', 'configure') change_sed_delimiter('@', ';', 'utils/FixMakefile') change_sed_delimiter('@', ';', 'utils/FixMakefile.sed.default') - # After that, it's relatively standard. - configure("-prefix=%s" % prefix) + # TAU configure, despite the name , seems to be a manually written script (nothing related to autotools). + # As such it has a few #peculiarities# that make this build quite hackish. + options = ["-prefix=%s" % prefix, + "-iowrapper", + "-pdt=%s" % spec['pdt'].prefix, + "-scorep=%s" % spec['scorep'].prefix] + # FIXME : I give up trying to compile OTF support with clang, as it seems there is no easy way to ignore errors + #if 'otf2' in spec: + # options.append("-otf=%s" % spec['otf2'].prefix) + # If download is active, download and build suggested dependencies + if '+download' in spec: + options.extend(['-bfd=download', + '-unwind=download', + '-asmdex=download']) + else: + options.extend(["-bfd=%s" % spec['binutils'].prefix]) + # TODO : unwind and asmdex are still missing + + if '+openmp' in spec: + options.append('-openmp') + + if '+mpi' in spec: + options.append('-mpi') + + if '+phase' in spec: + options.append('-PROFILEPHASE') + + if '+comm' in spec: + options.append('-PROFILECOMMUNICATORS') + + compiler_specific_options = self.set_compiler_options() + options.extend(compiler_specific_options) + configure(*options) make("install") # Link arch-specific directories into prefix since there is # only one arch per prefix the way spack installs. self.link_tau_arch_dirs() - def link_tau_arch_dirs(self): for subdir in os.listdir(self.prefix): for d in ('bin', 'lib'): From f3837dd6867da8483c8981c3134dcc24dca9580b Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Dec 2015 16:15:30 +0100 Subject: [PATCH 170/910] tau : removed direct otf2 support --- var/spack/packages/scorep/package.py | 14 +++++--------- var/spack/packages/tau/package.py | 16 ++++------------ 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/var/spack/packages/scorep/package.py b/var/spack/packages/scorep/package.py index fa9e339a490..0820f2d8ac3 100644 --- a/var/spack/packages/scorep/package.py +++ b/var/spack/packages/scorep/package.py @@ -37,9 +37,8 @@ class Scorep(Package): version('1.4.2', '3b9a042b13bdd5836452354e6567f71e', url='http://www.vi-hps.org/upload/packages/scorep/scorep-1.4.2.tar.gz') - #version('1.3', '9db6f957b7f51fa01377a9537867a55c', - # url='http://www.vi-hps.org/upload/packages/scorep/scorep-1.3.tar.gz') - #version('1.2.3', '4978084e7cbd05b94517aa8beaea0817') + version('1.3', '9db6f957b7f51fa01377a9537867a55c', + url='http://www.vi-hps.org/upload/packages/scorep/scorep-1.3.tar.gz') ########## # Dependencies for SCORE-P are quite tight. See the homepage for more information. @@ -48,14 +47,11 @@ class Scorep(Package): depends_on('opari2@1.1.4', when='@1.4.2') depends_on('cube@4.3:4.4', when='@1.4.2') # SCOREP 1.3 - + depends_on("otf2@1.4", when='@1.3') + depends_on("opari2@1.1.4", when='@1.3') + depends_on("cube@4.2.3", when='@1.3') ########## - # FIXME : check version for 1.3 - #depends_on("otf2") - #depends_on("opari2") - #depends_on("cube") - depends_on("mpi") depends_on("papi") diff --git a/var/spack/packages/tau/package.py b/var/spack/packages/tau/package.py index 69f00ae4ea8..9febb647b39 100644 --- a/var/spack/packages/tau/package.py +++ b/var/spack/packages/tau/package.py @@ -44,22 +44,22 @@ class Tau(Package): version('2.24', '57ce33539c187f2e5ec68f0367c76db4') version('2.23.1', '6593b47ae1e7a838e632652f0426fe72') - # FIXME : shmem variant missing - variant('download', default=True, description='Downloads and builds various dependencies') + # TODO : shmem variant missing + variant('download', default=False, description='Downloads and builds various dependencies') variant('openmp', default=True, description='Use OpenMP threads') variant('mpi', default=True, description='Specify use of TAU MPI wrapper library') variant('phase', default=True, description='Generate phase based profiles') variant('comm', default=True, description=' Generate profiles with MPI communicator info') + # TODO : Try to build direct OTF2 support? Some parts of the OTF support library in TAU are non-conformant, + # TODO : and fail at compile-time. Further, SCOREP is compiled with OTF2 support. depends_on('pdt') # Required for TAU instrumentation depends_on('scorep') - #depends_on('otf2', when='%gcc') # FIXME : still missing logic for intel and other compilers depends_on('binutils', when='~download') depends_on('mpi', when='+mpi') def set_compiler_options(self): - name = self.compiler.name # Name of the current toolchain useropt = ["-O2", self.rpath_args] ########## @@ -77,11 +77,6 @@ def set_compiler_options(self): '-cc=%s' % self.compiler.cc_names[0]] if self.compiler.fc: compiler_options.append('-fortran=%s' % self.compiler.fc_names[0]) - # It turns out that some parts of the OTF support library in TAU are non-conformant, so if we want - # to compile them we must suppress some warnings. With Clang it seems impossible to do so, so OTF - # support is not built - #if name == 'gcc': - # useropt.append("-fpermissive") ########## # Construct the string of custom compiler flags and append it to compiler related options @@ -102,9 +97,6 @@ def install(self, spec, prefix): "-iowrapper", "-pdt=%s" % spec['pdt'].prefix, "-scorep=%s" % spec['scorep'].prefix] - # FIXME : I give up trying to compile OTF support with clang, as it seems there is no easy way to ignore errors - #if 'otf2' in spec: - # options.append("-otf=%s" % spec['otf2'].prefix) # If download is active, download and build suggested dependencies if '+download' in spec: options.extend(['-bfd=download', From abd3d3946d8a4d5b73f26a682c1728cf9d4d05df Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 18 Dec 2015 19:23:45 -0800 Subject: [PATCH 171/910] Format to 80 char width. --- lib/spack/spack/cmd/uninstall.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 191d9d88e8d..03873bb5f80 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -42,9 +42,9 @@ def setup_parser(subparser): help="Remove regardless of whether other packages depend on this one.") subparser.add_argument( '-a', '--all', action='store_true', dest='all', - help="USE CAREFULLY. Remove ALL installed packages that match each supplied spec. " + - "i.e., if you say uninstall libelf, ALL versions of libelf are uninstalled. " + - "This is both useful and dangerous, like rm -r.") + help="USE CAREFULLY. Remove ALL installed packages that match each " + + "supplied spec. i.e., if you say uninstall libelf, ALL versions of " + + "libelf are uninstalled. This is both useful and dangerous, like rm -r.") subparser.add_argument( 'packages', nargs=argparse.REMAINDER, help="specs of packages to uninstall") @@ -81,7 +81,8 @@ def uninstall(parser, args): pkgs.append(s.package) except spack.packages.UnknownPackageError, e: - # The package.py file has gone away -- but still want to uninstall. + # The package.py file has gone away -- but still want to + # uninstall. spack.Package(s).do_uninstall(force=True) # Sort packages to be uninstalled by the number of installed dependents From 3e1ba6cc246d126bb3f900ed27192902bb1f57fc Mon Sep 17 00:00:00 2001 From: John Westlund Date: Fri, 18 Dec 2015 21:28:44 -0800 Subject: [PATCH 172/910] remove possible trailing slash in _sp_prefix or _sp_share_dir --- share/spack/setup-env.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 9f4ddd3fafa..5b03aa89555 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -175,6 +175,6 @@ _sp_share_dir=$(cd "$(dirname $_sp_source_file)" && pwd) _sp_prefix=$(cd "$(dirname $(dirname $_sp_share_dir))" && pwd) # TODO: fix SYS_TYPE to something non-LLNL-specific -_spack_pathadd DK_NODE "$_sp_share_dir/dotkit/$SYS_TYPE" -_spack_pathadd MODULEPATH "$_sp_share_dir/modules/$SYS_TYPE" -_spack_pathadd PATH "$_sp_prefix/bin" +_spack_pathadd DK_NODE "${_sp_share_dir%/}/dotkit/$SYS_TYPE" +_spack_pathadd MODULEPATH "${_sp_share_dir%/}/modules/$SYS_TYPE" +_spack_pathadd PATH "${_sp_prefix%/}/bin" From ce8bd2094403ca5844f164e65b22ad4cfdab4a9a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 18 Dec 2015 22:23:51 -0800 Subject: [PATCH 173/910] Fix import error. --- lib/spack/spack/database.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index bf54055a241..a6f1cc50777 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -54,6 +54,7 @@ from spack.version import Version from spack.spec import Spec from spack.error import SpackError +from spack.packages import UnknownPackageError # DB goes in this directory underneath the root _db_dirname = '.spack-db' From b7e926eef6a248ec6f67cf30bdb2ce7dea09b8a2 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 18 Dec 2015 22:24:22 -0800 Subject: [PATCH 174/910] minor formatting. --- lib/spack/spack/cmd/extensions.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index c2cf288877c..2ce6f406cae 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -54,7 +54,9 @@ def extensions(parser, args): if not args.spec: tty.die("extensions requires a package spec.") + # # Checks + # spec = spack.cmd.parse_specs(args.spec) if len(spec) > 1: tty.die("Can only list extensions for one package.") @@ -70,7 +72,9 @@ def extensions(parser, args): if not args.mode: args.mode = 'short' + # # List package names of extensions + # extensions = spack.db.extensions_for(spec) if not extensions: tty.msg("%s has no extensions." % spec.cshort_spec) @@ -79,7 +83,9 @@ def extensions(parser, args): tty.msg("%d extensions:" % len(extensions)) colify(ext.name for ext in extensions) + # # List specs of installed extensions. + # installed = [s.spec for s in spack.installed_db.installed_extensions_for(spec)] print if not installed: @@ -88,7 +94,9 @@ def extensions(parser, args): tty.msg("%d installed:" % len(installed)) spack.cmd.find.display_specs(installed, mode=args.mode) + # # List specs of activated extensions. + # activated = spack.install_layout.extension_map(spec) print if not activated: From 52e3364efa4fa18d392316371e7164bf3899a638 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 18 Dec 2015 22:24:35 -0800 Subject: [PATCH 175/910] Fix #244: errors on uninstall - Extension logic didn't take conditional deps into account. - Extension methods now check for whether the extnesion is in the extendee map AND whether the dependency is actually present in the spec yet. --- lib/spack/spack/hooks/extensions.py | 4 +--- lib/spack/spack/package.py | 28 ++++++++++++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py index b4847d697f5..627184cabdd 100644 --- a/lib/spack/spack/hooks/extensions.py +++ b/lib/spack/spack/hooks/extensions.py @@ -27,9 +27,7 @@ def pre_uninstall(pkg): - # Need to do this b/c uninstall does not automatically do it. - # TODO: store full graph info in stored .spec file. - pkg.spec.normalize() + assert(pkg.spec.concrete) if pkg.is_extension: if pkg.activated: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index daba5cd352d..4d75726e06e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -487,9 +487,15 @@ def extendee_spec(self): if name == dep.name: return dep - # Otherwise return the spec from the extends() directive - spec, kwargs = self.extendees[name] - return spec + # if the spec is concrete already, then it extends something + # that is an *optional* dependency, and the dep isn't there. + if self.spec._concrete: + return None + else: + # If it's not concrete, then return the spec from the + # extends() directive since that is all we know so far. + spec, kwargs = self.extendees[name] + return spec @property @@ -497,18 +503,28 @@ def extendee_args(self): """Spec of the extendee of this package, or None if it is not an extension.""" if not self.extendees: return None + + # TODO: allow multiple extendees. name = next(iter(self.extendees)) return self.extendees[name][1] @property def is_extension(self): - return len(self.extendees) > 0 + # if it is concrete, it's only an extension if it actually + # dependes on the extendee. + if self.spec._concrete: + return self.extendee_spec is not None + else: + # If not, then it's an extension if it *could* be an extension + return bool(self.extendees) def extends(self, spec): - return (spec.name in self.extendees and - spec.satisfies(self.extendees[spec.name][0])) + if not spec.name in self.extendees: + return False + s = self.extendee_spec + return s and s.satisfies(spec) @property From 7cc4d88726501d6d93adf86f8080e65523e2400f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 19 Dec 2015 02:06:47 -0800 Subject: [PATCH 176/910] Fix #254: libtool & distutils want certain compiler names. This changes the compiler wrappers so that they are called by the same name as the wrapped compiler. Many builds make assumptions about compiler names, and we need the spack compilers to be recognizable so that build systems will get their flags right. This adds per-compiler subdirectories to lib/spack/spack/env directory that contain symlinks to cc for the C, C++, F77, and F90 compilers. The build now sets CC, CXX, F77, and F90 to point to these links instead of to the generically named cc, c++, f77, and f90 wrappers. --- lib/spack/env/cc | 14 +++++------ lib/spack/env/clang/clang | 1 + lib/spack/env/clang/clang++ | 1 + lib/spack/env/gcc/g++ | 1 + lib/spack/env/gcc/gcc | 1 + lib/spack/env/gcc/gfortran | 1 + lib/spack/env/intel/icc | 1 + lib/spack/env/intel/icpc | 1 + lib/spack/env/intel/ifort | 1 + lib/spack/env/pgi/case-insensitive/pgCC | 1 + lib/spack/env/pgi/pgcc | 1 + lib/spack/env/pgi/pgf77 | 1 + lib/spack/env/pgi/pgf90 | 1 + lib/spack/env/xl/xlc | 1 + lib/spack/env/xl/xlc++ | 1 + lib/spack/env/xl/xlf | 1 + lib/spack/env/xl/xlf90 | 1 + lib/spack/spack/build_environment.py | 32 ++++++++++++++++++------- lib/spack/spack/compilers/clang.py | 6 +++++ lib/spack/spack/compilers/gcc.py | 6 +++++ lib/spack/spack/compilers/intel.py | 6 +++++ lib/spack/spack/compilers/pgi.py | 6 +++++ lib/spack/spack/compilers/xl.py | 6 +++++ 23 files changed, 77 insertions(+), 15 deletions(-) create mode 120000 lib/spack/env/clang/clang create mode 120000 lib/spack/env/clang/clang++ create mode 120000 lib/spack/env/gcc/g++ create mode 120000 lib/spack/env/gcc/gcc create mode 120000 lib/spack/env/gcc/gfortran create mode 120000 lib/spack/env/intel/icc create mode 120000 lib/spack/env/intel/icpc create mode 120000 lib/spack/env/intel/ifort create mode 120000 lib/spack/env/pgi/case-insensitive/pgCC create mode 120000 lib/spack/env/pgi/pgcc create mode 120000 lib/spack/env/pgi/pgf77 create mode 120000 lib/spack/env/pgi/pgf90 create mode 120000 lib/spack/env/xl/xlc create mode 120000 lib/spack/env/xl/xlc++ create mode 120000 lib/spack/env/xl/xlf create mode 120000 lib/spack/env/xl/xlf90 diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 75a63f6fac3..0966277a91e 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -86,22 +86,22 @@ done # command=$(basename "$0") case "$command" in - cc|gcc|c89|c99|clang|xlc) + cc|c89|c99|gcc|clang|icc|pgcc|xlc) command="$SPACK_CC" language="C" ;; - c++|CC|g++|clang++|xlC) + c++|CC|g++|clang++|icpc|pgCC|xlc++) command="$SPACK_CXX" language="C++" ;; - f77|xlf) - command="$SPACK_F77" - language="Fortran 77" - ;; - fc|f90|f95|xlf90) + f90|fc|f95|gfortran|ifort|pgf90|xlf90) command="$SPACK_FC" language="Fortran 90" ;; + f77|gfortran|ifort|pgf77|xlf) + command="$SPACK_F77" + language="Fortran 77" + ;; cpp) mode=cpp ;; diff --git a/lib/spack/env/clang/clang b/lib/spack/env/clang/clang new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/clang/clang @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/clang/clang++ b/lib/spack/env/clang/clang++ new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/clang/clang++ @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/gcc/g++ b/lib/spack/env/gcc/g++ new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/gcc/g++ @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/gcc/gcc b/lib/spack/env/gcc/gcc new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/gcc/gcc @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/gcc/gfortran b/lib/spack/env/gcc/gfortran new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/gcc/gfortran @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/intel/icc b/lib/spack/env/intel/icc new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/intel/icc @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/intel/icpc b/lib/spack/env/intel/icpc new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/intel/icpc @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/intel/ifort b/lib/spack/env/intel/ifort new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/intel/ifort @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/pgi/case-insensitive/pgCC b/lib/spack/env/pgi/case-insensitive/pgCC new file mode 120000 index 00000000000..e2deb67f3b6 --- /dev/null +++ b/lib/spack/env/pgi/case-insensitive/pgCC @@ -0,0 +1 @@ +../../cc \ No newline at end of file diff --git a/lib/spack/env/pgi/pgcc b/lib/spack/env/pgi/pgcc new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/pgi/pgcc @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/pgi/pgf77 b/lib/spack/env/pgi/pgf77 new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/pgi/pgf77 @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/pgi/pgf90 b/lib/spack/env/pgi/pgf90 new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/pgi/pgf90 @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/xl/xlc b/lib/spack/env/xl/xlc new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/xl/xlc @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/xl/xlc++ b/lib/spack/env/xl/xlc++ new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/xl/xlc++ @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/xl/xlf b/lib/spack/env/xl/xlf new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/xl/xlf @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/xl/xlf90 b/lib/spack/env/xl/xlf90 new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/xl/xlf90 @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index dac25d99401..20154a55b3d 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -88,10 +88,14 @@ def set_compiler_environment_variables(pkg): compiler = pkg.compiler # Set compiler variables used by CMake and autotools - os.environ['CC'] = join_path(spack.build_env_path, 'cc') - os.environ['CXX'] = join_path(spack.build_env_path, 'c++') - os.environ['F77'] = join_path(spack.build_env_path, 'f77') - os.environ['FC'] = join_path(spack.build_env_path, 'f90') + assert all(key in pkg.compiler.link_paths + for key in ('cc', 'cxx', 'f77', 'fc')) + + link_dir = spack.build_env_path + os.environ['CC'] = join_path(link_dir, pkg.compiler.link_paths['cc']) + os.environ['CXX'] = join_path(link_dir, pkg.compiler.link_paths['cxx']) + os.environ['F77'] = join_path(link_dir, pkg.compiler.link_paths['f77']) + os.environ['FC'] = join_path(link_dir, pkg.compiler.link_paths['fc']) # Set SPACK compiler variables so that our wrapper knows what to call if compiler.cc: @@ -110,11 +114,23 @@ def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. """ # Add spack build environment path with compiler wrappers first in - # the path. We handle case sensitivity conflicts like "CC" and - # "cc" by putting one in the /case-insensitive + # the path. We add both spack.env_path, which includes default + # wrappers (cc, c++, f77, f90), AND a subdirectory containing + # compiler-specific symlinks. The latter ensures that builds that + # are sensitive to the *name* of the compiler see the right name + # when we're building wtih the wrappers. + # + # Conflicts on case-insensitive systems (like "CC" and "cc") are + # handled by putting one in the /case-insensitive # directory. Add that to the path too. - env_paths = [spack.build_env_path, - join_path(spack.build_env_path, 'case-insensitive')] + env_paths = [] + def add_env_path(path): + env_paths.append(path) + ci = join_path(path, 'case-insensitive') + if os.path.isdir(ci): env_paths.append(ci) + add_env_path(spack.build_env_path) + add_env_path(join_path(spack.build_env_path, pkg.compiler.name)) + path_put_first("PATH", env_paths) path_set(SPACK_ENV_PATH, env_paths) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index b34d1f2f9c7..340051019cc 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -37,6 +37,12 @@ class Clang(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = [] + # Named wrapper links within spack.build_env_path + link_paths = { 'cc' : 'clang/clang', + 'cxx' : 'clang/clang++', + # Use default wrappers for fortran, in case provided in compilers.yaml + 'f77' : 'f77', + 'fc' : 'f90' } @classmethod def default_version(self, comp): diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 2886888d578..495b638a3aa 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -42,6 +42,12 @@ class Gcc(Compiler): # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. suffixes = [r'-mp-\d\.\d'] + # Named wrapper links within spack.build_env_path + link_paths = {'cc' : 'gcc/gcc', + 'cxx' : 'gcc/g++', + 'f77' : 'gcc/gfortran', + 'fc' : 'gcc/gfortran' } + @property def cxx11_flag(self): if self.version < ver('4.3'): diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 12984299746..69e97647906 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -37,6 +37,12 @@ class Intel(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['ifort'] + # Named wrapper links within spack.build_env_path + link_paths = { 'cc' : 'intel/icc', + 'cxx' : 'intel/icpc', + 'f77' : 'intel/ifort', + 'fc' : 'intel/ifort' } + @property def cxx11_flag(self): if self.version < ver('11.1'): diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 6999eb50dea..9ac74cfbdb5 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -37,6 +37,12 @@ class Pgi(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['pgf95', 'pgf90'] + # Named wrapper links within spack.build_env_path + link_paths = { 'cc' : 'pgi/pgcc', + 'cxx' : 'pgi/case-insensitive/pgCC', + 'f77' : 'pgi/pgf77', + 'fc' : 'pgi/pgf90' } + @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index 308f811eb4a..c1d55109a37 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -38,6 +38,12 @@ class Xl(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['xlf90','xlf90_r','xlf95','xlf95_r','xlf2003','xlf2003_r','xlf2008','xlf2008_r'] + # Named wrapper links within spack.build_env_path + link_paths = { 'cc' : 'xl/xlc', + 'cxx' : 'xl/xlc++', + 'f77' : 'xl/xlf', + 'fc' : 'xl/xlf90' } + @property def cxx11_flag(self): if self.version < ver('13.1'): From 8dd1f3ffa59dfc9a36572e3f3c4b80a3cdad4f97 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Sat, 19 Dec 2015 07:43:34 -0800 Subject: [PATCH 177/910] added mrnet 5.0.1 --- var/spack/packages/mrnet/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/mrnet/package.py b/var/spack/packages/mrnet/package.py index 3eb33d07140..fed944e45f3 100644 --- a/var/spack/packages/mrnet/package.py +++ b/var/spack/packages/mrnet/package.py @@ -7,6 +7,7 @@ class Mrnet(Package): version('4.0.0', 'd00301c078cba57ef68613be32ceea2f') version('4.1.0', '5a248298b395b329e2371bf25366115c') + version('5.0.1', '17f65738cf1b9f9b95647ff85f69ecdd') variant('lwthreads', default=False, description="Also build the MRNet LW threadsafe libraries") parallel = False From 429e15c4a64d29926f1b7a7168c23592e7dec0b7 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Mon, 14 Dec 2015 15:22:51 -0500 Subject: [PATCH 178/910] spack python: add -c option Allows passing program in as a string. Example: $ spack python -c 'print 2+3' 5 Also imports spack module by default into the environment. --- lib/spack/spack/cmd/python.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/python.py b/lib/spack/spack/cmd/python.py index e26b8d3e797..5325e8fd9a5 100644 --- a/lib/spack/spack/cmd/python.py +++ b/lib/spack/spack/cmd/python.py @@ -31,6 +31,8 @@ import spack def setup_parser(subparser): + subparser.add_argument( + '-c', dest='python_command', help='Command to execute.') subparser.add_argument( 'python_args', nargs=argparse.REMAINDER, help="File to run plus arguments.") @@ -38,7 +40,8 @@ def setup_parser(subparser): def python(parser, args): # Fake a main python shell by setting __name__ to __main__. - console = code.InteractiveConsole({'__name__' : '__main__'}) + console = code.InteractiveConsole({'__name__' : '__main__', + 'spack' : spack}) if "PYTHONSTARTUP" in os.environ: startup_file = os.environ["PYTHONSTARTUP"] @@ -47,7 +50,10 @@ def python(parser, args): console.runsource(startup.read(), startup_file, 'exec') python_args = args.python_args - if python_args: + python_command = args.python_command + if python_command: + console.runsource(python_command) + elif python_args: sys.argv = python_args with open(python_args[0]) as file: console.runsource(file.read(), python_args[0], 'exec') From 527154e6dfa0c9e5f35aed56adad3279bac4eac3 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Mon, 14 Dec 2015 15:36:53 -0500 Subject: [PATCH 179/910] fix env module sys_type on non-LLNL systems --- share/spack/setup-env.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 5b03aa89555..47202f6087a 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -173,8 +173,8 @@ fi # _sp_share_dir=$(cd "$(dirname $_sp_source_file)" && pwd) _sp_prefix=$(cd "$(dirname $(dirname $_sp_share_dir))" && pwd) - -# TODO: fix SYS_TYPE to something non-LLNL-specific -_spack_pathadd DK_NODE "${_sp_share_dir%/}/dotkit/$SYS_TYPE" -_spack_pathadd MODULEPATH "${_sp_share_dir%/}/modules/$SYS_TYPE" _spack_pathadd PATH "${_sp_prefix%/}/bin" + +_sp_sys_type=$(spack-python -c 'print(spack.architecture.sys_type())') +_spack_pathadd DK_NODE "${_sp_share_dir%/}/dotkit/$_sp_sys_type" +_spack_pathadd MODULEPATH "${_sp_share_dir%/}/modules/$_sp_sys_type" From e403576cf9862d409de3be57dc2cabb1caad47c5 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Sat, 19 Dec 2015 18:38:23 -0500 Subject: [PATCH 180/910] new package: cfitsio --- var/spack/packages/cfitsio/package.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 var/spack/packages/cfitsio/package.py diff --git a/var/spack/packages/cfitsio/package.py b/var/spack/packages/cfitsio/package.py new file mode 100644 index 00000000000..8ba3a469e82 --- /dev/null +++ b/var/spack/packages/cfitsio/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Cfitsio(Package): + """ + CFITSIO is a library of C and Fortran subroutines for reading and writing + data files in FITS (Flexible Image Transport System) data format. + """ + homepage = 'http://heasarc.gsfc.nasa.gov/fitsio/' + version('3.370', 'abebd2d02ba5b0503c633581e3bfa116') + + def url_for_version(self, v): + url = 'ftp://heasarc.gsfc.nasa.gov/software/fitsio/c/cfitsio{}.tar.gz' + return url.format(str(v).replace('.', '')) + + def install(self, spec, prefix): + configure('--prefix=' + prefix) + make() + make('install') + From 32f96b5c73ff25d2a206423b514c831c2ab074fa Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 19 Dec 2015 20:04:00 -0800 Subject: [PATCH 181/910] Fix stale code in python version test so that it actually works. --- lib/spack/external/pyqver2.py | 3 ++- lib/spack/spack/test/python_version.py | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/spack/external/pyqver2.py b/lib/spack/external/pyqver2.py index 4a16e2811eb..46902397485 100755 --- a/lib/spack/external/pyqver2.py +++ b/lib/spack/external/pyqver2.py @@ -30,7 +30,8 @@ StandardModules = { "__future__": (2, 1), "abc": (2, 6), - "argparse": (2, 7), +# skip argparse now that it's in lib/spack/external +# "argparse": (2, 7), "ast": (2, 6), "atexit": (2, 0), "bz2": (2, 3), diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index ba570b416f8..2ea5febb11d 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -63,10 +63,6 @@ def check_python_versions(self, *files): all_issues = {} for fn in files: - if fn != '/Users/gamblin2/src/spack/var/spack/packages/vim/package.py': - continue - print fn - with open(fn) as pyfile: versions = pyqver2.get_versions(pyfile.read()) for ver, reasons in versions.items(): From 3d3bea618e1e72c713794e82b9bb190d0ebf8624 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 19 Dec 2015 20:08:14 -0800 Subject: [PATCH 182/910] Fix empty format strings in some packages. --- var/spack/packages/cfitsio/package.py | 3 +-- var/spack/packages/global/package.py | 6 +++--- var/spack/packages/tmuxinator/package.py | 5 ++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/var/spack/packages/cfitsio/package.py b/var/spack/packages/cfitsio/package.py index 8ba3a469e82..ff450cb5f35 100644 --- a/var/spack/packages/cfitsio/package.py +++ b/var/spack/packages/cfitsio/package.py @@ -9,11 +9,10 @@ class Cfitsio(Package): version('3.370', 'abebd2d02ba5b0503c633581e3bfa116') def url_for_version(self, v): - url = 'ftp://heasarc.gsfc.nasa.gov/software/fitsio/c/cfitsio{}.tar.gz' + url = 'ftp://heasarc.gsfc.nasa.gov/software/fitsio/c/cfitsio{0}.tar.gz' return url.format(str(v).replace('.', '')) def install(self, spec, prefix): configure('--prefix=' + prefix) make() make('install') - diff --git a/var/spack/packages/global/package.py b/var/spack/packages/global/package.py index a77b1bdc09b..e8f06516d9a 100644 --- a/var/spack/packages/global/package.py +++ b/var/spack/packages/global/package.py @@ -4,7 +4,7 @@ class Global(Package): """ The Gnu Global tagging system """ - # FIXME: add a proper url for your package's homepage here. + homepage = "http://www.gnu.org/software/global" url = "http://tamacom.com/global/global-6.5.tar.gz" @@ -13,9 +13,9 @@ class Global(Package): depends_on('exuberant-ctags') def install(self, spec, prefix): - config_args = ['--prefix={}'.format(prefix)] + config_args = ['--prefix={0}'.format(prefix)] - config_args.append('--with-exuberant-ctags={}'.format( + config_args.append('--with-exuberant-ctags={0}'.format( os.path.join(spec['exuberant-ctags'].prefix.bin, 'ctags'))) configure(*config_args) diff --git a/var/spack/packages/tmuxinator/package.py b/var/spack/packages/tmuxinator/package.py index 26c061cbd63..77ae063e5d8 100644 --- a/var/spack/packages/tmuxinator/package.py +++ b/var/spack/packages/tmuxinator/package.py @@ -5,7 +5,7 @@ class Tmuxinator(Package): homepage = "https://github.com/tmuxinator/tmuxinator" url = "https://github.com/tmuxinator/tmuxinator" - version('0.6.11', + version('0.6.11', git='https://github.com/tmuxinator/tmuxinator', tag='v0.6.11') @@ -13,5 +13,4 @@ class Tmuxinator(Package): def install(self, spec, prefix): gem('build', 'tmuxinator.gemspec') - gem('install', 'tmuxinator-{}.gem'.format(self.version)) - + gem('install', 'tmuxinator-{0}.gem'.format(self.version)) From 614c4d1876f488c699b8982b084699877dc87066 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Sun, 20 Dec 2015 13:27:43 +0100 Subject: [PATCH 183/910] moving write_make_inc into a patch function --- var/spack/packages/scotch/package.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/scotch/package.py b/var/spack/packages/scotch/package.py index 7e54b4fd8dc..f92fa915f96 100644 --- a/var/spack/packages/scotch/package.py +++ b/var/spack/packages/scotch/package.py @@ -104,6 +104,10 @@ def write_make_inc(self): with open('Makefile.inc', 'w') as fh: fh.write('\n'.join(makefile_inc)) + def patch(self): + with working_dir('src'): + self.write_make_inc() + def install(self, spec, prefix): targets = ['scotch'] if '+mpi' in self.spec: @@ -115,7 +119,6 @@ def install(self, spec, prefix): targets.append('ptesmumps') with working_dir('src'): - self.write_make_inc() for app in targets: make(app, parallel=(not app=='ptesmumps')) From 27d52badb18ed4281c69730de432c7d6dd5d3f67 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Sun, 20 Dec 2015 13:38:22 +0100 Subject: [PATCH 184/910] removing completely the write_make_inc function --- var/spack/packages/scotch/package.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/var/spack/packages/scotch/package.py b/var/spack/packages/scotch/package.py index f92fa915f96..8229ed86867 100644 --- a/var/spack/packages/scotch/package.py +++ b/var/spack/packages/scotch/package.py @@ -72,8 +72,7 @@ def extra_features(self, makefile_inc, defines): makefile_inc.append('LDFLAGS = %s' % ' '.join(ldflags)) - - def write_make_inc(self): + def patch(self): makefile_inc = [] defines = [ '-DCOMMON_RANDOM_FIXED_SEED', @@ -101,12 +100,9 @@ def write_make_inc(self): '' ]) - with open('Makefile.inc', 'w') as fh: - fh.write('\n'.join(makefile_inc)) - - def patch(self): with working_dir('src'): - self.write_make_inc() + with open('Makefile.inc', 'w') as fh: + fh.write('\n'.join(makefile_inc)) def install(self, spec, prefix): targets = ['scotch'] From 14c9c612752da6c46fb02cca7757176f0338d627 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sun, 20 Dec 2015 17:38:35 +0100 Subject: [PATCH 185/910] tau : scorep is now a variant --- var/spack/packages/tau/package.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/tau/package.py b/var/spack/packages/tau/package.py index 9febb647b39..31492397d84 100644 --- a/var/spack/packages/tau/package.py +++ b/var/spack/packages/tau/package.py @@ -46,6 +46,7 @@ class Tau(Package): # TODO : shmem variant missing variant('download', default=False, description='Downloads and builds various dependencies') + variant('scorep', default=False, description='Activates SCOREP support') variant('openmp', default=True, description='Use OpenMP threads') variant('mpi', default=True, description='Specify use of TAU MPI wrapper library') variant('phase', default=True, description='Generate phase based profiles') @@ -54,7 +55,7 @@ class Tau(Package): # TODO : Try to build direct OTF2 support? Some parts of the OTF support library in TAU are non-conformant, # TODO : and fail at compile-time. Further, SCOREP is compiled with OTF2 support. depends_on('pdt') # Required for TAU instrumentation - depends_on('scorep') + depends_on('scorep', when='+scorep') depends_on('binutils', when='~download') depends_on('mpi', when='+mpi') @@ -95,8 +96,7 @@ def install(self, spec, prefix): # As such it has a few #peculiarities# that make this build quite hackish. options = ["-prefix=%s" % prefix, "-iowrapper", - "-pdt=%s" % spec['pdt'].prefix, - "-scorep=%s" % spec['scorep'].prefix] + "-pdt=%s" % spec['pdt'].prefix] # If download is active, download and build suggested dependencies if '+download' in spec: options.extend(['-bfd=download', @@ -106,6 +106,9 @@ def install(self, spec, prefix): options.extend(["-bfd=%s" % spec['binutils'].prefix]) # TODO : unwind and asmdex are still missing + if '+scorep' in spec: + options.append("-scorep=%s" % spec['scorep'].prefix) + if '+openmp' in spec: options.append('-openmp') From 0a191d6b09e86d5b6eb66e97dde3cad197708f29 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 21 Dec 2015 15:02:48 +0100 Subject: [PATCH 186/910] scalasca : updated package build (depends on scorep, does not build twice). --- var/spack/packages/scalasca/package.py | 96 +++++++++++++------------- 1 file changed, 47 insertions(+), 49 deletions(-) diff --git a/var/spack/packages/scalasca/package.py b/var/spack/packages/scalasca/package.py index cf7a40c1f53..595b7ad0b85 100644 --- a/var/spack/packages/scalasca/package.py +++ b/var/spack/packages/scalasca/package.py @@ -1,65 +1,63 @@ -# FIXME: Add copyright +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## from spack import * + class Scalasca(Package): - """Scalasca is a software tool that supports the performance optimization - of parallel programs by measuring and analyzing their runtime behavior. - The analysis identifies potential performance bottlenecks - in - particular those concerning communication and synchronization - and - offers guidance in exploring their causes.""" + """ + Scalasca is a software tool that supports the performance optimization of parallel programs by measuring and + analyzing their runtime behavior. The analysis identifies potential performance bottlenecks - in particular those + concerning communication and synchronization - and offers guidance in exploring their causes. + """ - # FIXME: add a proper url for your package's homepage here. homepage = "http://www.scalasca.org" - url = "http://apps.fz-juelich.de/scalasca/releases/scalasca/2.1/dist/scalasca-2.1.tar.gz" + url = "http://apps.fz-juelich.de/scalasca/releases/scalasca/2.1/dist/scalasca-2.1.tar.gz" - version('2.1', 'bab9c2b021e51e2ba187feec442b96e6', - url = 'http://apps.fz-juelich.de/scalasca/releases/scalasca/2.1/dist/scalasca-2.1.tar.gz' ) + version('2.2.2', '2bafce988b0522d18072f7771e491ab9', + url='http://apps.fz-juelich.de/scalasca/releases/scalasca/2.2/dist/scalasca-2.2.2.tar.gz') + + version('2.1', 'bab9c2b021e51e2ba187feec442b96e6', + url='http://apps.fz-juelich.de/scalasca/releases/scalasca/2.1/dist/scalasca-2.1.tar.gz') depends_on("mpi") - depends_on("otf2@1.4") - depends_on("cube@4.2.3") - - backend_user_provided = """\ -CC=cc -CXX=c++ -F77=f77 -FC=f90 -CFLAGS=-fPIC -CXXFLAGS=-fPIC -""" - frontend_user_provided = """\ -CC_FOR_BUILD=cc -CXX_FOR_BUILD=c++ -F77_FOR_BUILD=f70 -FC_FOR_BUILD=f90 -CFLAGS_FOR_BUILD=-fPIC -CXXFLAGS_FOR_BUILD=-fPIC -""" - mpi_user_provided = """\ -MPICC=mpicc -MPICXX=mpicxx -MPIF77=mpif77 -MPIFC=mpif90 -MPI_CFLAGS=-fPIC -MPI_CXXFLAGS=-fPIC -""" + ########## + # Fix dependencies for Scalasca according to what stated in the release page + # The OTF2 library path should be detected automatically from SCOREP + # SCALASCA 2.2.2 + depends_on("scorep@1.4:", when='@2.2.2') + depends_on("cube@4.3:", when='@2.2.2') + # SCALASCA 2.1 + depends_on("scorep@1.3", when='@2.1') + depends_on("cube@4.2:", when='@2.1') + ########## def install(self, spec, prefix): configure_args = ["--prefix=%s" % prefix, - "--with-custom-compilers", - "--with-otf2=%s" % spec['otf2'].prefix.bin, "--with-cube=%s" % spec['cube'].prefix.bin, "--enable-shared"] - configure(*configure_args) - make() - make("install") - - # FIXME: Modify the configure line to suit your build system here. - configure("--prefix=%s" % prefix) - - # FIXME: Add logic to build and install here - make() - make("install") + make("install") \ No newline at end of file From a411c497abe3dbc0e98f4278a2d5908058802e38 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 21 Dec 2015 16:10:55 +0100 Subject: [PATCH 187/910] scalasca + dependencies : simplified build after #255 --- var/spack/packages/cube/package.py | 77 +++++++++++------------ var/spack/packages/opari2/package.py | 81 ++++++++++-------------- var/spack/packages/otf2/package.py | 85 ++++++++++---------------- var/spack/packages/scalasca/package.py | 2 +- var/spack/packages/scorep/package.py | 20 ++---- 5 files changed, 106 insertions(+), 159 deletions(-) diff --git a/var/spack/packages/cube/package.py b/var/spack/packages/cube/package.py index cc1c684594f..17d388c33d4 100644 --- a/var/spack/packages/cube/package.py +++ b/var/spack/packages/cube/package.py @@ -1,58 +1,55 @@ -# FIXME: Add copyright statement +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. # +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + from spack import * -from contextlib import closing + class Cube(Package): - """Cube the profile viewer for Score-P and Scalasca profiles. It - displays a multi-dimensional performance space consisting - of the dimensions (i) performance metric, (ii) call path, - and (iii) system resource.""" + """ + Cube the profile viewer for Score-P and Scalasca profiles. It displays a multi-dimensional performance space + consisting of the dimensions: + - performance metric + - call path + - system resource + """ homepage = "http://www.scalasca.org/software/cube-4.x/download.html" - url = "http://apps.fz-juelich.de/scalasca/releases/cube/4.2/dist/cube-4.2.3.tar.gz" + url = "http://apps.fz-juelich.de/scalasca/releases/cube/4.2/dist/cube-4.2.3.tar.gz" version('4.3.3', '07e109248ed8ffc7bdcce614264a2909', url='http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3.3.tar.gz') - version('4.2.3', '8f95b9531f5a8f8134f279c2767c9b20') + version('4.2.3', '8f95b9531f5a8f8134f279c2767c9b20', + url="http://apps.fz-juelich.de/scalasca/releases/cube/4.2/dist/cube-4.2.3.tar.gz") - version('4.3TP1', 'a2090fbc7b2ba394bd5c09ba971e237f', - url = 'http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz') - - # Using CC as C++ compiler provides quirky workaround for a Score-P build system attempt - # to guess a matching C compiler when configuring scorep-score - backend_user_provided = """\ -CC=cc -CXX=CC -F77=f77 -FC=f90 -#CFLAGS=-fPIC -#CXXFLAGS=-fPIC -""" - frontend_user_provided = """\ -CC_FOR_BUILD=cc -CXX_FOR_BUILD=CC -F77_FOR_BUILD=f70 -FC_FOR_BUILD=f90 -""" + # TODO : add variant that builds GUI on top of Qt def install(self, spec, prefix): - # Use a custom compiler configuration, otherwise the score-p - # build system messes with spack's compiler settings. - # Create these three files in the build directory - - with closing(open("vendor/common/build-config/platforms/platform-backend-user-provided", "w")) as backend_file: - backend_file.write(self.backend_user_provided) - with closing(open("vendor/common/build-config/platforms/platform-frontend-user-provided", "w")) as frontend_file: - frontend_file.write(self.frontend_user_provided) - configure_args = ["--prefix=%s" % prefix, - "--with-custom-compilers", - "--without-paraver", + "--without-paraver", "--without-gui"] - configure(*configure_args) - make(parallel=False) make("install", parallel=False) diff --git a/var/spack/packages/opari2/package.py b/var/spack/packages/opari2/package.py index 3f8c65377df..c68978f5c0a 100644 --- a/var/spack/packages/opari2/package.py +++ b/var/spack/packages/opari2/package.py @@ -1,18 +1,38 @@ -# FIXME: Add copyright statement here +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## from spack import * -from contextlib import closing class Opari2(Package): - """OPARI2 is a source-to-source instrumentation tool for OpenMP and - hybrid codes. It surrounds OpenMP directives and runtime library - calls with calls to the POMP2 measurement interface. - OPARI2 will provide you with a new initialization method that allows - for multi-directory and parallel builds as well as the usage of - pre-instrumented libraries. Furthermore, an efficient way of - tracking parent-child relationships was added. Additionally, we - extended OPARI2 to support instrumentation of OpenMP 3.0 - tied tasks. """ + """ + OPARI2 is a source-to-source instrumentation tool for OpenMP and hybrid codes. It surrounds OpenMP directives and + runtime library calls with calls to the POMP2 measurement interface. OPARI2 will provide you with a new + initialization method that allows for multi-directory and parallel builds as well as the usage of pre-instrumented + libraries. Furthermore, an efficient way of tracking parent-child relationships was added. Additionally, we extended + OPARI2 to support instrumentation of OpenMP 3.0 tied tasks. + """ homepage = "http://www.vi-hps.org/projects/score-p" url = "http://www.vi-hps.org/upload/packages/opari2/opari2-1.1.2.tar.gz" @@ -21,47 +41,8 @@ class Opari2(Package): url='http://www.vi-hps.org/upload/packages/opari2/opari2-1.1.4.tar.gz') version('1.1.2', '9a262c7ca05ff0ab5f7775ae96f3539e') - backend_user_provided = """\ -CC=cc -CXX=c++ -F77=f77 -FC=f90 -CFLAGS=-fPIC -CXXFLAGS=-fPIC -""" - frontend_user_provided = """\ -CC_FOR_BUILD=cc -CXX_FOR_BUILD=c++ -F77_FOR_BUILD=f70 -FC_FOR_BUILD=f90 -CFLAGS_FOR_BUILD=-fPIC -CXXFLAGS_FOR_BUILD=-fPIC -""" - mpi_user_provided = """\ -MPICC=mpicc -MPICXX=mpicxx -MPIF77=mpif77 -MPIFC=mpif90 -MPI_CFLAGS=-fPIC -MPI_CXXFLAGS=-fPIC -""" - def install(self, spec, prefix): - # Use a custom compiler configuration, otherwise the score-p - # build system messes with spack's compiler settings. - # Create these three files in the build directory - with closing(open("platform-backend-user-provided", "w")) as backend_file: - backend_file.write(self.backend_user_provided) - with closing(open("platform-frontend-user-provided", "w")) as frontend_file: - frontend_file.write(self.frontend_user_provided) - with closing(open("platform-mpi-user-provided", "w")) as mpi_file: - mpi_file.write(self.mpi_user_provided) - - # FIXME: Modify the configure line to suit your build system here. configure("--prefix=%s" % prefix, - "--with-custom-compilers", "--enable-shared") - - # FIXME: Add logic to build and install here make() make("install") diff --git a/var/spack/packages/otf2/package.py b/var/spack/packages/otf2/package.py index b3d3a5b6633..c3d61bc2283 100644 --- a/var/spack/packages/otf2/package.py +++ b/var/spack/packages/otf2/package.py @@ -1,12 +1,35 @@ -# FIXME: Add copyright +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## from spack import * -from contextlib import closing -import os + class Otf2(Package): - """The Open Trace Format 2 is a highly scalable, memory efficient event - trace data format plus support library.""" + """ + The Open Trace Format 2 is a highly scalable, memory efficient event trace data format plus support library. + """ homepage = "http://www.vi-hps.org/score-p" url = "http://www.vi-hps.org/upload/packages/otf2/otf2-1.4.tar.gz" @@ -22,57 +45,11 @@ class Otf2(Package): version('1.2.1', '8fb3e11fb7489896596ae2c7c83d7fc8', url="http://www.vi-hps.org/upload/packages/otf2/otf2-1.2.1.tar.gz") - backend_user_provided = """\ -CC=cc -CXX=c++ -F77=f77 -FC=f90 -CFLAGS=-fPIC -CXXFLAGS=-fPIC -""" - frontend_user_provided = """\ -CC_FOR_BUILD=cc -CXX_FOR_BUILD=c++ -F77_FOR_BUILD=f70 -FC_FOR_BUILD=f90 -CFLAGS_FOR_BUILD=-fPIC -CXXFLAGS_FOR_BUILD=-fPIC -""" - mpi_user_provided = """\ -MPICC=cc -MPICXX=c++ -MPIF77=f77 -MPIFC=f90 -MPI_CFLAGS=-fPIC -MPI_CXXFLAGS=-fPIC -""" - - @when('@:1.2.1') - def version_specific_args(self): - return ["--with-platform=disabled", "CC=cc", "CXX=c++", "F77=f77", "F90=f90", "CFLAGS=-fPIC", "CXXFLAGS=-fPIC"] - - @when('@1.3:') - def version_specific_args(self): - # TODO: figure out what scorep's build does as of otf2 1.3 - return ["--with-custom-compilers"] - def install(self, spec, prefix): - # Use a custom compiler configuration, otherwise the score-p - # build system messes with spack's compiler settings. - # Create these three files in the build directory - with closing(open("platform-backend-user-provided", "w")) as backend_file: - backend_file.write(self.backend_user_provided) - with closing(open("platform-frontend-user-provided", "w")) as frontend_file: - frontend_file.write(self.frontend_user_provided) - with closing(open("platform-mpi-user-provided", "w")) as mpi_file: - mpi_file.write(self.mpi_user_provided) - configure_args=["--prefix=%s" % prefix, - "--enable-shared"] - - configure_args.extend(self.version_specific_args()) - + "--enable-shared", + "CFLAGS=-fPIC", + "CXXFLAGS=-fPIC"] configure(*configure_args) - make() make("install") diff --git a/var/spack/packages/scalasca/package.py b/var/spack/packages/scalasca/package.py index 595b7ad0b85..6de14564b29 100644 --- a/var/spack/packages/scalasca/package.py +++ b/var/spack/packages/scalasca/package.py @@ -44,7 +44,7 @@ class Scalasca(Package): depends_on("mpi") ########## - # Fix dependencies for Scalasca according to what stated in the release page + # Hard-code dependencies for Scalasca according to what stated in the release page # The OTF2 library path should be detected automatically from SCOREP # SCALASCA 2.2.2 depends_on("scorep@1.4:", when='@2.2.2') diff --git a/var/spack/packages/scorep/package.py b/var/spack/packages/scorep/package.py index 0820f2d8ac3..5127e814b68 100644 --- a/var/spack/packages/scorep/package.py +++ b/var/spack/packages/scorep/package.py @@ -28,9 +28,9 @@ class Scorep(Package): """ - The Score-P measurement infrastructure is a highly scalable and - easy-to-use tool suite for profiling, event tracing, and online - analysis of HPC applications.""" + The Score-P measurement infrastructure is a highly scalable and easy-to-use tool suite for profiling, event + tracing, and online analysis of HPC applications. + """ homepage = "http://www.vi-hps.org/projects/score-p" url = "http://www.vi-hps.org/upload/packages/scorep/scorep-1.2.3.tar.gz" @@ -55,15 +55,6 @@ class Scorep(Package): depends_on("mpi") depends_on("papi") - def get_compiler_config_line(self): - backend_user_provided = ['CC=%s' % self.compiler.cc_names[0], - 'CXX=%s' % self.compiler.cxx_names[0], - 'F77=%s' % self.compiler.f77_names[0] if len(self.compiler.f77_names) else "", - 'FC=%s' % self.compiler.fc_names[0] if len(self.compiler.fc_names) else "", - 'CFLAGS=-fPIC %s' % self.rpath_args, - 'CXXFLAGS=-fPIC %s'% self.rpath_args] - return backend_user_provided - def install(self, spec, prefix): configure = Executable( join_path(self.stage.source_path, 'configure') ) with working_dir('spack-build', create=True): @@ -73,8 +64,9 @@ def install(self, spec, prefix): "--with-cube=%s" % spec['cube'].prefix.bin, "--with-papi-header=%s" % spec['papi'].prefix.include, "--with-papi-lib=%s" % spec['papi'].prefix.lib, - "--enable-shared"] - configure_args.extend(self.get_compiler_config_line()) + "--enable-shared", + "CFLAGS=-fPIC", + "CXXFLAGS=-fPIC"] configure(*configure_args) make() make("install") From ba22fc8b78841728500966d471db7b5bfd26cf56 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 21 Dec 2015 10:21:33 -0500 Subject: [PATCH 188/910] Simplify Spack cc script --- lib/spack/env/cc | 289 ++++++++++++++++++++++------------------------- 1 file changed, 133 insertions(+), 156 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index f3fadfcf45b..053295f42d5 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -65,7 +65,7 @@ function die { } for param in $parameters; do - if [ -z "${!param}" ]; then + if [[ -z ${!param} ]]; then die "Spack compiler must be run from spack! Input $param was missing!" fi done @@ -114,114 +114,118 @@ case "$command" in esac # Finish setting up the mode. -if [ -z "$mode" ]; then +if [[ -z $mode ]]; then mode=ccld for arg in "$@"; do - if [ "$arg" = -v -o "$arg" = -V -o "$arg" = --version -o "$arg" = -dumpversion ]; then - mode=vcheck - break - elif [ "$arg" = -E ]; then - mode=cpp - break - elif [ "$arg" = -c ]; then - mode=cc - break - elif [ "$arg" = -S ]; then - mode=as - echo "spac cc mode as" "$@" - exit 1 - break - fi + case "$arg" in + -v|-V|--version|-dumpversion) + mode=vcheck + break + ;; + -E) + mode=cpp + break + ;; + -c) + mode=cc + break + ;; + -S) + mode=as + break + ;; + esac done fi # Dump the version and exist if we're in testing mode. -if [ "$SPACK_TEST_COMMAND" = "dump-mode" ]; then +if [[ $SPACK_TEST_COMMAND = dump-mode ]]; then echo "$mode" exit fi # Check that at least one of the real commands was actually selected, # otherwise we don't know what to execute. -if [ -z "$command" ]; then +if [[ -z $command ]]; then die "ERROR: Compiler '$SPACK_COMPILER_SPEC' does not support compiling $language programs." fi # Save original command for debug logging input_command="$@" - -# -# Now do real parsing of the command line args, trying hard to keep -# non-rpath linker arguments in the proper order w.r.t. other command -# line arguments. This is important for things like groups. -# -includes=() -libraries=() -libs=() -rpaths=() -other_args=() -all_args=("$@") - -while [ -n "$1" ]; do - case "$1" in - -I*) - arg="${1#-I}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - includes+=("$arg") - ;; - -L*) - arg="${1#-L}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - libraries+=("$arg") - ;; - -l*) - arg="${1#-l}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - libs+=("$arg") - ;; - -Wl,*) - arg="${1#-Wl,}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ "$arg" = -rpath=* ]]; then - rpaths+=("${arg#-rpath=}") - elif [[ "$arg" = -rpath,* ]]; then - rpaths+=("${arg#-rpath,}") - elif [[ "$arg" = -rpath ]]; then - shift; arg="$1" - if [[ "$arg" != -Wl,* ]]; then - die "-Wl,-rpath was not followed by -Wl,*" - fi - rpaths+=("${arg#-Wl,}") - else - other_args+=("-Wl,$arg") - fi - ;; - -Xlinker,*) - arg="${1#-Xlinker,}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ "$arg" = -rpath=* ]]; then - rpaths+=("${arg#-rpath=}") - elif [[ "$arg" = -rpath,* ]]; then - rpaths+=("${arg#-rpath,}") - elif [[ "$arg" = -rpath ]]; then - shift; arg="$1" - if [[ "$arg" != -Xlinker,* ]]; then - die "-Xlinker,-rpath was not followed by -Xlinker,*" - fi - rpaths+=("${arg#-Xlinker,}") - else - other_args+=("-Xlinker,$arg") - fi - ;; - *) - other_args+=("$1") - ;; - esac - shift -done +args=("$@") # Dump parsed values for unit testing if asked for -if [ -n "$SPACK_TEST_COMMAND" ]; then +if [[ -n $SPACK_TEST_COMMAND ]]; then + + # + # Now do real parsing of the command line args, trying hard to keep + # non-rpath linker arguments in the proper order w.r.t. other command line + # arguments. This is important for things like groups. + # + includes=() + libraries=() + libs=() + rpaths=() + other_args=() + + while [[ -n $1 ]]; do + case "$1" in + -I*) + arg="${1#-I}" + if [[ -z $arg ]]; then shift; arg="$1"; fi + includes+=("$arg") + ;; + -L*) + arg="${1#-L}" + if [[ -z $arg ]]; then shift; arg="$1"; fi + libraries+=("$arg") + ;; + -l*) + arg="${1#-l}" + if [[ -z $arg ]]; then shift; arg="$1"; fi + libs+=("$arg") + ;; + -Wl,*) + arg="${1#-Wl,}" + if [[ -z $arg ]]; then shift; arg="$1"; fi + if [[ $arg = -rpath=* ]]; then + rpaths+=("${arg#-rpath=}") + elif [[ $arg = -rpath,* ]]; then + rpaths+=("${arg#-rpath,}") + elif [[ $arg = -rpath ]]; then + shift; arg="$1" + if [[ $arg != -Wl,* ]]; then + die "-Wl,-rpath was not followed by -Wl,*" + fi + rpaths+=("${arg#-Wl,}") + else + other_args+=("-Wl,$arg") + fi + ;; + -Xlinker,*) + arg="${1#-Xlinker,}" + if [[ -z $arg ]]; then shift; arg="$1"; fi + if [[ $arg = -rpath=* ]]; then + rpaths+=("${arg#-rpath=}") + elif [[ $arg = -rpath,* ]]; then + rpaths+=("${arg#-rpath,}") + elif [[ $arg = -rpath ]]; then + shift; arg="$1" + if [[ $arg != -Xlinker,* ]]; then + die "-Xlinker,-rpath was not followed by -Xlinker,*" + fi + rpaths+=("${arg#-Xlinker,}") + else + other_args+=("-Xlinker,$arg") + fi + ;; + *) + other_args+=("$1") + ;; + esac + shift + done + IFS=$'\n' case "$SPACK_TEST_COMMAND" in dump-includes) echo "${includes[*]}";; @@ -246,8 +250,8 @@ if [ -n "$SPACK_TEST_COMMAND" ]; then echo "${other_args[*]}" ;; *) - echo "ERROR: Unknown test command" - exit 1 ;; + die "ERROR: Unknown test command" + ;; esac exit fi @@ -255,66 +259,44 @@ fi # Read spack dependencies from the path environment variable IFS=':' read -ra deps <<< "$SPACK_DEPENDENCIES" for dep in "${deps[@]}"; do - if [ -d "$dep/include" ]; then - includes+=("$dep/include") - all_args=("-I$dep/include" ${all_args[@]}) + if [[ -d $dep/include ]]; then + args=("-I$dep/include" "${args[@]}") fi - if [ -d "$dep/lib" ]; then - libraries+=("$dep/lib") - rpaths+=("$dep/lib") - if [ "$mode" = ccld ]; then - all_args=("-L$dep/lib" "-Wl,-rpath,$dep/lib" ${all_args[@]}) - elif [ "$mode" = ld ]; then - all_args=("-L$dep/lib" "-rpath" "$dep/lib" ${all_args[@]}) - fi + if [[ -d $dep/lib ]]; then + # libraries+=("$dep/lib") + if [[ $mode = ccld ]]; then + args=("-L$dep/lib" "-Wl,-rpath,$dep/lib" "${args[@]}") + elif [[ $mode = ld ]]; then + args=("-L$dep/lib" "-rpath" "$dep/lib" "${args[@]}") + fi fi - if [ -d "$dep/lib64" ]; then - libraries+=("$dep/lib64") - rpaths+=("$dep/lib64") - if [ "$mode" = ccld ]; then - all_args=("-L$dep/lib" "-Wl,-rpath,$dep/lib" ${all_args[@]}) - elif [ "$mode" = ld ]; then - all_args=("-L$dep/lib" "-rpath" "$dep/lib" ${all_args[@]}) - fi + if [[ -d $dep/lib64 ]]; then + # libraries+=("$dep/lib64") + if [[ $mode = ccld ]]; then + args=("-L$dep/lib" "-Wl,-rpath,$dep/lib" "${args[@]}") + elif [[ $mode = ld ]]; then + args=("-L$dep/lib" "-rpath" "$dep/lib" "${args[@]}") + fi fi done # Include all -L's and prefix/whatever dirs in rpath -for dir in "${libraries[@]}"; do - [[ dir = $SPACK_INSTALL* ]] && rpaths+=("$dir") - if [ "$mode" = ccld ]; then - [[ dir = $SPACK_INSTALL* ]] && all_args=("-Wl,-rpath,$dir" ${all_args[@]}) - elif [ "$mode" = ld ]; then - [[ dir = $SPACK_INSTALL* ]] && all_args=("-rpath" "$dir" ${all_args[@]}) - fi -done -rpaths+=("$SPACK_PREFIX/lib") -rpaths+=("$SPACK_PREFIX/lib64") -if [ "$mode" = ccld ]; then - all_args=("-Wl,-rpath,$SPACK_PREFIX/lib" "-Wl,-rpath,$SPACK_PREFIX/lib64" ${all_args[@]}) -elif [ "$mode" = ld ]; then - all_args=("-rpath" "$SPACK_PREFIX/lib" "-rpath" "$SPACK_PREFIX/lib64" ${all_args[@]}) -fi - -# Put the arguments together -args=() -for dir in "${includes[@]}"; do args+=("-I$dir"); done -args+=("${other_args[@]}") -for dir in "${libraries[@]}"; do args+=("-L$dir"); done -for lib in "${libs[@]}"; do args+=("-l$lib"); done - -if [ "$mode" = ccld ]; then - for dir in "${rpaths[@]}"; do - args+=("-Wl,-rpath") - args+=("-Wl,$dir"); - done -elif [ "$mode" = ld ]; then - for dir in "${rpaths[@]}"; do - args+=("-rpath") - args+=("$dir"); - done +if [[ $mode = ccld ]]; then + # for dir in "${libraries[@]}"; do + # if [[ dir = $SPACK_INSTALL* ]]; then + # args=("-Wl,-rpath,$dir" "${args[@]}") + # fi + # done + args=("-Wl,-rpath,$SPACK_PREFIX/lib" "-Wl,-rpath,$SPACK_PREFIX/lib64" ${args[@]}) +elif [[ $mode = ld ]]; then + # for dir in "${libraries[@]}"; do + # if [[ dir = $SPACK_INSTALL* ]]; then + # args=("-rpath" "$dir" "${args[@]}") + # fi + # done + args=("-rpath" "$SPACK_PREFIX/lib" "-rpath" "$SPACK_PREFIX/lib64" ${args[@]}) fi # @@ -330,34 +312,29 @@ unset DYLD_LIBRARY_PATH # IFS=':' read -ra env_path <<< "$PATH" IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" -spack_env_dirs+=(".") +spack_env_dirs+=("" ".") PATH="" for dir in "${env_path[@]}"; do remove="" for rm_dir in "${spack_env_dirs[@]}"; do - if [ "$dir" = "$rm_dir" ]; then remove=True; fi + if [[ $dir = $rm_dir ]]; then remove=True; fi done - if [ -z "$remove" ]; then - if [ -z "$PATH" ]; then - PATH="$dir" - else - PATH="$PATH:$dir" - fi + if [[ -z $remove ]]; then + PATH="${PATH:+$PATH:}$dir" fi done export PATH full_command=("$command") -# full_command+=("${args[@]}") -full_command+=("${all_args[@]}") +full_command+=("${args[@]}") # # Write the input and output commands to debug logs if it's asked for. # -if [ "$SPACK_DEBUG" = "TRUE" ]; then +if [[ $SPACK_DEBUG = TRUE ]]; then input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.in.log" output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.out.log" - echo "$input_command" >> $input_log + echo "$input_command" >> $input_log echo "$mode ${full_command[@]}" >> $output_log fi From f9326053b998f008bdc4e8b08ba3225f5f7d8dec Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Mon, 21 Dec 2015 13:56:19 -0800 Subject: [PATCH 189/910] remove numpy compiler patch --- var/spack/packages/py-numpy/package.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/var/spack/packages/py-numpy/package.py b/var/spack/packages/py-numpy/package.py index efa109a3e9c..4c085fba6e5 100644 --- a/var/spack/packages/py-numpy/package.py +++ b/var/spack/packages/py-numpy/package.py @@ -13,13 +13,6 @@ class PyNumpy(Package): depends_on('netlib-blas+fpic') depends_on('netlib-lapack+shared') - def patch(self): - filter_file( - "possible_executables = \['(gfortran|g77|ifort|efl)", - "possible_executables = ['fc", - "numpy/distutils/fcompiler/gnu.py", - "numpy/distutils/fcompiler/intel.py") - def install(self, spec, prefix): with open('site.cfg', 'w') as f: f.write('[DEFAULT]\n') From c3aaf005e2ec2844fa02a60414978692b67727a4 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Mon, 21 Dec 2015 15:14:59 -0800 Subject: [PATCH 190/910] new python versions --- var/spack/packages/python/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 12b282910b4..30e6ef6bb86 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -17,7 +17,9 @@ class Python(Package): version('2.7.8', 'd235bdfa75b8396942e360a70487ee00') version('2.7.10', 'c685ef0b8e9f27b5e3db5db12b268ac6') + version('2.7.11', '1dbcc848b4cd8399a8199d000f9f823c') version('3.5.0', 'd149d2812f10cbe04c042232e7964171') + version('3.5.1', 'e9ea6f2623fffcdd871b7b19113fde80') depends_on("openssl") depends_on("bzip2") From fe0fdf60b46716fa20318c627ea80ff4e3398263 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 21 Dec 2015 15:35:47 -0800 Subject: [PATCH 191/910] Add ability to prefer particular versions in packages. - Adding `preferred=True` to a version directive will change its sort order in concretization. - This provides us a rudimentary ability to keep the Spack stack stable as new versions are added. - Having multiple stacks will come next, but this at least allows us to specify default versions of things instead of always taking the newest. --- lib/spack/spack/concretize.py | 11 +++++++++-- var/spack/packages/python/package.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index d9419da7840..85cdb202d55 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -40,7 +40,6 @@ from spack.version import * - class DefaultConcretizer(object): """This class doesn't have any state, it just provides some methods for concretization. You can subclass it to override just some of the @@ -68,9 +67,17 @@ def concretize_version(self, spec): # If there are known available versions, return the most recent # version that satisfies the spec pkg = spec.package + + # Key function to sort versions first by whether they were + # marked `preferred=True`, then by most recent. + def preferred_key(v): + prefer = pkg.versions[v].get('preferred', False) + return (prefer, v) + valid_versions = sorted( [v for v in pkg.versions - if any(v.satisfies(sv) for sv in spec.versions)]) + if any(v.satisfies(sv) for sv in spec.versions)], + key=preferred_key) if valid_versions: spec.versions = ver([valid_versions[-1]]) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 30e6ef6bb86..0da7f4ce898 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -17,7 +17,7 @@ class Python(Package): version('2.7.8', 'd235bdfa75b8396942e360a70487ee00') version('2.7.10', 'c685ef0b8e9f27b5e3db5db12b268ac6') - version('2.7.11', '1dbcc848b4cd8399a8199d000f9f823c') + version('2.7.11', '1dbcc848b4cd8399a8199d000f9f823c', preferred=True) version('3.5.0', 'd149d2812f10cbe04c042232e7964171') version('3.5.1', 'e9ea6f2623fffcdd871b7b19113fde80') From 2ba6bb21fbac14b6eec9b4ba4fb7ea1f11e9df6f Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Mon, 21 Dec 2015 18:48:30 -0500 Subject: [PATCH 192/910] fix #249 (tmp spack-stage directories conflict) --- lib/spack/spack/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 02eeed01fa1..ccd7dc25c87 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -112,8 +112,9 @@ tmp_dirs = [] _default_tmp = tempfile.gettempdir() if _default_tmp != os.getcwd(): - tmp_dirs.append(os.path.join(_default_tmp, 'spack-stage')) -tmp_dirs.append('/nfs/tmp2/%u/spack-stage') + tmp_dirs = [ join_path(_default_tmp, '%u', 'spack-stage'), + join_path(_default_tmp, 'spack-stage') ] +tmp_dirs.append('/nfs/tmp2/%u/spack-stage') # TODO: remove # Whether spack should allow installation of unsafe versions of # software. "Unsafe" versions are ones it doesn't have a checksum From bfc72c04fb57df07286e8b5a488cf039c4add11b Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Mon, 21 Dec 2015 18:55:25 -0500 Subject: [PATCH 193/910] fix non-mpi hdf5 package build Fixes a couple of issues in c5b7eba45743ebd131233c035101a0e64bb0c1c7. --- var/spack/packages/hdf5/package.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/var/spack/packages/hdf5/package.py b/var/spack/packages/hdf5/package.py index 44d4ede2787..adac79d9bbd 100644 --- a/var/spack/packages/hdf5/package.py +++ b/var/spack/packages/hdf5/package.py @@ -26,16 +26,14 @@ def install(self, spec, prefix): if '+mpi' in spec: extra_args.extend([ "--enable-parallel", - "CC=%s" % spec['mpich'].prefix.bin + "/mpicc", - "CXX=%s" % spec['mpich'].prefix.bin + "/mpic++", + "CC=%s" % spec['mpi'].prefix.bin + "/mpicc", + "CXX=%s" % spec['mpi'].prefix.bin + "/mpic++", ]) configure( "--prefix=%s" % prefix, "--with-zlib=%s" % spec['zlib'].prefix, "--enable-shared", - "CC=%s" % spec['mpi'].prefix.bin + "/mpicc", - "CXX=%s" % spec['mpi'].prefix.bin + "/mpic++", *extra_args) make() From 61f07b0c4d50c831d4723341c1a896c54d82c221 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Sun, 20 Dec 2015 20:15:02 -0500 Subject: [PATCH 194/910] fix a few comment typos --- bin/spack | 2 +- lib/spack/spack/build_environment.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/spack b/bin/spack index a0b260ccaf3..cd46cf6180b 100755 --- a/bin/spack +++ b/bin/spack @@ -150,7 +150,7 @@ def main(): sys.stderr.write('\n') tty.die("Keyboard interrupt.") - # Allow commands to return values if they want to exit with some ohter code. + # Allow commands to return values if they want to exit with some other code. if return_val is None: sys.exit(0) elif isinstance(return_val, int): diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 20154a55b3d..84c2f6015b5 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -145,7 +145,7 @@ def add_env_path(path): # Install root prefix os.environ[SPACK_INSTALL] = spack.install_path - # Remove these vars from the environment during build becaus they + # Remove these vars from the environment during build because they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. pop_keys(os.environ, "LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH") From fc43f3c730d5cfad0de920610946ca6a825f972b Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 22 Dec 2015 10:11:18 +0100 Subject: [PATCH 195/910] valgrind : added package --- var/spack/packages/valgrind/package.py | 55 ++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 var/spack/packages/valgrind/package.py diff --git a/var/spack/packages/valgrind/package.py b/var/spack/packages/valgrind/package.py new file mode 100644 index 00000000000..0b030d73e99 --- /dev/null +++ b/var/spack/packages/valgrind/package.py @@ -0,0 +1,55 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Valgrind(Package): + """ + Valgrind is an instrumentation framework for building dynamic analysis tools. There are Valgrind tools that can + automatically detect many memory management and threading bugs, and profile your programs in detail. You can also + use Valgrind to build new tools. + + Valgrind is Open Source / Free Software, and is freely available under the GNU General Public License, version 2. + """ + homepage = "http://valgrind.org/" + url = "http://valgrind.org/downloads/valgrind-3.11.0.tar.bz2" + + version('3.11.0', '4ea62074da73ae82e0162d6550d3f129') + version('3.10.1', '60ddae962bc79e7c95cfc4667245707f') + version('3.10.0', '7c311a72a20388aceced1aa5573ce970') + + variant('mpi', default=True, description='Activates MPI support for valgrind') + variant('boost', default=True, description='Activates boost support for valgrind') + + depends_on('mpi', when='+mpi') + depends_on('boost', when='+boost') + + def install(self, spec, prefix): + options = ['--prefix=%s' % prefix, + '--enable-ubsan'] + configure(*options) + make() + make("install") From d85a46380cd0b2b113c8e3de111298abff2dd24f Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Tue, 22 Dec 2015 06:50:18 -0800 Subject: [PATCH 196/910] fix conflict with setuptools and cached pyc files --- var/spack/packages/python/package.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 30e6ef6bb86..2213244d6d7 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -35,7 +35,8 @@ def install(self, spec, prefix): # Rest of install is pretty standard except setup.py needs to be able to read the CPPFLAGS # and LDFLAGS as it scans for the library and headers to build - configure("--prefix=%s" % prefix, + configure_args= [ + "--prefix=%s" % prefix, "--with-threads", "--enable-shared", "CPPFLAGS=-I%s/include -I%s/include -I%s/include -I%s/include -I%s/include -I%s/include" % ( @@ -45,7 +46,11 @@ def install(self, spec, prefix): "LDFLAGS=-L%s/lib -L%s/lib -L%s/lib -L%s/lib -L%s/lib -L%s/lib" % ( spec['openssl'].prefix, spec['bzip2'].prefix, spec['readline'].prefix, spec['ncurses'].prefix, - spec['sqlite'].prefix, spec['zlib'].prefix)) + spec['sqlite'].prefix, spec['zlib'].prefix) + ] + if spec.satisfies('@3:'): + configure_args.append('--without-ensurepip') + configure(*configure_args) make() make("install") @@ -116,6 +121,7 @@ def python_ignore(self, ext_pkg, args): patterns.append(r'setuptools\.pth') patterns.append(r'bin/easy_install[^/]*$') patterns.append(r'setuptools.*egg$') + patterns.append(r'__pycache__/.*.pyc$') return match_predicate(ignore_arg, patterns) From 792b03325562e5577d018b188577915ea4dd5e7d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 22 Dec 2015 07:54:25 -0800 Subject: [PATCH 197/910] Slightly more robust tmp directory search. - remove getcwd() check (seems arbitrary -- if users set their TMPDIR to this why stop them?) - try a number of common locations and try per-user directories in them first. --- lib/spack/spack/__init__.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index ccd7dc25c87..92cb417a850 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -24,6 +24,7 @@ ############################################################################## import os import tempfile +import getpass from llnl.util.filesystem import * # This lives in $prefix/lib/spack/spack/__file__ @@ -111,10 +112,17 @@ # that it can create. tmp_dirs = [] _default_tmp = tempfile.gettempdir() -if _default_tmp != os.getcwd(): - tmp_dirs = [ join_path(_default_tmp, '%u', 'spack-stage'), - join_path(_default_tmp, 'spack-stage') ] -tmp_dirs.append('/nfs/tmp2/%u/spack-stage') # TODO: remove +_tmp_user = getpass.getuser() + +_tmp_candidates = (_default_tmp, '/nfs/tmp2', '/tmp', '/var/tmp') +for path in _tmp_candidates: + # don't add a second username if it's already unique by user. + if not _tmp_user in path: + tmp_dirs.append(join_path(path, '%u', 'spack-stage')) + +for path in _tmp_candidates: + if not path in tmp_dirs: + tmp_dirs.append(join_path(path, 'spack-stage')) # Whether spack should allow installation of unsafe versions of # software. "Unsafe" versions are ones it doesn't have a checksum From 3db1c1761ee634f4c2a7d254a4aeca1d04f20967 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Tue, 22 Dec 2015 08:03:17 -0800 Subject: [PATCH 198/910] only ignore cached site.py file --- var/spack/packages/python/package.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 2213244d6d7..920629e9109 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -117,11 +117,10 @@ def python_ignore(self, ext_pkg, args): # Ignore pieces of setuptools installed by other packages. if ext_pkg.name != 'py-setuptools': - patterns.append(r'/site\.pyc?$') + patterns.append(r'/site.*\.pyc?$') patterns.append(r'setuptools\.pth') patterns.append(r'bin/easy_install[^/]*$') patterns.append(r'setuptools.*egg$') - patterns.append(r'__pycache__/.*.pyc$') return match_predicate(ignore_arg, patterns) From 41fd380d1b85f07b6ab05b2fecd729e125c0c023 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Tue, 22 Dec 2015 09:15:47 -0800 Subject: [PATCH 199/910] fixed site pyc regular expression --- var/spack/packages/python/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 920629e9109..eb14c98931f 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -117,7 +117,7 @@ def python_ignore(self, ext_pkg, args): # Ignore pieces of setuptools installed by other packages. if ext_pkg.name != 'py-setuptools': - patterns.append(r'/site.*\.pyc?$') + patterns.append(r'/site[^/]*\.pyc?$') patterns.append(r'setuptools\.pth') patterns.append(r'bin/easy_install[^/]*$') patterns.append(r'setuptools.*egg$') From 3427174eb065a6e74fc279076423c2c5c5e8e737 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 22 Dec 2015 15:49:14 -0500 Subject: [PATCH 200/910] Correct quoting --- lib/spack/env/cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 565f959ca4f..4ac8f2e7f4d 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -265,7 +265,7 @@ for dep in "${deps[@]}"; do if [[ -d $dep/lib ]]; then # libraries+=("$dep/lib") - if [[ $mode = ccld ]]; then + if [[ $mode = ccld ]]; then args=("-L$dep/lib" "-Wl,-rpath,$dep/lib" "${args[@]}") elif [[ $mode = ld ]]; then args=("-L$dep/lib" "-rpath" "$dep/lib" "${args[@]}") @@ -289,14 +289,14 @@ if [[ $mode = ccld ]]; then # args=("-Wl,-rpath,$dir" "${args[@]}") # fi # done - args=("-Wl,-rpath,$SPACK_PREFIX/lib" "-Wl,-rpath,$SPACK_PREFIX/lib64" ${args[@]}) + args=("-Wl,-rpath,$SPACK_PREFIX/lib" "-Wl,-rpath,$SPACK_PREFIX/lib64" "${args[@]}") elif [[ $mode = ld ]]; then # for dir in "${libraries[@]}"; do # if [[ dir = $SPACK_INSTALL* ]]; then # args=("-rpath" "$dir" "${args[@]}") # fi # done - args=("-rpath" "$SPACK_PREFIX/lib" "-rpath" "$SPACK_PREFIX/lib64" ${args[@]}) + args=("-rpath" "$SPACK_PREFIX/lib" "-rpath" "$SPACK_PREFIX/lib64" "${args[@]}") fi # @@ -325,8 +325,7 @@ for dir in "${env_path[@]}"; do done export PATH -full_command=("$command") -full_command+=("${args[@]}") +full_command=("$command" "${args[@]}") # # Write the input and output commands to debug logs if it's asked for. From 6dc1fc03c13027f3fd1553af4d23ed00ea929985 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 21 Dec 2015 10:34:41 -0500 Subject: [PATCH 201/910] Update hwloc; don't require libpciaccess on OS X - hwloc 1.11.2 is available. - libpciaccess is not supported on OS X; don't require it there. --- var/spack/packages/hwloc/package.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/hwloc/package.py b/var/spack/packages/hwloc/package.py index 452a7d7ce3b..35a526ceb87 100644 --- a/var/spack/packages/hwloc/package.py +++ b/var/spack/packages/hwloc/package.py @@ -15,15 +15,17 @@ class Hwloc(Package): homepage = "http://www.open-mpi.org/projects/hwloc/" url = "http://www.open-mpi.org/software/hwloc/v1.9/downloads/hwloc-1.9.tar.gz" + version('1.11.2', '486169cbe111cdea57be12638828ebbf', + url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.2.tar.bz2') version('1.11.1', '002742efd3a8431f98d6315365a2b543', url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.1.tar.bz2') version('1.9', '1f9f9155682fe8946a97c08896109508') - depends_on('libpciaccess') + # libpciaccess is not supported on OS X + depends_on('libpciaccess', when='=linux') def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() make("install") - From 6831ee6f0a98bfdc2c0847d0a0dea6c91fcb1d9b Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 22 Dec 2015 16:24:05 -0500 Subject: [PATCH 202/910] Skip installing libpciaccess on Darwin --- var/spack/packages/hwloc/package.py | 3 +-- var/spack/packages/libpciaccess/package.py | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/hwloc/package.py b/var/spack/packages/hwloc/package.py index 35a526ceb87..7ebede76a3b 100644 --- a/var/spack/packages/hwloc/package.py +++ b/var/spack/packages/hwloc/package.py @@ -21,8 +21,7 @@ class Hwloc(Package): url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.1.tar.bz2') version('1.9', '1f9f9155682fe8946a97c08896109508') - # libpciaccess is not supported on OS X - depends_on('libpciaccess', when='=linux') + depends_on('libpciaccess') def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/packages/libpciaccess/package.py b/var/spack/packages/libpciaccess/package.py index 6022fc34a30..403bafbbd2c 100644 --- a/var/spack/packages/libpciaccess/package.py +++ b/var/spack/packages/libpciaccess/package.py @@ -1,4 +1,5 @@ from spack import * +import os.path class Libpciaccess(Package): """Generic PCI access library.""" @@ -13,6 +14,12 @@ class Libpciaccess(Package): depends_on('libtool') def install(self, spec, prefix): + # libpciaccess does not support OS X + if spec.satisfies('=darwin-x86_64'): + # create a dummy directory + mkdir(prefix.lib) + return + from subprocess import call call(["./autogen.sh"]) configure("--prefix=%s" % prefix) From 552d8ca5f083daa467cf3a852fec76be21eec8ee Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 22 Dec 2015 16:40:09 -0500 Subject: [PATCH 203/910] Update rsync to 3.1.2 --- var/spack/packages/rsync/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/rsync/package.py b/var/spack/packages/rsync/package.py index 8ae21b1cb9b..76aec3096dd 100644 --- a/var/spack/packages/rsync/package.py +++ b/var/spack/packages/rsync/package.py @@ -5,6 +5,7 @@ class Rsync(Package): homepage = "https://rsync.samba.org" url = "https://download.samba.org/pub/rsync/rsync-3.1.1.tar.gz" + version('3.1.2', '0f758d7e000c0f7f7d3792610fad70cb') version('3.1.1', '43bd6676f0b404326eee2d63be3cdcfe') def install(self, spec, prefix): From a540a68561db4067b66b4d4d0920b217fea4fda4 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 22 Dec 2015 18:51:49 -0500 Subject: [PATCH 204/910] Make OpenSSL build on Darwin --- var/spack/packages/openssl/package.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index 40648fca492..82c315da33d 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -17,12 +17,24 @@ class Openssl(Package): parallel = False def install(self, spec, prefix): - config = Executable("./config") - config("--prefix=%s" % prefix, - "--openssldir=%s/etc/openssl" % prefix, - "zlib", - "no-krb5", - "shared") + if spec.satisfies("=darwin-x86_64"): + perl = which('perl') + perl("./Configure", + "--prefix=%s" % prefix, + "--openssldir=%s/etc/openssl" % prefix, + "zlib", + "no-krb5", + "shared", + "darwin64-x86_64-cc") + perl('-pi', '-e', 's/-arch x86_64//g', 'Makefile') + else: + exit(1) + config = Executable("./config") + config("--prefix=%s" % prefix, + "--openssldir=%s/etc/openssl" % prefix, + "zlib", + "no-krb5", + "shared") make() make("install") From 60ec79eac3f185f6389166ff801a767c66c7470e Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 22 Dec 2015 18:53:43 -0500 Subject: [PATCH 205/910] New package PCRE2 This is a major update of the package PCRE. It seems both PCRE and PCRE2 can be installed at the same time; their APIs are different. --- var/spack/packages/pcre2/package.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 var/spack/packages/pcre2/package.py diff --git a/var/spack/packages/pcre2/package.py b/var/spack/packages/pcre2/package.py new file mode 100644 index 00000000000..6a0244a15e9 --- /dev/null +++ b/var/spack/packages/pcre2/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Pcre2(Package): + """The PCRE2 package contains Perl Compatible Regular Expression + libraries. These are useful for implementing regular expression + pattern matching using the same syntax and semantics as Perl 5.""" + homepage = "http://www.pcre.org""" + url = "ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre2-10.20.tar.bz2" + + version('10.20', 'dcd027c57ecfdc8a6c3af9d0acf5e3f7') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") From 2b89d9b1db620c3b4a0e391ffc6621717aa03382 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 22 Dec 2015 16:54:41 -0800 Subject: [PATCH 206/910] More consistent URL parsing when finding versions. Versions found by wildcard URLs are different from versions found by parse_version, etc. The wildcards are constructed more haphazardly than the very specific URL patterns in url.py, so they can get things wrong. e.g., for this URL: https://software.lanl.gov/MeshTools/trac/attachment/wiki/WikiStart/mstk-2.25rc1.tgz We miss the 'rc' and only return 2.25r as the version if we ONLY use URL wildcards. Future: Maybe use the regexes from url.py to scrape web pages, and then compare them for similarity with the original URL, instead of trying to make a structured wildcard URL pattern? This might yield better results. --- lib/spack/spack/package.py | 12 +++++++++--- lib/spack/spack/url.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b95afb073df..2e7678b4b03 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1200,6 +1200,8 @@ def find_versions_of_archive(*archive_urls, **kwargs): for aurl in archive_urls: list_urls.add(spack.url.find_list_url(aurl)) + print list_urls + # Grab some web pages to scrape. page_map = {} for lurl in list_urls: @@ -1224,9 +1226,13 @@ def find_versions_of_archive(*archive_urls, **kwargs): for page_url, content in page_map.iteritems(): # extract versions from matches. for regex in regexes: - versions.update( - (Version(m.group(1)), urljoin(page_url, m.group(0))) - for m in re.finditer(regex, content)) + print regex + print + + for m in re.finditer(regex, content): + url = urljoin(page_url, m.group(0)) + ver = spack.url.parse_version(url) + versions[ver] = url return versions diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 02c0b83e260..ed5a6d2a44f 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -210,7 +210,7 @@ def parse_version_offset(path): (r'-((\d+\.)*\d+)$', stem), # e.g. foobar-4.5.1b, foobar4.5RC, foobar.v4.5.1b - (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP?)\d*)$', stem), + (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP)?\d*)$', stem), # e.g. foobar-4.5.0-beta1, or foobar-4.50-beta (r'-((\d+\.)*\d+-beta(\d+)?)$', stem), From 0d4b77a24b883d15ee2c297a983b8a8dc807de6f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 22 Dec 2015 17:02:14 -0800 Subject: [PATCH 207/910] revert accidental print statements. --- lib/spack/spack/package.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 2e7678b4b03..6673e4f3929 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1200,8 +1200,6 @@ def find_versions_of_archive(*archive_urls, **kwargs): for aurl in archive_urls: list_urls.add(spack.url.find_list_url(aurl)) - print list_urls - # Grab some web pages to scrape. page_map = {} for lurl in list_urls: @@ -1226,9 +1224,6 @@ def find_versions_of_archive(*archive_urls, **kwargs): for page_url, content in page_map.iteritems(): # extract versions from matches. for regex in regexes: - print regex - print - for m in re.finditer(regex, content): url = urljoin(page_url, m.group(0)) ver = spack.url.parse_version(url) From d63cb8b537a9fc12a2a1ee5b22f2b20e19d90dc1 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 22 Dec 2015 17:05:23 -0800 Subject: [PATCH 208/910] Fix bug in URL regex. --- lib/spack/spack/url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index ed5a6d2a44f..02c0b83e260 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -210,7 +210,7 @@ def parse_version_offset(path): (r'-((\d+\.)*\d+)$', stem), # e.g. foobar-4.5.1b, foobar4.5RC, foobar.v4.5.1b - (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP)?\d*)$', stem), + (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP?)\d*)$', stem), # e.g. foobar-4.5.0-beta1, or foobar-4.50-beta (r'-((\d+\.)*\d+-beta(\d+)?)$', stem), From 21dae4577a6990384a4fe330cdef09b902d6044e Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 23 Dec 2015 10:20:13 +0100 Subject: [PATCH 209/910] R : updated package --- var/spack/packages/R/package.py | 66 ++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/var/spack/packages/R/package.py b/var/spack/packages/R/package.py index 2e6f65a7429..2471dff09b6 100644 --- a/var/spack/packages/R/package.py +++ b/var/spack/packages/R/package.py @@ -1,33 +1,49 @@ from spack import * -class R(Package): - """R is 'GNU S', a freely available language and environment for - statistical computing and graphics which provides a wide va - riety of statistical and graphical techniques: linear and - nonlinear modelling, statistical tests, time series analysis, - classification, clustering, etc. Please consult the R project - homepage for further information.""" - homepage = "http://www.example.com" - url = "http://cran.cnr.berkeley.edu/src/base/R-3/R-3.1.2.tar.gz" +class R(Package): + """ + R is 'GNU S', a freely available language and environment for statistical computing and graphics which provides a + wide variety of statistical and graphical techniques: linear and nonlinear modelling, statistical tests, time series + analysis, classification, clustering, etc. Please consult the R project homepage for further information. + """ + homepage = "https://www.r-project.org" + url = "http://cran.cnr.berkeley.edu/src/base/R-3/R-3.1.2.tar.gz" + + version('3.2.3', '1ba3dac113efab69e706902810cc2970') + version('3.2.2', '57cef5c2e210a5454da1979562a10e5b') + version('3.2.1', 'c2aac8b40f84e08e7f8c9068de9239a3') + version('3.2.0', '66fa17ad457d7e618191aa0f52fc402e') + version('3.1.3', '53a85b884925aa6b5811dfc361d73fc4') version('3.1.2', '3af29ec06704cbd08d4ba8d69250ae74') - depends_on("readline") - depends_on("ncurses") - depends_on("icu") - depends_on("glib") - depends_on("zlib") - depends_on("libtiff") - depends_on("jpeg") - depends_on("cairo") - depends_on("pango") - depends_on("freetype") - depends_on("tcl") - depends_on("tk") + variant('external-lapack', default=False, description='Links to externally installed BLAS/LAPACK') + + # Virtual dependencies + depends_on('blas', when='+external-lapack') + depends_on('lapack', when='+external-lapack') + + # Concrete dependencies + depends_on('readline') + depends_on('ncurses') + depends_on('icu') + depends_on('glib') + depends_on('zlib') + depends_on('libtiff') + depends_on('jpeg') + depends_on('cairo') + depends_on('pango') + depends_on('freetype') + depends_on('tcl') + depends_on('tk') def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--enable-R-shlib", - "--enable-BLAS-shlib") + options = ['--prefix=%s' % prefix, + '--enable-R-shlib', + '--enable-BLAS-shlib'] + if '+external-lapack' in spec: + options.extend(['--with-blas', '--with-lapack']) + + configure(*options) make() - make("install") + make('install') From 5ca5884ad6285fd766a2f704bceb40b1cf63750f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 23 Dec 2015 13:13:07 -0800 Subject: [PATCH 210/910] Fix #235: Smarter web spidering -- use parsed links instead of reconstructing. - Move `find_versions_of_archive` from spack.package to `spack.util.web`. - `spider` funciton now just uses the link parsing it already does to return links. We evaluate actual links found in the scraped pages instead of trying to reconstruct them naively. - Add `spack url-parse` command, which you can use to show how Spack interprets the name and version in a URL. --- lib/spack/spack/cmd/create.py | 4 +- lib/spack/spack/cmd/url-parse.py | 75 ++++++++++++++++ lib/spack/spack/package.py | 46 +--------- lib/spack/spack/util/web.py | 148 +++++++++++++++++++++++-------- 4 files changed, 187 insertions(+), 86 deletions(-) create mode 100644 lib/spack/spack/cmd/url-parse.py diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 5e42860f3e7..9ecb709110c 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -34,8 +34,8 @@ import spack import spack.cmd import spack.cmd.checksum -import spack.package import spack.url +import spack.util.web from spack.util.naming import * import spack.util.crypto as crypto @@ -166,7 +166,7 @@ def create(parser, args): tty.msg("This looks like a URL for %s version %s." % (name, version)) tty.msg("Creating template for package %s" % name) - versions = spack.package.find_versions_of_archive(url) + versions = spack.util.web.find_versions_of_archive(url) rkeys = sorted(versions.keys(), reverse=True) versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys))) diff --git a/lib/spack/spack/cmd/url-parse.py b/lib/spack/spack/cmd/url-parse.py new file mode 100644 index 00000000000..077c793d2ee --- /dev/null +++ b/lib/spack/spack/cmd/url-parse.py @@ -0,0 +1,75 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import sys + +import llnl.util.tty as tty + +import spack +import spack.url +from spack.util.web import find_versions_of_archive + +description = "Show parsing of a URL, optionally spider web for other versions." + +def setup_parser(subparser): + subparser.add_argument('url', help="url of a package archive") + subparser.add_argument( + '-s', '--spider', action='store_true', help="Spider the source page for versions.") + + +def print_name_and_version(url): + name, ns, nl, ntup, ver, vs, vl, vtup = spack.url.substitution_offsets(url) + underlines = [" "] * max(ns+nl, vs+vl) + for i in range(ns, ns+nl): + underlines[i] = '-' + for i in range(vs, vs+vl): + underlines[i] = '~' + + print " %s" % url + print " %s" % ''.join(underlines) + + +def url_parse(parser, args): + url = args.url + + ver, vs, vl = spack.url.parse_version_offset(url) + name, ns, nl = spack.url.parse_name_offset(url, ver) + + tty.msg("Parsing URL:") + try: + print_name_and_version(url) + except spack.url.UrlParseError as e: + tty.error(str(e)) + + print + tty.msg("Substituting version 9.9.9b:") + newurl = spack.url.substitute_version(url, '9.9.9b') + print_name_and_version(newurl) + + if args.spider: + print + tty.msg("Spidering for versions:") + versions = find_versions_of_archive(url) + for v in sorted(versions): + print "%-20s%s" % (v, versions[v]) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 6673e4f3929..b44554e4183 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1164,7 +1164,7 @@ def fetch_remote_versions(self): raise VersionFetchError(self.__class__) try: - return find_versions_of_archive( + return spack.util.web.find_versions_of_archive( *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) except spack.error.NoNetworkConnectionError, e: tty.die("Package.fetch_versions couldn't connect to:", @@ -1188,50 +1188,6 @@ def rpath_args(self): return " ".join("-Wl,-rpath=%s" % p for p in self.rpath) -def find_versions_of_archive(*archive_urls, **kwargs): - list_url = kwargs.get('list_url', None) - list_depth = kwargs.get('list_depth', 1) - - # Generate a list of list_urls based on archive urls and any - # explicitly listed list_url in the package - list_urls = set() - if list_url: - list_urls.add(list_url) - for aurl in archive_urls: - list_urls.add(spack.url.find_list_url(aurl)) - - # Grab some web pages to scrape. - page_map = {} - for lurl in list_urls: - pages = spack.util.web.get_pages(lurl, depth=list_depth) - page_map.update(pages) - - # Scrape them for archive URLs - regexes = [] - for aurl in archive_urls: - # This creates a regex from the URL with a capture group for - # the version part of the URL. The capture group is converted - # to a generic wildcard, so we can use this to extract things - # on a page that look like archive URLs. - url_regex = spack.url.wildcard_version(aurl) - - # We'll be a bit more liberal and just look for the archive - # part, not the full path. - regexes.append(os.path.basename(url_regex)) - - # Build a version list from all the matches we find - versions = {} - for page_url, content in page_map.iteritems(): - # extract versions from matches. - for regex in regexes: - for m in re.finditer(regex, content): - url = urljoin(page_url, m.group(0)) - ver = spack.url.parse_version(url) - versions[ver] = url - - return versions - - def validate_package_url(url_string): """Determine whether spack can handle a particular URL or not.""" url = urlparse(url_string) diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 94384e9c86f..e26daef2964 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import re +import os import sys import subprocess import urllib2, cookielib @@ -70,7 +71,9 @@ def _spider(args): """ url, visited, root, opener, depth, max_depth, raise_on_error = args - pages = {} + pages = {} # dict from page URL -> text content. + links = set() # set of all links seen on visited pages. + try: # Make a HEAD request first to check the content type. This lets # us ignore tarballs and gigantic files. @@ -99,42 +102,45 @@ def _spider(args): page = response.read() pages[response_url] = page - # If we're not at max depth, parse out the links in the page + # Parse out the links in the page + link_parser = LinkParser() + subcalls = [] + link_parser.feed(page) + + while link_parser.links: + raw_link = link_parser.links.pop() + abs_link = urlparse.urljoin(response_url, raw_link) + + links.add(abs_link) + + # Skip stuff that looks like an archive + if any(raw_link.endswith(suf) for suf in ALLOWED_ARCHIVE_TYPES): + continue + + # Skip things outside the root directory + if not abs_link.startswith(root): + continue + + # Skip already-visited links + if abs_link in visited: + continue + + # If we're not at max depth, follow links. if depth < max_depth: - link_parser = LinkParser() - subcalls = [] - link_parser.feed(page) + subcalls.append((abs_link, visited, root, None, + depth+1, max_depth, raise_on_error)) + visited.add(abs_link) - while link_parser.links: - raw_link = link_parser.links.pop() - - # Skip stuff that looks like an archive - if any(raw_link.endswith(suf) for suf in ALLOWED_ARCHIVE_TYPES): - continue - - # Evaluate the link relative to the page it came from. - abs_link = urlparse.urljoin(response_url, raw_link) - - # Skip things outside the root directory - if not abs_link.startswith(root): - continue - - # Skip already-visited links - if abs_link in visited: - continue - - subcalls.append((abs_link, visited, root, None, depth+1, max_depth, raise_on_error)) - visited.add(abs_link) - - if subcalls: - try: - pool = Pool(processes=len(subcalls)) - dicts = pool.map(_spider, subcalls) - for d in dicts: - pages.update(d) - finally: - pool.terminate() - pool.join() + if subcalls: + try: + pool = Pool(processes=len(subcalls)) + results = pool.map(_spider, subcalls) + for sub_pages, sub_links in results: + pages.update(sub_pages) + links.update(sub_links) + finally: + pool.terminate() + pool.join() except urllib2.URLError, e: tty.debug(e) @@ -155,10 +161,10 @@ def _spider(args): # Other types of errors are completely ignored, except in debug mode. tty.debug("Error in _spider: %s" % e) - return pages + return pages, links -def get_pages(root_url, **kwargs): +def spider(root_url, **kwargs): """Gets web pages from a root URL. If depth is specified (e.g., depth=2), then this will also fetches pages linked from the root and its children up to depth. @@ -167,5 +173,69 @@ def get_pages(root_url, **kwargs): performance over a sequential fetch. """ max_depth = kwargs.setdefault('depth', 1) - pages = _spider((root_url, set(), root_url, None, 1, max_depth, False)) - return pages + pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False)) + return pages, links + + +def find_versions_of_archive(*archive_urls, **kwargs): + """Scrape web pages for new versions of a tarball. + + Arguments: + archive_urls: + URLs for different versions of a package. Typically these + are just the tarballs from the package file itself. By + default, this searches the parent directories of archives. + + Keyword Arguments: + list_url: + + URL for a listing of archives. Spack wills scrape these + pages for download links that look like the archive URL. + + list_depth: + Max depth to follow links on list_url pages. + + """ + list_url = kwargs.get('list_url', None) + list_depth = kwargs.get('list_depth', 1) + + # Generate a list of list_urls based on archive urls and any + # explicitly listed list_url in the package + list_urls = set() + if list_url: + list_urls.add(list_url) + for aurl in archive_urls: + list_urls.add(spack.url.find_list_url(aurl)) + + # Grab some web pages to scrape. + pages = {} + links = set() + for lurl in list_urls: + p, l = spider(lurl, depth=list_depth) + pages.update(p) + links.update(l) + + # Scrape them for archive URLs + regexes = [] + for aurl in archive_urls: + # This creates a regex from the URL with a capture group for + # the version part of the URL. The capture group is converted + # to a generic wildcard, so we can use this to extract things + # on a page that look like archive URLs. + url_regex = spack.url.wildcard_version(aurl) + + # We'll be a bit more liberal and just look for the archive + # part, not the full path. + regexes.append(os.path.basename(url_regex)) + + # Build a dict version -> URL from any links that match the wildcards. + versions = {} + for url in links: + if any(re.search(r, url) for r in regexes): + try: + ver = spack.url.parse_version(url) + versions[ver] = url + except spack.url.UndetectableVersionError as e: + continue + + return versions From a58dd8f6abb00281af1b469981dce760cd1c55ff Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 23 Dec 2015 18:19:26 -0500 Subject: [PATCH 211/910] Remove leftover debug statement --- var/spack/packages/openssl/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index 82c315da33d..a51bace6665 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -28,7 +28,6 @@ def install(self, spec, prefix): "darwin64-x86_64-cc") perl('-pi', '-e', 's/-arch x86_64//g', 'Makefile') else: - exit(1) config = Executable("./config") config("--prefix=%s" % prefix, "--openssldir=%s/etc/openssl" % prefix, From 309877088d81dcac3ed4e45327ab83a99957e93b Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 22 Dec 2015 18:54:47 -0500 Subject: [PATCH 212/910] Build Julia --- var/spack/packages/julia/package.py | 55 +++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 var/spack/packages/julia/package.py diff --git a/var/spack/packages/julia/package.py b/var/spack/packages/julia/package.py new file mode 100644 index 00000000000..2c67b8e4439 --- /dev/null +++ b/var/spack/packages/julia/package.py @@ -0,0 +1,55 @@ +from spack import * + +class Julia(Package): + """The Julia Language: A fresh approach to technical computing""" + homepage = "http://julialang.org" + url = "http://github.com/JuliaLang/julia/releases/download/v0.4.2/julia-0.4.2.tar.gz" + + version('0.4.2', 'ccfeb4f4090c8b31083f5e1ccb03eb06') + + # Build-time dependencies + # depends_on("cmake") + # depends_on("awk") + # depends_on("m4") + # depends_on("pkg-config") + + # Run-time dependencies + #TODO depends_on("arpack") + #TODO depends_on("fftw") + #TODO depends_on("git") + #TODO depends_on("gmp") + # depends_on("libedit") + #TODO depends_on("mpfr") + #TODO depends_on("pcre2") + # depends_on("zlib") + + # BLAS and LAPACK: Julia prefers 64-bit versions on 64-bit systems. OpenBLAS + # has an option for this; make it available as variant. + + # LLVM: Julia works only with specific versions, and might require patches. + # Thus we let Julia install its own LLVM. + + # Other possible dependencies: + # USE_SYSTEM_OPENLIBM=0 + # USE_SYSTEM_OPENSPECFUN=0 + # USE_SYSTEM_DSFMT=0 + # USE_SYSTEM_SUITESPARSE=0 + # USE_SYSTEM_UTF8PROC=0 + # USE_SYSTEM_LIBGIT2=0 + + def install(self, spec, prefix): + options = ["CC=cc", + "CXX=c++", + "FC=fc", + #TODO "USE_SYSTEM_ARPACK=1", + #TODO "USE_SYSTEM_FFTW=1", + #TODO "USE_SYSTEM_GMP=1", + #TODO "USE_SYSTEM_MPFR=1", + #TODO "USE_SYSTEM_PCRE=1", + "prefix=%s" % prefix] + with open('Make.user', 'w') as f: + f.write('\n'.join(options) + '\n') + which('env')() + which('env')('which', 'cc') + make() + make("install") From 51f546fe92e46b9737ecf779d2f30787105c95ff Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 23 Dec 2015 08:40:55 -0500 Subject: [PATCH 213/910] Add comments to package file --- var/spack/packages/julia/package.py | 45 ++++++++++++++++++----------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/var/spack/packages/julia/package.py b/var/spack/packages/julia/package.py index 2c67b8e4439..9fd946c9054 100644 --- a/var/spack/packages/julia/package.py +++ b/var/spack/packages/julia/package.py @@ -1,4 +1,5 @@ from spack import * +import os class Julia(Package): """The Julia Language: A fresh approach to technical computing""" @@ -13,19 +14,27 @@ class Julia(Package): # depends_on("m4") # depends_on("pkg-config") + # I think that Julia requires the dependencies above, but it builds find (on + # my system) without these. We should enable them as necessary. + # Run-time dependencies - #TODO depends_on("arpack") - #TODO depends_on("fftw") - #TODO depends_on("git") - #TODO depends_on("gmp") - # depends_on("libedit") - #TODO depends_on("mpfr") - #TODO depends_on("pcre2") - # depends_on("zlib") + # depends_on("arpack") + # depends_on("fftw +float") + # depends_on("gmp") + # depends_on("mpfr") + # depends_on("pcre2") + + # ARPACK: Requires BLAS and LAPACK; needs to use the same version as Julia. # BLAS and LAPACK: Julia prefers 64-bit versions on 64-bit systems. OpenBLAS # has an option for this; make it available as variant. + # FFTW: Something doesn't work when using a pre-installed FFTW library; need + # to investigate. + + # GMP, MPFR: Something doesn't work when using a pre-installed FFTW library; + # need to investigate. + # LLVM: Julia works only with specific versions, and might require patches. # Thus we let Julia install its own LLVM. @@ -38,18 +47,20 @@ class Julia(Package): # USE_SYSTEM_LIBGIT2=0 def install(self, spec, prefix): - options = ["CC=cc", - "CXX=c++", - "FC=fc", - #TODO "USE_SYSTEM_ARPACK=1", - #TODO "USE_SYSTEM_FFTW=1", - #TODO "USE_SYSTEM_GMP=1", - #TODO "USE_SYSTEM_MPFR=1", + # Explicitly setting CC, CXX, or FC breaks building libuv, one of + # Julia's dependencies. This might be a Darwin-specific problem. Given + # how Spack sets up compilers, Julia should still use Spack's compilers, + # even if we don't specify them explicitly. + options = [#"CC=cc", + #"CXX=c++", + #"FC=fc", + #"USE_SYSTEM_ARPACK=1", + #"USE_SYSTEM_FFTW=1", + #"USE_SYSTEM_GMP=1", + #"USE_SYSTEM_MPFR=1", #TODO "USE_SYSTEM_PCRE=1", "prefix=%s" % prefix] with open('Make.user', 'w') as f: f.write('\n'.join(options) + '\n') - which('env')() - which('env')('which', 'cc') make() make("install") From f37f872e5f4920f9dc22237deed5895b94700136 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 23 Dec 2015 16:23:58 -0800 Subject: [PATCH 214/910] Fix #85 and #228: errors fetching VCS packages from a mirror. - Stage and fetcher were not being set up properly when fetching using a different fetch strategy than the default one for the package. - This is fixed but fetch/stage/mirror logic is still too complicated and long-term needs a rethink. - Spack will now print a warning when fetching a checksum-less tarball from a mirror -- users should be careful to use https or local filesystem mirrors for this. --- lib/spack/spack/fetch_strategy.py | 4 +-- lib/spack/spack/stage.py | 53 +++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index a9374fb34b9..0657146bf63 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -687,7 +687,7 @@ def for_package_version(pkg, version): class FetchError(spack.error.SpackError): - def __init__(self, msg, long_msg): + def __init__(self, msg, long_msg=None): super(FetchError, self).__init__(msg, long_msg) @@ -705,7 +705,7 @@ def __init__(self, msg, long_msg): class NoDigestError(FetchError): - def __init__(self, msg, long_msg): + def __init__(self, msg, long_msg=None): super(NoDigestError, self).__init__(msg, long_msg) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 754344fc016..76ca7273cb9 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -82,14 +82,18 @@ def __init__(self, url_or_fetch_strategy, **kwargs): stage object later). If name is not provided, then this stage will be given a unique name automatically. """ + # TODO: fetch/stage coupling needs to be reworked -- the logic + # TODO: here is convoluted and not modular enough. if isinstance(url_or_fetch_strategy, basestring): self.fetcher = fs.from_url(url_or_fetch_strategy) elif isinstance(url_or_fetch_strategy, fs.FetchStrategy): self.fetcher = url_or_fetch_strategy else: raise ValueError("Can't construct Stage without url or fetch strategy") - self.fetcher.set_stage(self) + self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. + self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. + self.name = kwargs.get('name') self.mirror_path = kwargs.get('mirror_path') @@ -198,17 +202,18 @@ def _setup(self): @property def archive_file(self): """Path to the source archive within this stage directory.""" - if not isinstance(self.fetcher, fs.URLFetchStrategy): - return None + paths = [] + if isinstance(self.fetcher, fs.URLFetchStrategy): + paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url))) - paths = [os.path.join(self.path, os.path.basename(self.fetcher.url))] if self.mirror_path: paths.append(os.path.join(self.path, os.path.basename(self.mirror_path))) for path in paths: if os.path.exists(path): return path - return None + else: + return None @property @@ -238,23 +243,34 @@ def fetch(self): """Downloads an archive or checks out code from a repository.""" self.chdir() - fetchers = [self.fetcher] + fetchers = [self.default_fetcher] # TODO: move mirror logic out of here and clean it up! + # TODO: Or @alalazo may have some ideas about how to use a + # TODO: CompositeFetchStrategy here. + self.skip_checksum_for_mirror = True if self.mirror_path: urls = ["%s/%s" % (m, self.mirror_path) for m in _get_mirrors()] + # If this archive is normally fetched from a tarball URL, + # then use the same digest. `spack mirror` ensures that + # the checksum will be the same. digest = None - if isinstance(self.fetcher, fs.URLFetchStrategy): - digest = self.fetcher.digest - fetchers = [fs.URLFetchStrategy(url, digest) - for url in urls] + fetchers - for f in fetchers: - f.set_stage(self) + if isinstance(self.default_fetcher, fs.URLFetchStrategy): + digest = self.default_fetcher.digest + + # Have to skip the checkesum for things archived from + # repositories. How can this be made safer? + self.skip_checksum_for_mirror = not bool(digest) + + for url in urls: + fetchers.insert(0, fs.URLFetchStrategy(url, digest)) for fetcher in fetchers: try: - fetcher.fetch() + fetcher.set_stage(self) + self.fetcher = fetcher + self.fetcher.fetch() break except spack.error.SpackError, e: tty.msg("Fetching from %s failed." % fetcher) @@ -262,13 +278,22 @@ def fetch(self): continue else: errMessage = "All fetchers failed for %s" % self.name + self.fetcher = self.default_fetcher raise fs.FetchError(errMessage, None) def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" - self.fetcher.check() + if self.fetcher is not self.default_fetcher and self.skip_checksum_for_mirror: + tty.warn("Fetching from mirror without a checksum!", + "This package is normally checked out from a version " + "control system, but it has been archived on a spack " + "mirror. This means we cannot know a checksum for the " + "tarball in advance. Be sure that your connection to " + "this mirror is secure!.") + else: + self.fetcher.check() def expand_archive(self): From 01f7e768f6638add78e213f0430d091c471cb230 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 23 Dec 2015 22:49:42 -0500 Subject: [PATCH 215/910] Enable several variants for HDF5 +cxx, +fortran: Enable support for the respective languages +threadsafe: Enable multi-threading --- var/spack/packages/hdf5/package.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/hdf5/package.py b/var/spack/packages/hdf5/package.py index adac79d9bbd..fdc77c121a9 100644 --- a/var/spack/packages/hdf5/package.py +++ b/var/spack/packages/hdf5/package.py @@ -10,12 +10,15 @@ class Hdf5(Package): url = "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.13/src/hdf5-1.8.13.tar.gz" list_url = "http://www.hdfgroup.org/ftp/HDF5/releases" list_depth = 3 - + version('1.8.16', 'b8ed9a36ae142317f88b0c7ef4b9c618') version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') version('1.8.13', 'c03426e9e77d7766944654280b467289') + variant('cxx', default=False, description='Enable C++ support') + variant('fortran', default=False, description='Enable Fortran support') variant('mpi', default=False, description='Enable MPI support') + variant('threadsafe', default=False, description='Enable multithreading') depends_on("mpi", when='+mpi') depends_on("zlib") @@ -23,11 +26,36 @@ class Hdf5(Package): # TODO: currently hard-coded to use OpenMPI def install(self, spec, prefix): extra_args = [] + if '+cxx' in spec: + extra_args.extend([ + '--enable-cxx' + ]) + if '+fortran' in spec: + extra_args.extend([ + '--enable-fortran', + '--enable-fortran2003' + ]) if '+mpi' in spec: + # The HDF5 configure script warns if cxx and mpi are enabled + # together. There doesn't seem to be a real reason for this, except + # that parts of the MPI interface are not accessible via the C++ + # interface. Since they are still accessible via the C interface, + # this is not actually a problem. extra_args.extend([ "--enable-parallel", + "--enable-unsupported", "CC=%s" % spec['mpi'].prefix.bin + "/mpicc", "CXX=%s" % spec['mpi'].prefix.bin + "/mpic++", + "FC=%s" % spec['mpi'].prefix.bin + "/mpifort", + ]) + if '+threads' in spec: + if '+cxx' in spec or '+fortran' in spec: + die("Cannot use variant +threads with either +cxx or +fortran") + extra_args.extend([ + '--enable-threadsafe', + '--disable-hl', + 'CPPFLAGS=-DHDatexit=""', + 'CFLAGS=-DHDatexit=""' ]) configure( From 52625de52c396b995716c0451ba169df0f3a4b1f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 29 Nov 2015 16:02:51 -0800 Subject: [PATCH 216/910] Fix #154 -- better log messages for do_patch() --- lib/spack/spack/package.py | 40 +++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b44554e4183..84bcb15f7fe 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -733,9 +733,10 @@ def do_patch(self): # Construct paths to special files in the archive dir used to # keep track of whether patches were successfully applied. - archive_dir = self.stage.source_path - good_file = join_path(archive_dir, '.spack_patched') - bad_file = join_path(archive_dir, '.spack_patch_failed') + archive_dir = self.stage.source_path + good_file = join_path(archive_dir, '.spack_patched') + no_patches_file = join_path(archive_dir, '.spack_no_patches') + bad_file = join_path(archive_dir, '.spack_patch_failed') # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. @@ -749,29 +750,46 @@ def do_patch(self): if os.path.isfile(good_file): tty.msg("Already patched %s" % self.name) return + elif os.path.isfile(no_patches_file): + tty.msg("No patches needed for %s." % self.name) + return # Apply all the patches for specs that match this one + patched = False for spec, patch_list in self.patches.items(): if self.spec.satisfies(spec): for patch in patch_list: - tty.msg('Applying patch %s' % patch.path_or_url) try: patch.apply(self.stage) + tty.msg('Applied patch %s' % patch.path_or_url) + patched = True except: # Touch bad file if anything goes wrong. + tty.msg('Patch %s failed.' % patch.path_or_url) touch(bad_file) raise - # patch succeeded. Get rid of failed file & touch good file so we - # don't try to patch again again next time. + if has_patch_fun: + try: + self.patch() + tty.msg("Ran patch() for %s." % self.name) + patched = True + except: + tty.msg("patch() function failed for %s." % self.name) + touch(bad_file) + raise + + # Get rid of any old failed file -- patches have either succeeded + # or are not needed. This is mostly defensive -- it's needed + # if the restage() method doesn't clean *everything* (e.g., for a repo) if os.path.isfile(bad_file): os.remove(bad_file) - touch(good_file) - if has_patch_fun: - self.patch() - - tty.msg("Patched %s" % self.name) + # touch good or no patches file so that we skip next time. + if patched: + touch(good_file) + else: + touch(no_patches_file) def do_fake_install(self): From be12720eb797e5a3a6031067248035db7d291962 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 25 Dec 2015 14:02:56 -0500 Subject: [PATCH 217/910] Use `filter_file` instead of calling perl --- var/spack/packages/openssl/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index a51bace6665..e1039fcbe12 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -26,7 +26,7 @@ def install(self, spec, prefix): "no-krb5", "shared", "darwin64-x86_64-cc") - perl('-pi', '-e', 's/-arch x86_64//g', 'Makefile') + filter_file(r'-arch x86_64', '', 'Makefile') else: config = Executable("./config") config("--prefix=%s" % prefix, From 34401cf0c3ee019fd6b8fb739e2a511c6de0870c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 25 Dec 2015 14:00:33 -0800 Subject: [PATCH 218/910] Rework Spack config: keep user & site config in memory. - User and site config are now kept separately in memory. - Merging is done on demand when client code requests the configuration. - Allows user/site config to be updated independently of each other by commands. - simplifies config logic (no more tracking merged files) --- .gitignore | 1 + lib/spack/spack/cmd/compiler.py | 7 +- lib/spack/spack/cmd/config.py | 12 +- lib/spack/spack/cmd/mirror.py | 4 +- lib/spack/spack/compilers/__init__.py | 76 ++-- lib/spack/spack/config.py | 409 +++++++++++---------- lib/spack/spack/spec.py | 1 + lib/spack/spack/stage.py | 5 +- lib/spack/spack/test/config.py | 21 +- lib/spack/spack/test/database.py | 3 +- lib/spack/spack/test/mock_packages_test.py | 6 +- 11 files changed, 281 insertions(+), 264 deletions(-) diff --git a/.gitignore b/.gitignore index 1c6ca4c99e1..4b97de5d507 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ /etc/spackconfig /share/spack/dotkit /share/spack/modules +/TAGS diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 2a64dc914ed..6efc9a3347a 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -65,10 +65,11 @@ def compiler_add(args): if c.spec not in spack.compilers.all_compilers()] if compilers: - spack.compilers.add_compilers_to_config('user', *compilers) + spack.compilers.add_compilers_to_config('user', compilers) n = len(compilers) - tty.msg("Added %d new compiler%s to %s" % ( - n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) + s = 's' if n > 1 else '' + filename = spack.config.get_config_filename('user', 'compilers') + tty.msg("Added %d new compiler%s to %s" % (n, s, filename)) colify(reversed(sorted(c.spec for c in compilers)), indent=4) else: tty.msg("Found no new compilers") diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index 8c18f88b648..603023d891a 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -44,22 +44,22 @@ def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='config_command') get_parser = sp.add_parser('get', help='Print configuration values.') - get_parser.add_argument('category', help="Configuration category to print.") + get_parser.add_argument('section', help="Configuration section to print.") edit_parser = sp.add_parser('edit', help='Edit configuration file.') - edit_parser.add_argument('category', help="Configuration category to edit") + edit_parser.add_argument('section', help="Configuration section to edit") def config_get(args): - spack.config.print_category(args.category) + spack.config.print_section(args.section) def config_edit(args): if not args.scope: args.scope = 'user' - if not args.category: - args.category = None - config_file = spack.config.get_config_scope_filename(args.scope, args.category) + if not args.section: + args.section = None + config_file = spack.config.get_config_filename(args.scope, args.section) spack.editor(config_file) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 9a507e69db3..2b257939278 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -76,7 +76,7 @@ def mirror_add(args): url = 'file://' + url mirror_dict = { args.name : url } - spack.config.add_to_mirror_config({ args.name : url }) + spack.config.update_config('mirrors', { args.name : url }, 'user') def mirror_remove(args): @@ -90,7 +90,7 @@ def mirror_remove(args): def mirror_list(args): """Print out available mirrors to the console.""" - sec_names = spack.config.get_mirror_config() + sec_names = spack.config.get_config('mirrors') if not sec_names: tty.msg("No mirrors configured.") return diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index b7b021a1ac8..a1980f1cdfa 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -35,6 +35,7 @@ import spack.error import spack.spec import spack.config +import spack.architecture from spack.util.multiproc import parmap from spack.compiler import Compiler @@ -55,23 +56,48 @@ def converter(cspec_like): return converter -def _get_config(): - """Get a Spack config, but make sure it has compiler configuration - first.""" +def _to_dict(compiler): + """Return a dict version of compiler suitable to insert in YAML.""" + return { + str(compiler.spec) : dict( + (attr, getattr(compiler, attr, None)) + for attr in _required_instance_vars) + } + + +def get_compiler_config(arch=None): + """Return the compiler configuration for the specified architecture. + + If the compiler configuration designates some compilers for + 'all' architectures, those are merged into the result, as well. + + """ # If any configuration file has compilers, just stick with the # ones already configured. - config = spack.config.get_compilers_config() - existing = [spack.spec.CompilerSpec(s) - for s in config] - if existing: - return config + config = spack.config.get_config('compilers') - compilers = find_compilers(*get_path('PATH')) - add_compilers_to_config('user', *compilers) + if arch is None: + arch = spack.architecture.sys_type() - # After writing compilers to the user config, return a full config - # from all files. - return spack.config.get_compilers_config() + if arch not in config: + config[arch] = {} + compilers = find_compilers(*get_path('PATH')) + for compiler in compilers: + config[arch].update(_to_dict(compiler)) + spack.config.update_config('compilers', config, 'user') + + # Merge 'all' compilers with arch-specific ones. + merged_config = config.get('all', {}) + merged_config = spack.config._merge_yaml(merged_config, config[arch]) + + return merged_config + + +def all_compilers(arch=None): + """Return a set of specs for all the compiler versions currently + available to build with. These are instances of CompilerSpec. + """ + return [spack.spec.CompilerSpec(s) for s in get_compiler_config(arch)] _cached_default_compiler = None @@ -123,20 +149,6 @@ def find_compilers(*path): return clist -def add_compilers_to_config(scope, *compilers): - compiler_config_tree = {} - for compiler in compilers: - compiler_entry = {} - for c in _required_instance_vars: - val = getattr(compiler, c) - if not val: - val = "None" - compiler_entry[c] = val - compiler_config_tree[str(compiler.spec)] = compiler_entry - spack.config.add_to_compiler_config(compiler_config_tree, scope) - - - def supported_compilers(): """Return a set of names of compilers supported by Spack. @@ -152,14 +164,6 @@ def supported(compiler_spec): return compiler_spec.name in supported_compilers() -def all_compilers(): - """Return a set of specs for all the compiler versions currently - available to build with. These are instances of CompilerSpec. - """ - configuration = _get_config() - return [spack.spec.CompilerSpec(s) for s in configuration] - - @_auto_compiler_spec def find(compiler_spec): """Return specs of available compilers that match the supplied @@ -172,7 +176,7 @@ def compilers_for_spec(compiler_spec): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ - config = _get_config() + config = get_compiler_config() def get_compiler(cspec): items = config[str(cspec)] diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 36bf8a7fc33..7d7a87c7dc6 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -67,25 +67,54 @@ categorize entries beneath them in the tree. At the root of the tree, entries like ''cc'' and ''cxx'' are specified as name/value pairs. -Spack returns these trees as nested dicts. The dict for the above example -would looks like: +``config.get_config()`` returns these trees as nested dicts, but it +strips the first level off. So, ``config.get_config('compilers')`` +would return something like this for the above example: - { 'compilers' : - { 'chaos_5_x86_64_ib' : - { 'gcc@4.4.7' : - { 'cc' : '/usr/bin/gcc', - 'cxx' : '/usr/bin/g++' - 'f77' : '/usr/bin/gfortran' - 'fc' : '/usr/bin/gfortran' } - } - { 'bgqos_0' : - { 'cc' : '/usr/local/bin/mpixlc' } - } - } + { 'chaos_5_x86_64_ib' : + { 'gcc@4.4.7' : + { 'cc' : '/usr/bin/gcc', + 'cxx' : '/usr/bin/g++' + 'f77' : '/usr/bin/gfortran' + 'fc' : '/usr/bin/gfortran' } + } + { 'bgqos_0' : + { 'cc' : '/usr/local/bin/mpixlc' } } -Some convenience functions, like get_mirrors_config and -``get_compilers_config`` may strip off the top-levels of the tree and -return subtrees. +Likewise, the ``mirrors.yaml`` file's first line must be ``mirrors:``, +but ``get_config()`` strips that off too. + +Precedence +=============================== + +``config.py`` routines attempt to recursively merge configuration +across scopes. So if there are ``compilers.py`` files in both the +site scope and the user scope, ``get_config('compilers')`` will return +merged dictionaries of *all* the compilers available. If a user +compiler conflicts with a site compiler, Spack will overwrite the site +configuration wtih the user configuration. If both the user and site +``mirrors.yaml`` files contain lists of mirrors, then ``get_config()`` +will return a concatenated list of mirrors, with the user config items +first. + +Sometimes, it is useful to *completely* override a site setting with a +user one. To accomplish this, you can use *two* colons at the end of +a key in a configuration file. For example, this: + + compilers:: + chaos_5_x86_64_ib: + gcc@4.4.7: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + bgqos_0: + xlc@12.1: + cc: /usr/local/bin/mpixlc + ... + +Will make Spack take compilers *only* from the user configuration, and +the site configuration will be ignored. """ import os @@ -96,80 +125,132 @@ import llnl.util.tty as tty from llnl.util.filesystem import mkdirp -from llnl.util.lang import memoized import spack +from spack.error import SpackError + +"""List of valid config sections.""" +valid_sections = ('compilers', 'mirrors', 'repos') -_config_sections = {} -class _ConfigCategory: - name = None - filename = None - merge = True - def __init__(self, name, filename, merge, strip): - self.name = name - self.filename = filename - self.merge = merge - self.strip = strip - self.files_read_from = [] - self.result_dict = {} - _config_sections[name] = self +def check_section(section): + """Raise a ValueError if the section is not a valid section.""" + if section not in valid_sections: + raise ValueError("Invalid config section: '%s'. Options are %s." + % (section, valid_sections)) -_ConfigCategory('repos', 'repos.yaml', True, True) -_ConfigCategory('compilers', 'compilers.yaml', True, True) -_ConfigCategory('mirrors', 'mirrors.yaml', True, True) -_ConfigCategory('view', 'views.yaml', True, True) -_ConfigCategory('order', 'orders.yaml', True, True) -"""Names of scopes and their corresponding configuration files.""" -config_scopes = [('site', os.path.join(spack.etc_path, 'spack')), - ('user', os.path.expanduser('~/.spack'))] +class ConfigScope(object): + """This class represents a configuration scope. + + A scope is one directory containing named configuration files. + Each file is a config "section" (e.g., mirrors, compilers, etc). + """ + def __init__(self, name, path): + self.name = name # scope name. + self.path = path # path to directory containing configs. + self.sections = {} # sections read from config files. + + + def get_section_filename(self, section): + check_section(section) + return os.path.join(self.path, "%s.yaml" % section) + + + def get_section(self, section): + if not section in self.sections: + path = self.get_section_filename(section) + data = _read_config_file(path) + self.sections[section] = {} if data is None else data + return self.sections[section] + + + def write_section(self, section): + filename = self.get_section_filename(section) + data = self.get_section(section) + try: + mkdirp(self.path) + with open(filename, 'w') as f: + yaml.dump(data, stream=f, default_flow_style=False) + except (yaml.YAMLError, IOError) as e: + raise ConfigFileError("Error writing to config file: '%s'" % str(e)) + + + def clear(self): + """Empty cached config information.""" + self.sections = {} + + +"""List of config scopes by name. + Later scopes in the list will override earlier scopes. +""" +config_scopes = [ + ConfigScope('site', os.path.join(spack.etc_path, 'spack')), + ConfigScope('user', os.path.expanduser('~/.spack'))] + +"""List of valid scopes, for convenience.""" +valid_scopes = (s.name for s in config_scopes) + + +def check_scope(scope): + if scope is None: + return 'user' + elif scope not in valid_scopes: + raise ValueError("Invalid config scope: '%s'. Must be one of %s." + % (scope, valid_scopes)) + return scope + + +def get_scope(scope): + scope = check_scope(scope) + return next(s for s in config_scopes if s.name == scope) -_compiler_by_arch = {} -@memoized def _read_config_file(filename): - """Read a YAML configuration file""" - + """Read a YAML configuration file.""" # Ignore nonexisting files. if not os.path.exists(filename): return None elif not os.path.isfile(filename): - tty.die("Invlaid configuration. %s exists but is not a file." % filename) + raise ConfigFileError( + "Invlaid configuration. %s exists but is not a file." % filename) elif not os.access(filename, os.R_OK): - tty.die("Configuration file %s is not readable." % filename) + raise ConfigFileError("Config file is not readable: %s." % filename) try: with open(filename) as f: return yaml.load(f) except MarkedYAMLError, e: - tty.die("Error parsing yaml%s: %s" % (str(e.context_mark), e.problem)) + raise ConfigFileError( + "Error parsing yaml%s: %s" % (str(e.context_mark), e.problem)) except IOError, e: - tty.die("Error reading configuration file %s: %s" % (filename, str(e))) + raise ConfigFileError( + "Error reading configuration file %s: %s" % (filename, str(e))) def clear_config_caches(): """Clears the caches for configuration files, which will cause them to be re-read upon the next request""" - for key,s in _config_sections.iteritems(): - s.files_read_from = [] - s.result_dict = {} - - _read_config_file.clear() - spack.config._compiler_by_arch = {} - spack.compilers._cached_default_compiler = None + for scope in config_scopes: + scope.clear() def _merge_yaml(dest, source): """Merges source into dest; entries in source take precedence over dest. + This routine may modify dest and should be assigned to dest, in + case dest was None to begin with, e.g.: + + dest = _merge_yaml(dest, source) + Config file authors can optionally end any attribute in a dict with `::` instead of `:`, and the key will override that of the parent instead of merging. + """ def they_are(t): return isinstance(dest, t) and isinstance(source, t) @@ -212,61 +293,31 @@ def substitute_spack_prefix(path): return path.replace('$spack', spack.prefix) -def get_config(category): - """Get the confguration tree for a category. +def get_config(section): + """Get configuration settings for a section. - Strips off the top-level category entry from the dict + Strips off the top-level section name from the YAML dict. """ - category = _config_sections[category] - if category.result_dict: - return category.result_dict + check_section(section) + merged_section = {} - category.result_dict = {} - for scope, scope_path in config_scopes: - path = os.path.join(scope_path, category.filename) - result = _read_config_file(path) - if not result: + for scope in config_scopes: + # read potentially cached data from the scope. + data = scope.get_section(section) + if not data or not section in data: continue - if category.strip: - if not category.name in result: - continue - result = result[category.name] + # extract data under the section name header + data = data[section] - # ignore empty sections for easy commenting of single-line configs. - if result is None: - continue + # ignore empty sections for easy commenting of single-line configs. + if not data: + continue - category.files_read_from.insert(0, path) - if category.merge: - category.result_dict = _merge_yaml(category.result_dict, result) - else: - category.result_dict = result + # merge config data from scopes. + merged_section = _merge_yaml(merged_section, data) - return category.result_dict - - -def get_compilers_config(arch=None): - """Get the compiler configuration from config files for the given - architecture. Strips off the architecture component of the - configuration""" - global _compiler_by_arch - if not arch: - arch = spack.architecture.sys_type() - if arch in _compiler_by_arch: - return _compiler_by_arch[arch] - - cc_config = get_config('compilers') - if arch in cc_config and 'all' in cc_config: - arch_compiler = dict(cc_config[arch]) - _compiler_by_arch[arch] = _merge_yaml(arch_compiler, cc_config['all']) - elif arch in cc_config: - _compiler_by_arch[arch] = cc_config[arch] - elif 'all' in cc_config: - _compiler_by_arch[arch] = cc_config['all'] - else: - _compiler_by_arch[arch] = {} - return _compiler_by_arch[arch] + return merged_section def get_repos_config(): @@ -284,119 +335,71 @@ def expand_repo_path(path): return [expand_repo_path(repo) for repo in repo_list] -def get_mirror_config(): - """Get the mirror configuration from config files""" - return get_config('mirrors') +def get_config_filename(scope, section): + """For some scope and section, get the name of the configuration file""" + scope = get_scope(scope) + return scope.get_section_filename(section) -def get_config_scope_dirname(scope): - """For a scope return the config directory""" - for s,p in config_scopes: - if s == scope: - return p - tty.die("Unknown scope %s. Valid options are %s" % - (scope, ", ".join([s for s,p in config_scopes]))) +def update_config(section, update_data, scope=None): + """Update the configuration file for a particular scope. + + Merges contents of update_data into the scope's data for the + specified section, then writes out the config file. + + update_data shoudl contain only the section's data, with the + top-level name stripped off. This can be a list, dict, or any + other yaml-ish structure. + + """ + # read in the config to ensure we've got current data + get_config(section) + + check_section(section) # validate section name + scope = get_scope(scope) # get ConfigScope object from string. + + # read only the requested section's data. + data = scope.get_section(section) + data = _merge_yaml(data, { section : update_data }) + scope.write_section(section) -def get_config_scope_filename(scope, category_name): - """For some scope and category, get the name of the configuration file""" - if not category_name in _config_sections: - tty.die("Unknown config category %s. Valid options are: %s" % - (category_name, ", ".join([s for s in _config_sections]))) - return os.path.join(get_config_scope_dirname(scope), _config_sections[category_name].filename) +def remove_from_config(section, key_to_rm, scope=None): + """Remove a configuration key and write updated configuration to disk. + Return True if something was removed, False otherwise. -def add_to_config(category_name, addition_dict, scope=None): - """Merge a new dict into a configuration tree and write the new - configuration to disk""" - get_config(category_name) - category = _config_sections[category_name] + """ + # ensure configs are current by reading in. + get_config(section) - # If scope is specified, use it. Otherwise use the last config scope that - # we successfully parsed data from. - file = None - path = None - if not scope and not category.files_read_from: - scope = 'user' + # check args and get the objects we need. + scope = get_scope(scope) + data = scope.get_section(section) + filename = scope.get_section_filename(section) - if scope: - try: - dir = get_config_scope_dirname(scope) - if not os.path.exists(dir): - mkdirp(dir) - path = os.path.join(dir, category.filename) - file = open(path, 'w') - except IOError, e: - pass - else: - for p in category.files_read_from: - try: - file = open(p, 'w') - except IOError, e: - pass - if file: - path = p - break; + # do some checks + if not data: + return False - if not file: - tty.die('Unable to write to config file %s' % path) + if not section in data: + raise ConfigFileError("Invalid configuration file: '%s'" % filename) - # Merge the new information into the existing file info, then write to disk - new_dict = _read_config_file(path) + if key_to_rm not in section[section]: + return False - if new_dict and category_name in new_dict: - new_dict = new_dict[category_name] - - new_dict = _merge_yaml(new_dict, addition_dict) - new_dict = { category_name : new_dict } - - # Install new dict as memoized value, and dump to disk - _read_config_file.cache[path] = new_dict - yaml.dump(new_dict, stream=file, default_flow_style=False) - file.close() - - # Merge the new information into the cached results - category.result_dict = _merge_yaml(category.result_dict, addition_dict) - - -def add_to_mirror_config(addition_dict, scope=None): - """Add mirrors to the configuration files""" - add_to_config('mirrors', addition_dict, scope) - - -def add_to_compiler_config(addition_dict, scope=None, arch=None): - """Add compilerss to the configuration files""" - if not arch: - arch = spack.architecture.sys_type() - add_to_config('compilers', { arch : addition_dict }, scope) - clear_config_caches() - - -def remove_from_config(category_name, key_to_rm, scope=None): - """Remove a configuration key and write a new configuration to disk""" - get_config(category_name) - scopes_to_rm_from = [scope] if scope else [s for s,p in config_scopes] - category = _config_sections[category_name] - - rmd_something = False - for s in scopes_to_rm_from: - path = get_config_scope_filename(scope, category_name) - result = _read_config_file(path) - if not result: - continue - if not key_to_rm in result[category_name]: - continue - with open(path, 'w') as f: - result[category_name].pop(key_to_rm, None) - yaml.dump(result, stream=f, default_flow_style=False) - category.result_dict.pop(key_to_rm, None) - rmd_something = True - return rmd_something + # remove the key from the section's configuration + del data[section][key_to_rm] + scope.write_section(section) """Print a configuration to stdout""" -def print_category(category_name): - if not category_name in _config_sections: - tty.die("Unknown config category %s. Valid options are: %s" % - (category_name, ", ".join([s for s in _config_sections]))) - yaml.dump(get_config(category_name), stream=sys.stdout, default_flow_style=False) +def print_section(section): + try: + yaml.dump(get_config(section), stream=sys.stdout, default_flow_style=False) + except (yaml.YAMLError, IOError) as e: + raise ConfigError("Error reading configuration: %s" % section) + + +class ConfigError(SpackError): pass +class ConfigFileError(ConfigError): pass diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 303df6df381..2f1b6e29ea3 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -531,6 +531,7 @@ def concrete(self): and self.architecture and self.compiler and self.compiler.concrete and self.dependencies.concrete) + return self._concrete diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 78930ecb5b2..da85bd6f213 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -242,7 +242,8 @@ def fetch(self): # TODO: move mirror logic out of here and clean it up! if self.mirror_path: - urls = ["%s/%s" % (m, self.mirror_path) for m in _get_mirrors()] + urls = ["%s/%s" % (u, self.mirror_path) + for name, u in spack.config.get_config('mirrors')] digest = None if isinstance(self.fetcher, fs.URLFetchStrategy): @@ -345,7 +346,7 @@ def destroy(self): def _get_mirrors(): """Get mirrors from spack configuration.""" - config = spack.config.get_mirror_config() + config = spack.config.get_config('mirrors') return [val for name, val in config.iteritems()] diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index b1195dfe4e9..fe6cec82fed 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -27,6 +27,7 @@ import os from tempfile import mkdtemp import spack +import spack.config from spack.test.mock_packages_test import * # Some sample compiler config data @@ -72,9 +73,9 @@ def setUp(self): super(ConfigTest, self).setUp() self.tmp_dir = mkdtemp('.tmp', 'spack-config-test-') spack.config.config_scopes = [ - ('test_low_priority', os.path.join(self.tmp_dir, 'low')), - ('test_high_priority', os.path.join(self.tmp_dir, 'high'))] - + spack.config.ConfigScope('test_low_priority', os.path.join(self.tmp_dir, 'low')), + spack.config.ConfigScope('test_high_priority', os.path.join(self.tmp_dir, 'high'))] + spack.config.valid_scopes = ('test_low_priority', 'test_high_priority') def tearDown(self): super(ConfigTest, self).tearDown() @@ -83,17 +84,19 @@ def tearDown(self): def check_config(self, comps, *compiler_names): """Check that named compilers in comps match Spack's config.""" - config = spack.config.get_compilers_config() + config = spack.config.get_config('compilers') compiler_list = ['cc', 'cxx', 'f77', 'f90'] for key in compiler_names: for c in compiler_list: - self.assertEqual(comps[key][c], config[key][c]) + expected = comps[key][c] + actual = config[key][c] + self.assertEqual(expected, actual) def test_write_key_in_memory(self): # Write b_comps "on top of" a_comps. - spack.config.add_to_compiler_config(a_comps, 'test_low_priority') - spack.config.add_to_compiler_config(b_comps, 'test_high_priority') + spack.config.update_config('compilers', a_comps, 'test_low_priority') + spack.config.update_config('compilers', b_comps, 'test_high_priority') # Make sure the config looks how we expect. self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') @@ -102,8 +105,8 @@ def test_write_key_in_memory(self): def test_write_key_to_disk(self): # Write b_comps "on top of" a_comps. - spack.config.add_to_compiler_config(a_comps, 'test_low_priority') - spack.config.add_to_compiler_config(b_comps, 'test_high_priority') + spack.config.update_config('compilers', a_comps, 'test_low_priority') + spack.config.update_config('compilers', b_comps, 'test_high_priority') # Clear caches so we're forced to read from disk. spack.config.clear_config_caches() diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 5ce010ae8ff..e1f7961bed1 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -79,7 +79,8 @@ class DatabaseTest(MockPackagesTest): def _mock_install(self, spec): s = Spec(spec) - pkg = spack.repo.get(s.concretized()) + s.concretize() + pkg = spack.repo.get(s) pkg.do_install(fake=True) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 2150b408760..6d92aacab93 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -41,9 +41,10 @@ def initmock(self): spack.config.clear_config_caches() self.real_scopes = spack.config.config_scopes + self.real_valid_scopes = spack.config.valid_scopes spack.config.config_scopes = [ - ('site', spack.mock_site_config), - ('user', spack.mock_user_config)] + spack.config.ConfigScope('site', spack.mock_site_config), + spack.config.ConfigScope('user', spack.mock_user_config)] # Store changes to the package's dependencies so we can # restore later. @@ -71,6 +72,7 @@ def cleanmock(self): """Restore the real packages path after any test.""" spack.repo.swap(self.db) spack.config.config_scopes = self.real_scopes + spack.config.valid_scopes = self.real_valid_scopes spack.config.clear_config_caches() # Restore dependency changes that happened during the test From bef52570aee9d9e287eb46b7461fc2fbcda8a033 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 25 Dec 2015 18:42:06 -0800 Subject: [PATCH 219/910] Default to scope with highest precedence instead of user scope, - Generalizes config scopes a bit more: nothing assumes there is a 'user' scope (this would break testing sometimes). --- lib/spack/spack/compilers/__init__.py | 2 +- lib/spack/spack/config.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 321452fddb7..a1b6d978dfd 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -84,7 +84,7 @@ def get_compiler_config(arch=None): compilers = find_compilers(*get_path('PATH')) for compiler in compilers: config[arch].update(_to_dict(compiler)) - spack.config.update_config('compilers', config, 'user') + spack.config.update_config('compilers', config) # Merge 'all' compilers with arch-specific ones. merged_config = config.get('all', {}) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index b401f59d7f5..aa6afd183ed 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -193,8 +193,16 @@ def clear(self): def check_scope(scope): + """Ensure that scope is valid, and return a valid scope if it is None. + + This should be used by routines in ``config.py`` to validate + scope name arguments, and to determine a default scope where no + scope is specified. + + """ if scope is None: - return 'user' + # default to the scope with highest precedence. + return config_scopes[-1] elif scope not in valid_scopes: raise ValueError("Invalid config scope: '%s'. Must be one of %s." % (scope, valid_scopes)) From 4e8e298eb3242cb25abe0cca315a74a100514dbf Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 27 Dec 2015 17:40:31 -0800 Subject: [PATCH 220/910] Uncomment disabled tests. --- lib/spack/spack/test/package_sanity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index 0b075d135d0..ee09040d0d8 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -44,7 +44,7 @@ def test_get_all_packages(self): self.check_db() - def ztest_get_all_mock_packages(self): + def test_get_all_mock_packages(self): """Get the mock packages once each too.""" db = RepoPath(spack.mock_packages_path) spack.repo.swap(db) @@ -52,7 +52,7 @@ def ztest_get_all_mock_packages(self): spack.repo.swap(db) - def ztest_url_versions(self): + def test_url_versions(self): """Check URLs for regular packages, if they are explicitly defined.""" for pkg in spack.repo.all_packages(): for v, vattrs in pkg.versions.items(): From 39e360f93a374314c28716ded3b1533d66cd62be Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 27 Dec 2015 17:51:11 -0800 Subject: [PATCH 221/910] Add custom YAML loader & dumper to track lines & maintain dict order. - Configs are now parsed with `spack.util.spack_yaml.load/dump` - Parser annotates returned data with `_start_mark` and `_end_mark` properties, so that we can recover what lines/files they came from. - Parser uses `OrderedDict` instead of `dict`. This will help maintain some sanity when round-tripping config files. --- lib/spack/spack/config.py | 79 ++++---- lib/spack/spack/test/__init__.py | 3 +- lib/spack/spack/test/config.py | 8 +- lib/spack/spack/test/mock_packages_test.py | 12 +- lib/spack/spack/test/yaml.py | 93 ++++++++++ lib/spack/spack/util/spack_yaml.py | 201 +++++++++++++++++++++ 6 files changed, 352 insertions(+), 44 deletions(-) create mode 100644 lib/spack/spack/test/yaml.py create mode 100644 lib/spack/spack/util/spack_yaml.py diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index aa6afd183ed..9e3b44085f4 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. @@ -120,8 +120,10 @@ import os import sys import copy + import yaml from yaml.error import MarkedYAMLError +from ordereddict_backport import OrderedDict import llnl.util.tty as tty from llnl.util.filesystem import mkdirp @@ -129,11 +131,20 @@ import spack from spack.error import SpackError -"""List of valid config sections.""" -valid_sections = ('compilers', 'mirrors', 'repos') +# Hacked yaml for configuration files preserves line numbers. +import spack.util.spack_yaml as syaml -def check_section(section): +"""Dict from section names -> function to check section YAML format.""" +valid_sections = ['compilers', 'mirrors', 'repos'] + +"""OrderedDict of config scopes keyed by name. + Later scopes will override earlier scopes. +""" +config_scopes = OrderedDict() + + +def validate_section(section): """Raise a ValueError if the section is not a valid section.""" if section not in valid_sections: raise ValueError("Invalid config section: '%s'. Options are %s." @@ -146,14 +157,19 @@ class ConfigScope(object): A scope is one directory containing named configuration files. Each file is a config "section" (e.g., mirrors, compilers, etc). """ + def __init__(self, name, path): self.name = name # scope name. self.path = path # path to directory containing configs. self.sections = {} # sections read from config files. + # Register in a dict of all ConfigScopes + # TODO: make this cleaner. Mocking up for testing is brittle. + global config_scopes + config_scopes[name] = self def get_section_filename(self, section): - check_section(section) + validate_section(section) return os.path.join(self.path, "%s.yaml" % section) @@ -161,7 +177,10 @@ def get_section(self, section): if not section in self.sections: path = self.get_section_filename(section) data = _read_config_file(path) - self.sections[section] = {} if data is None else data + if data is None: + self.sections[section] = {} + else: + self.sections[section] = data return self.sections[section] @@ -171,7 +190,7 @@ def write_section(self, section): try: mkdirp(self.path) with open(filename, 'w') as f: - yaml.dump(data, stream=f, default_flow_style=False) + syaml.dump(data, stream=f, default_flow_style=False) except (yaml.YAMLError, IOError) as e: raise ConfigFileError("Error writing to config file: '%s'" % str(e)) @@ -181,18 +200,11 @@ def clear(self): self.sections = {} -"""List of config scopes by name. - Later scopes in the list will override earlier scopes. -""" -config_scopes = [ - ConfigScope('site', os.path.join(spack.etc_path, 'spack')), - ConfigScope('user', os.path.expanduser('~/.spack'))] - -"""List of valid scopes, for convenience.""" -valid_scopes = (s.name for s in config_scopes) +ConfigScope('site', os.path.join(spack.etc_path, 'spack')), +ConfigScope('user', os.path.expanduser('~/.spack')) -def check_scope(scope): +def validate_scope(scope): """Ensure that scope is valid, and return a valid scope if it is None. This should be used by routines in ``config.py`` to validate @@ -202,16 +214,14 @@ def check_scope(scope): """ if scope is None: # default to the scope with highest precedence. - return config_scopes[-1] - elif scope not in valid_scopes: + return config_scopes.values()[-1] + + elif scope in config_scopes: + return config_scopes[scope] + + else: raise ValueError("Invalid config scope: '%s'. Must be one of %s." - % (scope, valid_scopes)) - return scope - - -def get_scope(scope): - scope = check_scope(scope) - return next(s for s in config_scopes if s.name == scope) + % (scope, config_scopes.keys())) def _read_config_file(filename): @@ -229,7 +239,7 @@ def _read_config_file(filename): try: with open(filename) as f: - return yaml.load(f) + return syaml.load(f) except MarkedYAMLError, e: raise ConfigFileError( @@ -243,7 +253,7 @@ def _read_config_file(filename): def clear_config_caches(): """Clears the caches for configuration files, which will cause them to be re-read upon the next request""" - for scope in config_scopes: + for scope in config_scopes.values(): scope.clear() @@ -306,10 +316,10 @@ def get_config(section): Strips off the top-level section name from the YAML dict. """ - check_section(section) + validate_section(section) merged_section = {} - for scope in config_scopes: + for scope in config_scopes.values(): # read potentially cached data from the scope. data = scope.get_section(section) if not data or not section in data: @@ -345,7 +355,7 @@ def expand_repo_path(path): def get_config_filename(scope, section): """For some scope and section, get the name of the configuration file""" - scope = get_scope(scope) + scope = validate_scope(scope) return scope.get_section_filename(section) @@ -363,8 +373,8 @@ def update_config(section, update_data, scope=None): # read in the config to ensure we've got current data get_config(section) - check_section(section) # validate section name - scope = get_scope(scope) # get ConfigScope object from string. + validate_section(section) # validate section name + scope = validate_scope(scope) # get ConfigScope object from string. # read only the requested section's data. data = scope.get_section(section) @@ -382,7 +392,7 @@ def remove_from_config(section, key_to_rm, scope=None): get_config(section) # check args and get the objects we need. - scope = get_scope(scope) + scope = validate_scope(scope) data = scope.get_section(section) filename = scope.get_section_filename(section) @@ -411,3 +421,4 @@ def print_section(section): class ConfigError(SpackError): pass class ConfigFileError(ConfigError): pass +class ConfigFormatError(ConfigError): pass diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 081e6c7b06c..9609fd2f36e 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -60,7 +60,8 @@ 'unit_install', 'lock', 'database', - 'namespace_trie'] + 'namespace_trie', + 'yaml'] def list_tests(): diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index f56287aa980..5f99dcb9032 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -26,6 +26,7 @@ import shutil import os from tempfile import mkdtemp +from ordereddict_backport import OrderedDict import spack import spack.config from spack.test.mock_packages_test import * @@ -72,10 +73,9 @@ class ConfigTest(MockPackagesTest): def setUp(self): super(ConfigTest, self).setUp() self.tmp_dir = mkdtemp('.tmp', 'spack-config-test-') - spack.config.config_scopes = [ - spack.config.ConfigScope('test_low_priority', os.path.join(self.tmp_dir, 'low')), - spack.config.ConfigScope('test_high_priority', os.path.join(self.tmp_dir, 'high'))] - spack.config.valid_scopes = ('test_low_priority', 'test_high_priority') + spack.config.config_scopes = OrderedDict() + spack.config.ConfigScope('test_low_priority', os.path.join(self.tmp_dir, 'low')) + spack.config.ConfigScope('test_high_priority', os.path.join(self.tmp_dir, 'high')) def tearDown(self): super(ConfigTest, self).tearDown() diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index d000271960f..320c2921b06 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -24,6 +24,7 @@ ############################################################################## import sys import unittest +from ordereddict_backport import OrderedDict import spack import spack.config @@ -41,10 +42,12 @@ def initmock(self): spack.config.clear_config_caches() self.real_scopes = spack.config.config_scopes - self.real_valid_scopes = spack.config.valid_scopes - spack.config.config_scopes = [ - spack.config.ConfigScope('site', spack.mock_site_config), - spack.config.ConfigScope('user', spack.mock_user_config)] + + # TODO: Mocking this up is kind of brittle b/c ConfigScope + # TODO: constructor modifies config_scopes. Make it cleaner. + spack.config.config_scopes = OrderedDict() + spack.config.ConfigScope('site', spack.mock_site_config) + spack.config.ConfigScope('user', spack.mock_user_config) # Store changes to the package's dependencies so we can # restore later. @@ -72,7 +75,6 @@ def cleanmock(self): """Restore the real packages path after any test.""" spack.repo.swap(self.db) spack.config.config_scopes = self.real_scopes - spack.config.valid_scopes = self.real_valid_scopes spack.config.clear_config_caches() # Restore dependency changes that happened during the test diff --git a/lib/spack/spack/test/yaml.py b/lib/spack/spack/test/yaml.py new file mode 100644 index 00000000000..5a357b8e696 --- /dev/null +++ b/lib/spack/spack/test/yaml.py @@ -0,0 +1,93 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +""" +Test Spack's custom YAML format. +""" +import unittest +import spack.util.spack_yaml as syaml + +test_file = """\ +config_file: + x86_64: + foo: /path/to/foo + bar: /path/to/bar + baz: /path/to/baz + some_list: + - item 1 + - item 2 + - item 3 + another_list: + [ 1, 2, 3 ] + some_key: some_string +""" + +test_data = { + 'config_file' : syaml.syaml_dict([ + ('x86_64', syaml.syaml_dict([ + ('foo', '/path/to/foo'), + ('bar', '/path/to/bar'), + ('baz', '/path/to/baz' )])), + ('some_list', [ 'item 1', 'item 2', 'item 3' ]), + ('another_list', [ 1, 2, 3 ]), + ('some_key', 'some_string') + ])} + +class YamlTest(unittest.TestCase): + + def setUp(self): + self.data = syaml.load(test_file) + + + def test_parse(self): + self.assertEqual(test_data, self.data) + + + def test_dict_order(self): + self.assertEqual( + ['x86_64', 'some_list', 'another_list', 'some_key'], + self.data['config_file'].keys()) + + self.assertEqual( + ['foo', 'bar', 'baz'], + self.data['config_file']['x86_64'].keys()) + + + def test_line_numbers(self): + def check(obj, start_line, end_line): + self.assertEqual(obj._start_mark.line, start_line) + self.assertEqual(obj._end_mark.line, end_line) + + check(self.data, 0, 12) + check(self.data['config_file'], 1, 12) + check(self.data['config_file']['x86_64'], 2, 5) + check(self.data['config_file']['x86_64']['foo'], 2, 2) + check(self.data['config_file']['x86_64']['bar'], 3, 3) + check(self.data['config_file']['x86_64']['baz'], 4, 4) + check(self.data['config_file']['some_list'], 6, 9) + check(self.data['config_file']['some_list'][0], 6, 6) + check(self.data['config_file']['some_list'][1], 7, 7) + check(self.data['config_file']['some_list'][2], 8, 8) + check(self.data['config_file']['another_list'], 10, 10) + check(self.data['config_file']['some_key'], 11, 11) diff --git a/lib/spack/spack/util/spack_yaml.py b/lib/spack/spack/util/spack_yaml.py new file mode 100644 index 00000000000..728e86b8ee3 --- /dev/null +++ b/lib/spack/spack/util/spack_yaml.py @@ -0,0 +1,201 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Enhanced YAML parsing for Spack. + +- ``load()`` preserves YAML Marks on returned objects -- this allows + us to access file and line information later. + +- ``Our load methods use ``OrderedDict`` class instead of YAML's + default unorderd dict. + +""" +import yaml +from yaml.nodes import * +from yaml.constructor import ConstructorError +from yaml.representer import SafeRepresenter +from ordereddict_backport import OrderedDict + +# Only export load and dump +__all__ = ['load', 'dump'] + +# Make new classes so we can add custom attributes. +# Also, use OrderedDict instead of just dict. +class syaml_dict(OrderedDict): + def __repr__(self): + mappings = ('%r: %r' % (k,v) for k,v in self.items()) + return '{%s}' % ', '.join(mappings) +class syaml_list(list): + __repr__ = list.__repr__ +class syaml_str(str): + __repr__ = str.__repr__ + +def mark(obj, node): + """Add start and end markers to an object.""" + obj._start_mark = node.start_mark + obj._end_mark = node.end_mark + + +class OrderedLineLoader(yaml.Loader): + """YAML loader that preserves order and line numbers. + + Mappings read in by this loader behave like an ordered dict. + Sequences, mappings, and strings also have new attributes, + ``_start_mark`` and ``_end_mark``, that preserve YAML line + information in the output data. + + """ + # + # Override construct_yaml_* so that they build our derived types, + # which allows us to add new attributes to them. + # + # The standard YAML constructors return empty instances and fill + # in with mappings later. We preserve this behavior. + # + def construct_yaml_str(self, node): + value = self.construct_scalar(node) + try: + value = value.encode('ascii') + except UnicodeEncodeError: + pass + value = syaml_str(value) + mark(value, node) + return value + + + def construct_yaml_seq(self, node): + data = syaml_list() + mark(data, node) + yield data + data.extend(self.construct_sequence(node)) + + + def construct_yaml_map(self, node): + data = syaml_dict() + mark(data, node) + yield data + value = self.construct_mapping(node) + data.update(value) + + # + # Override the ``construct_*`` routines. These fill in empty + # objects after yielded by the above ``construct_yaml_*`` methods. + # + def construct_sequence(self, node, deep=False): + if not isinstance(node, SequenceNode): + raise ConstructorError(None, None, + "expected a sequence node, but found %s" % node.id, + node.start_mark) + value = syaml_list(self.construct_object(child, deep=deep) + for child in node.value) + mark(value, node) + return value + + + def construct_mapping(self, node, deep=False): + """Store mappings as OrderedDicts instead of as regular python + dictionaries to preserve file ordering.""" + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + + mapping = syaml_dict() + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + try: + hash(key) + except TypeError, exc: + raise ConstructorError("while constructing a mapping", node.start_mark, + "found unacceptable key (%s)" % exc, key_node.start_mark) + value = self.construct_object(value_node, deep=deep) + if key in mapping: + raise ConstructorError("while constructing a mapping", node.start_mark, + "found already in-use key (%s)" % key, key_node.start_mark) + mapping[key] = value + + mark(mapping, node) + return mapping + +# register above new constructors +OrderedLineLoader.add_constructor(u'tag:yaml.org,2002:map', OrderedLineLoader.construct_yaml_map) +OrderedLineLoader.add_constructor(u'tag:yaml.org,2002:seq', OrderedLineLoader.construct_yaml_seq) +OrderedLineLoader.add_constructor(u'tag:yaml.org,2002:str', OrderedLineLoader.construct_yaml_str) + + + +class OrderedLineDumper(yaml.Dumper): + """Dumper that preserves ordering and formats ``syaml_*`` objects. + + This dumper preserves insertion ordering ``syaml_dict`` objects + when they're written out. It also has some custom formatters + for ``syaml_*`` objects so that they are formatted like their + regular Python equivalents, instead of ugly YAML pyobjects. + + """ + def represent_mapping(self, tag, mapping, flow_style=None): + value = [] + node = MappingNode(tag, value, flow_style=flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + if hasattr(mapping, 'items'): + # if it's a syaml_dict, preserve OrderedDict order. + # Otherwise do the default thing. + sort = not isinstance(mapping, syaml_dict) + mapping = mapping.items() + if sort: + mapping.sort() + + for item_key, item_value in mapping: + node_key = self.represent_data(item_key) + node_value = self.represent_data(item_value) + if not (isinstance(node_key, ScalarNode) and not node_key.style): + best_style = False + if not (isinstance(node_value, ScalarNode) and not node_value.style): + best_style = False + value.append((node_key, node_value)) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node + +# Make our special objects look like normal YAML ones. +OrderedLineDumper.add_representer(syaml_dict, OrderedLineDumper.represent_dict) +OrderedLineDumper.add_representer(syaml_list, OrderedLineDumper.represent_list) +OrderedLineDumper.add_representer(syaml_str, OrderedLineDumper.represent_str) + + +def load(*args, **kwargs): + """Load but modify the loader instance so that it will add __line__ + atrributes to the returned object.""" + kwargs['Loader'] = OrderedLineLoader + return yaml.load(*args, **kwargs) + + +def dump(*args, **kwargs): + kwargs['Dumper'] = OrderedLineDumper + return yaml.dump(*args, **kwargs) From ff0d871612c05d803fdabb8a5b870b7af961cdc2 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 27 Dec 2015 21:13:18 -0800 Subject: [PATCH 222/910] Remove mock_configs; tests no longer modify spack home directory. --- lib/spack/spack/__init__.py | 4 --- lib/spack/spack/test/mock_packages_test.py | 32 +++++++++++++++++-- .../site_spackconfig/compilers.yaml | 12 ------- 3 files changed, 30 insertions(+), 18 deletions(-) delete mode 100644 var/spack/mock_configs/site_spackconfig/compilers.yaml diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index de1a98d0928..973ba64b96a 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -78,10 +78,6 @@ packages_path = join_path(repos_path, "builtin") mock_packages_path = join_path(repos_path, "builtin.mock") -mock_config_path = join_path(var_path, "mock_configs") -mock_site_config = join_path(mock_config_path, "site_spackconfig") -mock_user_config = join_path(mock_config_path, "user_spackconfig") - # # This controls how spack lays out install prefixes and # stage directories. diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 320c2921b06..7642edcf4b7 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -23,14 +23,32 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys +import os import unittest +import tempfile from ordereddict_backport import OrderedDict +from llnl.util.filesystem import mkdirp + import spack import spack.config from spack.repository import RepoPath from spack.spec import Spec +mock_compiler_config = """\ +compilers: + all: + clang@3.3: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + gcc@4.5.0: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran +""" class MockPackagesTest(unittest.TestCase): def initmock(self): @@ -43,11 +61,21 @@ def initmock(self): spack.config.clear_config_caches() self.real_scopes = spack.config.config_scopes + # Mock up temporary configuration directories + self.temp_config = tempfile.mkdtemp() + self.mock_site_config = os.path.join(self.temp_config, 'site') + self.mock_user_config = os.path.join(self.temp_config, 'user') + mkdirp(self.mock_site_config) + mkdirp(self.mock_user_config) + comp_yaml = os.path.join(self.mock_site_config, 'compilers.yaml') + with open(comp_yaml, 'w') as f: + f.write(mock_compiler_config) + # TODO: Mocking this up is kind of brittle b/c ConfigScope # TODO: constructor modifies config_scopes. Make it cleaner. spack.config.config_scopes = OrderedDict() - spack.config.ConfigScope('site', spack.mock_site_config) - spack.config.ConfigScope('user', spack.mock_user_config) + spack.config.ConfigScope('site', self.mock_site_config) + spack.config.ConfigScope('user', self.mock_user_config) # Store changes to the package's dependencies so we can # restore later. diff --git a/var/spack/mock_configs/site_spackconfig/compilers.yaml b/var/spack/mock_configs/site_spackconfig/compilers.yaml deleted file mode 100644 index 0a2dc893e29..00000000000 --- a/var/spack/mock_configs/site_spackconfig/compilers.yaml +++ /dev/null @@ -1,12 +0,0 @@ -compilers: - all: - clang@3.3: - cc: /path/to/clang - cxx: /path/to/clang++ - f77: None - fc: None - gcc@4.5.0: - cc: /path/to/gcc - cxx: /path/to/g++ - f77: /path/to/gfortran - fc: /path/to/gfortran From 1f8ba53ca7767ca452f553f71d49eaf90fa19db8 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 28 Dec 2015 00:33:17 -0800 Subject: [PATCH 223/910] Rework compiler configuration and simplify config.py logic. - `spack compiler` subcommands now take an optional --scope argument. - no more `remove_from_config` in `config.py` -- `update` just overwrites b/c it's easier to just call `get_config`, modify YAML structures directly, and then call `update`. - Implemented `spack compiler remove`. --- lib/spack/spack/cmd/compiler.py | 49 ++++++++++---- lib/spack/spack/cmd/compilers.py | 5 ++ lib/spack/spack/compilers/__init__.py | 94 ++++++++++++++++++++------- lib/spack/spack/config.py | 51 ++++----------- 4 files changed, 125 insertions(+), 74 deletions(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index af1a22c9dd0..a3860abf769 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import sys import argparse import llnl.util.tty as tty @@ -41,17 +42,26 @@ def setup_parser(subparser): sp = subparser.add_subparsers( metavar='SUBCOMMAND', dest='compiler_command') - update_parser = sp.add_parser( - 'add', help='Add compilers to the Spack configuration.') - update_parser.add_argument('add_paths', nargs=argparse.REMAINDER) + add_parser = sp.add_parser('add', help='Add compilers to the Spack configuration.') + add_parser.add_argument('add_paths', nargs=argparse.REMAINDER) + add_parser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to modify.") - remove_parser = sp.add_parser('remove', help='remove compiler') - remove_parser.add_argument('path') + remove_parser = sp.add_parser('remove', help='Remove compiler by spec.') + remove_parser.add_argument( + '-a', '--all', action='store_true', help='Remove ALL compilers that match spec.') + remove_parser.add_argument('compiler_spec') + remove_parser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to modify.") - list_parser = sp.add_parser('list', help='list available compilers') + list_parser = sp.add_parser('list', help='list available compilers') + list_parser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to read from.") - info_parser = sp.add_parser('info', help='Show compiler paths.') + info_parser = sp.add_parser('info', help='Show compiler paths.') info_parser.add_argument('compiler_spec') + info_parser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to read from.") def compiler_add(args): @@ -62,13 +72,13 @@ def compiler_add(args): paths = get_path('PATH') compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers()] + if c.spec not in spack.compilers.all_compilers(scope=args.scope)] if compilers: - spack.compilers.add_compilers_to_config('user', compilers) + spack.compilers.add_compilers_to_config(compilers, scope=args.scope) n = len(compilers) s = 's' if n > 1 else '' - filename = spack.config.get_config_filename('user', 'compilers') + filename = spack.config.get_config_filename(args.scope, 'compilers') tty.msg("Added %d new compiler%s to %s" % (n, s, filename)) colify(reversed(sorted(c.spec for c in compilers)), indent=4) else: @@ -76,13 +86,26 @@ def compiler_add(args): def compiler_remove(args): - pass + cspec = CompilerSpec(args.compiler_spec) + compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) + + if not compilers: + tty.die("No compilers match spec %s." % cspec) + elif not args.all and len(compilers) > 1: + tty.error("Multiple compilers match spec %s. Choose one:" % cspec) + colify(reversed(sorted([c.spec for c in compilers])), indent=4) + tty.msg("Or, you can use `spack compiler remove -a` to remove all of them.") + sys.exit(1) + + for compiler in compilers: + spack.compilers.remove_compiler_from_config(compiler.spec, scope=args.scope) + tty.msg("Removed compiler %s." % compiler.spec) def compiler_info(args): """Print info about all compilers matching a spec.""" cspec = CompilerSpec(args.compiler_spec) - compilers = spack.compilers.compilers_for_spec(cspec) + compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) if not compilers: tty.error("No compilers match spec %s." % cspec) @@ -97,7 +120,7 @@ def compiler_info(args): def compiler_list(args): tty.msg("Available compilers") - index = index_by(spack.compilers.all_compilers(), 'name') + index = index_by(spack.compilers.all_compilers(scope=args.scope), 'name') for i, (name, compilers) in enumerate(index.items()): if i >= 1: print diff --git a/lib/spack/spack/cmd/compilers.py b/lib/spack/spack/cmd/compilers.py index c485a910ebd..7e09016f2dc 100644 --- a/lib/spack/spack/cmd/compilers.py +++ b/lib/spack/spack/cmd/compilers.py @@ -26,9 +26,14 @@ from llnl.util.tty.colify import colify from llnl.util.lang import index_by +import spack from spack.cmd.compiler import compiler_list description = "List available compilers. Same as 'spack compiler list'." +def setup_parser(subparser): + subparser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to read/modify.") + def compilers(parser, args): compiler_list(args) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index a1b6d978dfd..67dfaa3ac9c 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -49,10 +49,10 @@ _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] def _auto_compiler_spec(function): - def converter(cspec_like): + def converter(cspec_like, *args, **kwargs): if not isinstance(cspec_like, spack.spec.CompilerSpec): cspec_like = spack.spec.CompilerSpec(cspec_like) - return function(cspec_like) + return function(cspec_like, *args, **kwargs) return converter @@ -65,39 +65,87 @@ def _to_dict(compiler): } -def get_compiler_config(arch=None): +def get_compiler_config(arch=None, scope=None): """Return the compiler configuration for the specified architecture. - - If the compiler configuration designates some compilers for - 'all' architectures, those are merged into the result, as well. - """ # If any configuration file has compilers, just stick with the # ones already configured. - config = spack.config.get_config('compilers') + config = spack.config.get_config('compilers', scope=scope) + my_arch = spack.architecture.sys_type() if arch is None: - arch = spack.architecture.sys_type() + arch = my_arch - if arch not in config: + if arch in config: + return config[arch] + + # Only for the current arch in *highest* scope: automatically try to + # find compilers if none are configured yet. + if arch == my_arch and scope == 'user': config[arch] = {} compilers = find_compilers(*get_path('PATH')) for compiler in compilers: config[arch].update(_to_dict(compiler)) - spack.config.update_config('compilers', config) + spack.config.update_config('compilers', config, scope=scope) + return config[arch] - # Merge 'all' compilers with arch-specific ones. - merged_config = config.get('all', {}) - merged_config = spack.config._merge_yaml(merged_config, config[arch]) - - return merged_config + return {} -def all_compilers(arch=None): +def add_compilers_to_config(compilers, arch=None, scope=None): + """Add compilers to the config for the specified architecture. + + Arguments: + - compilers: a list of Compiler objects. + - arch: arch to add compilers for. + - scope: configuration scope to modify. + """ + if arch is None: + arch = spack.architecture.sys_type() + + compiler_config = get_compiler_config(arch, scope) + for compiler in compilers: + compiler_config[str(compiler.spec)] = dict( + (c, getattr(compiler, c, "None")) + for c in _required_instance_vars) + + update = { arch : compiler_config } + spack.config.update_config('compilers', update, scope) + + +@_auto_compiler_spec +def remove_compiler_from_config(compiler_spec, arch=None, scope=None): + """Remove compilers from the config, by spec. + + Arguments: + - compiler_specs: a list of CompilerSpec objects. + - arch: arch to add compilers for. + - scope: configuration scope to modify. + """ + if arch is None: + arch = spack.architecture.sys_type() + + compiler_config = get_compiler_config(arch, scope) + del compiler_config[str(compiler_spec)] + update = { arch : compiler_config } + + spack.config.update_config('compilers', update, scope) + + +def all_compilers(arch=None, scope=None): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ - return [spack.spec.CompilerSpec(s) for s in get_compiler_config(arch)] + # Get compilers for this architecture. + arch_config = get_compiler_config(arch, scope) + + # Merge 'all' compilers with arch-specific ones. + # Arch-specific compilers have higher precedence. + merged_config = get_compiler_config('all', scope=scope) + merged_config = spack.config._merge_yaml(merged_config, arch_config) + + # Return compiler specs for the result. + return [spack.spec.CompilerSpec(s) for s in merged_config] _cached_default_compiler = None @@ -165,18 +213,18 @@ def supported(compiler_spec): @_auto_compiler_spec -def find(compiler_spec): +def find(compiler_spec, arch=None, scope=None): """Return specs of available compilers that match the supplied compiler spec. Return an list if nothing found.""" - return [c for c in all_compilers() if c.satisfies(compiler_spec)] + return [c for c in all_compilers(arch, scope) if c.satisfies(compiler_spec)] @_auto_compiler_spec -def compilers_for_spec(compiler_spec): +def compilers_for_spec(compiler_spec, arch=None, scope=None): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ - config = get_compiler_config() + config = get_compiler_config(arch, scope) def get_compiler(cspec): items = config[str(cspec)] @@ -195,7 +243,7 @@ def get_compiler(cspec): return cls(cspec, *compiler_paths) - matches = find(compiler_spec) + matches = find(compiler_spec, arch, scope) return [get_compiler(cspec) for cspec in matches] diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 9e3b44085f4..d266d2e23ff 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -311,7 +311,7 @@ def substitute_spack_prefix(path): return path.replace('$spack', spack.prefix) -def get_config(section): +def get_config(section, scope=None): """Get configuration settings for a section. Strips off the top-level section name from the YAML dict. @@ -319,7 +319,12 @@ def get_config(section): validate_section(section) merged_section = {} - for scope in config_scopes.values(): + if scope is None: + scopes = config_scopes.values() + else: + scopes = [validate_scope(scope)] + + for scope in scopes: # read potentially cached data from the scope. data = scope.get_section(section) if not data or not section in data: @@ -362,52 +367,22 @@ def get_config_filename(scope, section): def update_config(section, update_data, scope=None): """Update the configuration file for a particular scope. - Merges contents of update_data into the scope's data for the - specified section, then writes out the config file. + Overwrites contents of a section in a scope with update_data, + then writes out the config file. - update_data shoudl contain only the section's data, with the - top-level name stripped off. This can be a list, dict, or any + update_data should have the top-level section name stripped off + (it will be re-added). Data itself can be a list, dict, or any other yaml-ish structure. """ # read in the config to ensure we've got current data get_config(section) - validate_section(section) # validate section name + validate_section(section) # validate section name scope = validate_scope(scope) # get ConfigScope object from string. # read only the requested section's data. - data = scope.get_section(section) - data = _merge_yaml(data, { section : update_data }) - scope.write_section(section) - - -def remove_from_config(section, key_to_rm, scope=None): - """Remove a configuration key and write updated configuration to disk. - - Return True if something was removed, False otherwise. - - """ - # ensure configs are current by reading in. - get_config(section) - - # check args and get the objects we need. - scope = validate_scope(scope) - data = scope.get_section(section) - filename = scope.get_section_filename(section) - - # do some checks - if not data: - return False - - if not section in data: - raise ConfigFileError("Invalid configuration file: '%s'" % filename) - - if key_to_rm not in section[section]: - return False - - # remove the key from the section's configuration - del data[section][key_to_rm] + scope.sections[section] = { section : update_data } scope.write_section(section) From e8e6368cc8ab03be4b635731ed43b0fcae47712a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 28 Dec 2015 01:14:41 -0800 Subject: [PATCH 224/910] Rework mirror configuration. - All of these work: - `spack mirror add` - `spack mirror remove` - `spack mirror list` - `spack mirror` subcommands (except create) now have their own --scope argument. - Mirror config is now stored sanely as an ordered list. --- lib/spack/spack/cmd/mirror.py | 50 ++++++++++++++++++++++++++++------- lib/spack/spack/stage.py | 6 +++-- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index a2d2e80f5e6..946b50350ba 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -60,15 +60,22 @@ def setup_parser(subparser): '-o', '--one-version-per-spec', action='store_const', const=1, default=0, help="Only fetch one 'preferred' version per spec, not all known versions.") + add_parser = sp.add_parser('add', help=mirror_add.__doc__) add_parser.add_argument('name', help="Mnemonic name for mirror.") add_parser.add_argument( 'url', help="URL of mirror directory created by 'spack mirror create'.") + add_parser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to modify.") remove_parser = sp.add_parser('remove', help=mirror_remove.__doc__) remove_parser.add_argument('name') + remove_parser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to modify.") list_parser = sp.add_parser('list', help=mirror_list.__doc__) + list_parser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to read from.") def mirror_add(args): @@ -77,31 +84,54 @@ def mirror_add(args): if url.startswith('/'): url = 'file://' + url - mirror_dict = { args.name : url } - spack.config.update_config('mirrors', { args.name : url }, 'user') + mirrors = spack.config.get_config('mirrors', scope=args.scope) + if not mirrors: + mirrors = [] + + for m in mirrors: + for name, u in m.items(): + if name == args.name: + tty.die("Mirror with name %s already exists." % name) + if u == url: + tty.die("Mirror with url %s already exists." % url) + # should only be one item per mirror dict. + + mirrors.insert(0, { args.name : url }) + spack.config.update_config('mirrors', mirrors, scope=args.scope) def mirror_remove(args): """Remove a mirror by name.""" name = args.name - rmd_something = spack.config.remove_from_config('mirrors', name) - if not rmd_something: - tty.die("No such mirror: %s" % name) + mirrors = spack.config.get_config('mirrors', scope=args.scope) + if not mirrors: + mirrors = [] + + names = [n for m in mirrors for n,u in m.items()] + if not name in names: + tty.die("No mirror with name %s" % name) + + old_mirror = mirrors.pop(names.index(name)) + spack.config.update_config('mirrors', mirrors, scope=args.scope) + tty.msg("Removed mirror %s with url %s." % old_mirror.popitem()) def mirror_list(args): """Print out available mirrors to the console.""" - sec_names = spack.config.get_config('mirrors') - if not sec_names: + mirrors = spack.config.get_config('mirrors', scope=args.scope) + if not mirrors: tty.msg("No mirrors configured.") return - max_len = max(len(s) for s in sec_names) + names = [n for m in mirrors for n,u in m.items()] + max_len = max(len(n) for n in names) fmt = "%%-%ds%%s" % (max_len + 4) - for name, val in sec_names.iteritems(): - print fmt % (name, val) + for m in mirrors: + for name, url in m.items(): + print fmt % (name, url) + # should only be one item per mirror dict. def _read_specs_from_file(filename): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 543a3a6223c..a9631d4b620 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -26,6 +26,7 @@ import re import shutil import tempfile +from urlparse import urljoin import llnl.util.tty as tty from llnl.util.filesystem import * @@ -242,8 +243,9 @@ def fetch(self): # TODO: move mirror logic out of here and clean it up! if self.mirror_path: - urls = ["%s/%s" % (u, self.mirror_path) - for name, u in spack.config.get_config('mirrors')] + mirrors = spack.config.get_config('mirrors') + mirrors = [(n,u) for m in mirrors for n,u in m.items()] + urls = [urljoin(u, self.mirror_path) for name, u in mirrors] digest = None if isinstance(self.fetcher, fs.URLFetchStrategy): From d7607973fa37ec2a5325111336be44f02cb95860 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Tue, 29 Dec 2015 09:04:54 -0800 Subject: [PATCH 225/910] fix for array handling in ancient bash fixes #284 --- share/spack/setup-env.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 47202f6087a..4faae0d2c9d 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -55,13 +55,12 @@ # avoids the need to come up with a user-friendly naming scheme for # spack dotfiles. ######################################################################## +arrtest[0]='test' || (echo 'Failure: arrays not supported in this version of bash.' && exit 2) + function spack { # save raw arguments into an array before butchering them - args=() - for a in "$@"; do - # yup, this is awful, blame bash2 compat - args=("${args[@]}" "$a") - done + declare -a args=( "$@" ) + # accumulate initial flags for main spack command _sp_flags="" while [[ "$1" =~ ^- ]]; do From 496e4856916cb881da916539635e9bfa3e1406af Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 10:13:46 -0800 Subject: [PATCH 226/910] removing check as it can cause failures in zsh --- share/spack/setup-env.sh | 1 - var/spack/packages/binutils/package.py | 5 + var/spack/packages/clang/package.py | 48 +++++--- var/spack/packages/gcc/package.py | 32 ++++- var/spack/packages/llvm/package.py | 154 ++++++++++++++++++++++--- 5 files changed, 204 insertions(+), 36 deletions(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 4faae0d2c9d..6503728d8f2 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -55,7 +55,6 @@ # avoids the need to come up with a user-friendly naming scheme for # spack dotfiles. ######################################################################## -arrtest[0]='test' || (echo 'Failure: arrays not supported in this version of bash.' && exit 2) function spack { # save raw arguments into an array before butchering them diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py index 123f4598f69..a79fc5fb699 100644 --- a/var/spack/packages/binutils/package.py +++ b/var/spack/packages/binutils/package.py @@ -11,6 +11,7 @@ class Binutils(Package): # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell variant('krellpatch', default=False, description="build with openspeedshop based patch.") + variant('gold', default=True, description="build the gold linker") patch('binutilskrell-2.24.patch', when='@2.24+krellpatch') variant('libiberty', default=False, description='Also install libiberty.') @@ -26,6 +27,10 @@ def install(self, spec, prefix): '--enable-targets=all', '--with-sysroot=/'] + if '+gold' in spec: + configure_args.append('--enable-gold') + configure_args.append('--enable-plugins') + if '+libiberty' in spec: configure_args.append('--enable-install-libiberty') diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py index e46e08d5f11..99e805d15d0 100644 --- a/var/spack/packages/clang/package.py +++ b/var/spack/packages/clang/package.py @@ -36,20 +36,42 @@ class Clang(Package): homepage = 'http://clang.llvm.org' url = 'http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz' - depends_on('llvm@3.7.0', when='@3.7.0') - depends_on('llvm@3.6.2', when='@3.6.2') - depends_on('llvm@3.5.1', when='@3.5.1') + clang_url = 'http://llvm.org/releases/%(version)s/cfe-%(version)s.src.tar.xz' - version('3.7.0', '8f9d27335e7331cf0a4711e952f21f01', url='http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz') - version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz') - version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz') + resources = { + 'clang-tools-extra' : { + 'url' : 'http://llvm.org/releases/%(version)s/clang-tools-extra-%(version)s.src.tar.xz', + 'destination' : 'tools' + }, + } + releases = [ + { + 'version' : '3.7.0', + 'md5':'8f9d27335e7331cf0a4711e952f21f01', + 'resources' : { 'clang-tools-extra' : 'd5a87dacb65d981a427a536f6964642e' } + }, + { + 'version' : '3.6.2', + 'md5':'ff862793682f714bb7862325b9c06e20', + 'resources' : { 'clang-tools-extra' : '3ebc1dc41659fcec3db1b47d81575e06' } + }, + { + 'version' : '3.5.1', + 'md5':'93f9532f8f7e6f1d8e5c1116907051cb', + 'resources' : { 'clang-tools-extra' : 'f13f31ed3038acadc6fa63fef812a246' } + }, + ] - ########## - # @3.7.0 - resource(name='clang-tools-extra', - url='http://llvm.org/releases/3.7.0/clang-tools-extra-3.7.0.src.tar.xz', - md5='d5a87dacb65d981a427a536f6964642e', destination='tools', when='@3.7.0') - ########## + for release in releases: + version(release['version'], release['md5'], url=clang_url % release) + depends_on('llvm@%(version)s' % release, when='@%(version)s' % release) + + for name, md5 in release['resources'].items(): + resource(name=name, + url=resources[name]['url'] % release, + md5=md5, + destination=resources[name]['destination'], + when='@%(version)s' % release) def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag @@ -92,4 +114,4 @@ def do_link(source_dir, destination_dir): # Link folder and files in lib llvm_dir = clang_lib_dir_at(spec['llvm'].prefix) clang_dir = clang_lib_dir_at(self.prefix) - do_link(llvm_dir, clang_dir) \ No newline at end of file + do_link(llvm_dir, clang_dir) diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py index 7ec160d5957..da90c220237 100644 --- a/var/spack/packages/gcc/package.py +++ b/var/spack/packages/gcc/package.py @@ -52,19 +52,45 @@ class Gcc(Package): depends_on("mpfr") depends_on("gmp") depends_on("mpc") # when @4.5: - depends_on("binutils~libiberty") + depends_on("binutils~libiberty+gold") # Save these until we can do optional deps. depends_on("isl", when=DEPENDS_ON_ISL_PREDICATE) #depends_on("ppl") #depends_on("cloog") + variant('lang_go', default=True, description="Include support for the go language") + variant('lang_fortran', default=True, description="Include support for the Fortran language") + variant('lang_c', default=True, description="Include support for the C language") + variant('lang_cxx', default=True, description="Include support for the C++ language") + variant('lang_ada', default=False, description="Include support for the ada language") + variant('lang_java', default=False, description="Include support for the java language") + variant('lang_objc', default=True, description="Include support for the Objective-C language") + variant('lang_objcxx', default=True, description="Include support for the Objective-C++ language") + variant('jit', default=False, description="Include support for JIT") + def install(self, spec, prefix): # libjava/configure needs a minor fix to install into spack paths. filter_file(r"'@.*@'", "'@[[:alnum:]]*@'", 'libjava/configure', string=True) - enabled_languages = set(('c', 'c++', 'fortran', 'java', 'objc')) - if spec.satisfies("@4.7.1:"): + enabled_languages = set()#('c', 'c++', 'fortran', 'java', 'objc') + if '+lang_fortran' in spec: + enabled_languages.add('fortran') + if '+lang_c' in spec: + enabled_languages.add('c') + if '+lang_cxx' in spec: + enabled_languages.add('c++') + if '+lang_java' in spec: + enabled_languages.add('java') + if '+lang_objc' in spec: + enabled_languages.add('objc') + if '+lang_objcxx' in spec: + enabled_languages.add('obj-c++') + if '+lang_ada' in spec: + enabled_languages.add('ada') + if '+jit' in spec: + enabled_languages.add('jit') + if spec.satisfies("@4.7.1:") and '+lang_go' in spec: enabled_languages.add('go') # Generic options to compile GCC diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index a3307584e08..8a36a873675 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -3,8 +3,9 @@ # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. -# Written by David Beckingsale, david@llnl.gov, All rights reserved. -# LLNL-CODE-647188 +# Written by David Beckingsale, david@llnl.gov +# Heavily modified by others, including Tom Scogland, scogland1@llnl.gov +# All rights reserved. LLNL-CODE-647188 # # For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. @@ -23,6 +24,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * +import os class Llvm(Package): @@ -40,33 +42,147 @@ class Llvm(Package): version('3.5.1', '2d3d8004f38852aa679e5945b8ce0b14', url='http://llvm.org/releases/3.5.1/llvm-3.5.1.src.tar.xz') version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package + variant('debug', default=False, description="Build a debug version of LLVM, this increases binary size considerably") + variant('clang', default=True, description="Build the LLVM C/C++/Objective-C compiler frontend") + variant('clang_extra', default=True, description="Build the clang extra tools") + variant('lldb', default=True, description="Build the LLVM debugger") + variant('internal_unwind', default=True, description="Build and use the libcxxabi built-in unwinder") + variant('compiler-rt', default=False, description="Compiler runtime including sanitizers and unwinding libraries") + variant('gold', default=True, description="Add support for the gold linker and build the gold linker plugin for LTO") + + # Universal dependency depends_on('python@2.7:') - variant('libcxx', default=False, description="Builds the LLVM Standard C++ library targeting C++11") + # lldb dependencies + depends_on('ncurses', when='+lldb') + depends_on('swig', when='+lldb') + depends_on('libedit', when='+lldb') - ########## - # @3.7.0 - resource(name='compiler-rt', - url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5', - destination='projects', when='@3.7.0') - resource(name='openmp', - url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f', - destination='projects', when='@3.7.0') - resource(name='libcxx', - url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd', - destination='projects', placement='libcxx', when='+libcxx@3.7.0') - resource(name='libcxxabi', - url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772', - destination='projects', placement='libcxxabi', when='+libcxx@3.7.0') - ########## + # gold support + depends_on('binutils+gold', when='+gold') + + base_url = 'http://llvm.org/releases/%%(version)s/%(pkg)s-%%(version)s.src.tar.xz' + llvm_url = base_url % { 'pkg' : 'llvm'} + + resources = { + 'compiler-rt' : { + 'url' : base_url % { 'pkg' : 'compiler-rt'}, + 'destination' : 'projects', + 'placement' : 'compiler-rt', + }, + 'openmp' : { + 'url' : base_url % { 'pkg' : 'openmp'}, + 'destination' : 'projects', + 'placement' : 'openmp', + }, + 'libcxx' : { + 'url' : base_url % { 'pkg' : 'libcxx'}, + 'destination' : 'projects', + 'placement' : 'libcxx', + }, + 'libcxxabi' : { + 'url' : base_url % { 'pkg' : 'libcxxabi'}, + 'destination' : 'projects', + 'placement' : 'libcxxabi', + }, + 'clang' : { + 'url' : base_url % { 'pkg' : 'cfe'}, + 'destination' : 'tools', + 'placement' : 'clang', + }, + 'clang-tools-extra' : { + 'url' : base_url % { 'pkg' : 'clang-tools-extra'}, + 'destination' : 'tools/clang/tools', + 'placement' : 'clang-tools-extra', + }, + 'lldb' : { + 'url' : base_url % { 'pkg' : 'lldb'}, + 'destination' : 'tools', + 'placement' : 'lldb', + }, + 'polly' : { + 'url' : base_url % { 'pkg' : 'polly'}, + 'destination' : 'tools', + 'placement' : 'polly', + }, + 'llvm-libunwind' : { + 'url' : base_url % { 'pkg' : 'libunwind'}, + 'destination' : 'projects', + 'placement' : 'libunwind', + }, + } + releases = [ + { + 'version' : '3.7.0', + 'md5':'b98b9495e5655a672d6cb83e1a180f8e', + 'resources' : { + # FIXME: compiler-rt + # 'compiler-rt' : '383c10affd513026f08936b5525523f5', + 'openmp' : 'f482c86fdead50ba246a1a2b0bbf206f', + 'polly' : '32f93ffc9cc7e042df22089761558f8b', + 'libcxx' : '46aa5175cbe1ad42d6e9c995968e56dd', + 'libcxxabi' : '5aa769e2fca79fa5335cfae8f6258772', + 'clang' : '8f9d27335e7331cf0a4711e952f21f01', + 'clang-tools-extra' : 'd5a87dacb65d981a427a536f6964642e', + 'lldb' : 'e5931740400d1dc3e7db4c7ba2ceff68', + 'llvm-libunwind' : '9a75392eb7eb8ed5c0840007e212baf5', + } + }, + { + 'version' : '3.6.2', + 'md5':'0c1ee3597d75280dee603bae9cbf5cc2', + 'resources' : { + 'compiler-rt' : 'e3bc4eb7ba8c39a6fe90d6c988927f3c', + 'openmp' : '65dd5863b9b270960a96817e9152b123', + 'libcxx' : '22214c90697636ef960a49aef7c1823a', + 'libcxxabi' : '17518e361e4e228f193dd91e8ef54ba2', + 'clang' : 'ff862793682f714bb7862325b9c06e20', + 'clang-tools-extra' : '3ebc1dc41659fcec3db1b47d81575e06', + 'lldb' : '51e5eb552f777b950bb0ff326e60d5f0', + } + }, + { + 'version' : '3.5.1', + 'md5':'2d3d8004f38852aa679e5945b8ce0b14', + 'resources' : { + 'compiler-rt' : 'd626cfb8a9712cb92b820798ab5bc1f8', + 'openmp' : '121ddb10167d7fc38b1f7e4b029cf059', + 'libcxx' : '406f09b1dab529f3f7879f4d548329d2', + 'libcxxabi' : 'b22c707e8d474a99865ad3c521c3d464', + 'clang' : '93f9532f8f7e6f1d8e5c1116907051cb', + 'clang-tools-extra' : 'f13f31ed3038acadc6fa63fef812a246', + 'lldb' : 'cc5ea8a414c62c33e760517f8929a204', + } + }, + ] + + for release in releases: + version(release['version'], release['md5'], url=llvm_url % release) + + for name, md5 in release['resources'].items(): + resource(name=name, + url=resources[name]['url'] % release, + md5=md5, + destination=resources[name]['destination'], + when='@%(version)s' % release, + placement=resources[name].get('placement', None)) def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag + build_type = 'Debug' if '+debug' in spec else 'Release' + + if '+clang' not in spec: + if '+clang_extra' in spec: + raise SpackException('The clang_extra variant requires the clang variant to be selected') + if '+lldb' in spec: + raise SpackException('The lldb variant requires the clang variant to be selected') with working_dir('spack-build', create=True): cmake('..', + '-DCMAKE_RELEASE_TYPE=' + build_type, '-DLLVM_REQUIRES_RTTI:BOOL=ON', '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix, *std_cmake_args) - make() + make("VERBOSE=1") make("install") + From 3e3667f254cd9c288fbb2641c51ed871a28f0112 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 30 Dec 2015 10:20:27 -0800 Subject: [PATCH 227/910] Revert "removing check as it can cause failures in zsh" This reverts commit 496e4856916cb881da916539635e9bfa3e1406af. Undo overly large set of changes -- LLVM wasn't ready. Tom can recommit just the zsh stuff. --- share/spack/setup-env.sh | 1 + var/spack/packages/binutils/package.py | 5 - var/spack/packages/clang/package.py | 48 +++----- var/spack/packages/gcc/package.py | 32 +---- var/spack/packages/llvm/package.py | 154 +++---------------------- 5 files changed, 36 insertions(+), 204 deletions(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 6503728d8f2..4faae0d2c9d 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -55,6 +55,7 @@ # avoids the need to come up with a user-friendly naming scheme for # spack dotfiles. ######################################################################## +arrtest[0]='test' || (echo 'Failure: arrays not supported in this version of bash.' && exit 2) function spack { # save raw arguments into an array before butchering them diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py index a79fc5fb699..123f4598f69 100644 --- a/var/spack/packages/binutils/package.py +++ b/var/spack/packages/binutils/package.py @@ -11,7 +11,6 @@ class Binutils(Package): # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell variant('krellpatch', default=False, description="build with openspeedshop based patch.") - variant('gold', default=True, description="build the gold linker") patch('binutilskrell-2.24.patch', when='@2.24+krellpatch') variant('libiberty', default=False, description='Also install libiberty.') @@ -27,10 +26,6 @@ def install(self, spec, prefix): '--enable-targets=all', '--with-sysroot=/'] - if '+gold' in spec: - configure_args.append('--enable-gold') - configure_args.append('--enable-plugins') - if '+libiberty' in spec: configure_args.append('--enable-install-libiberty') diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py index 99e805d15d0..e46e08d5f11 100644 --- a/var/spack/packages/clang/package.py +++ b/var/spack/packages/clang/package.py @@ -36,42 +36,20 @@ class Clang(Package): homepage = 'http://clang.llvm.org' url = 'http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz' - clang_url = 'http://llvm.org/releases/%(version)s/cfe-%(version)s.src.tar.xz' + depends_on('llvm@3.7.0', when='@3.7.0') + depends_on('llvm@3.6.2', when='@3.6.2') + depends_on('llvm@3.5.1', when='@3.5.1') - resources = { - 'clang-tools-extra' : { - 'url' : 'http://llvm.org/releases/%(version)s/clang-tools-extra-%(version)s.src.tar.xz', - 'destination' : 'tools' - }, - } - releases = [ - { - 'version' : '3.7.0', - 'md5':'8f9d27335e7331cf0a4711e952f21f01', - 'resources' : { 'clang-tools-extra' : 'd5a87dacb65d981a427a536f6964642e' } - }, - { - 'version' : '3.6.2', - 'md5':'ff862793682f714bb7862325b9c06e20', - 'resources' : { 'clang-tools-extra' : '3ebc1dc41659fcec3db1b47d81575e06' } - }, - { - 'version' : '3.5.1', - 'md5':'93f9532f8f7e6f1d8e5c1116907051cb', - 'resources' : { 'clang-tools-extra' : 'f13f31ed3038acadc6fa63fef812a246' } - }, - ] + version('3.7.0', '8f9d27335e7331cf0a4711e952f21f01', url='http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz') + version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz') + version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz') - for release in releases: - version(release['version'], release['md5'], url=clang_url % release) - depends_on('llvm@%(version)s' % release, when='@%(version)s' % release) - - for name, md5 in release['resources'].items(): - resource(name=name, - url=resources[name]['url'] % release, - md5=md5, - destination=resources[name]['destination'], - when='@%(version)s' % release) + ########## + # @3.7.0 + resource(name='clang-tools-extra', + url='http://llvm.org/releases/3.7.0/clang-tools-extra-3.7.0.src.tar.xz', + md5='d5a87dacb65d981a427a536f6964642e', destination='tools', when='@3.7.0') + ########## def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag @@ -114,4 +92,4 @@ def do_link(source_dir, destination_dir): # Link folder and files in lib llvm_dir = clang_lib_dir_at(spec['llvm'].prefix) clang_dir = clang_lib_dir_at(self.prefix) - do_link(llvm_dir, clang_dir) + do_link(llvm_dir, clang_dir) \ No newline at end of file diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py index da90c220237..7ec160d5957 100644 --- a/var/spack/packages/gcc/package.py +++ b/var/spack/packages/gcc/package.py @@ -52,45 +52,19 @@ class Gcc(Package): depends_on("mpfr") depends_on("gmp") depends_on("mpc") # when @4.5: - depends_on("binutils~libiberty+gold") + depends_on("binutils~libiberty") # Save these until we can do optional deps. depends_on("isl", when=DEPENDS_ON_ISL_PREDICATE) #depends_on("ppl") #depends_on("cloog") - variant('lang_go', default=True, description="Include support for the go language") - variant('lang_fortran', default=True, description="Include support for the Fortran language") - variant('lang_c', default=True, description="Include support for the C language") - variant('lang_cxx', default=True, description="Include support for the C++ language") - variant('lang_ada', default=False, description="Include support for the ada language") - variant('lang_java', default=False, description="Include support for the java language") - variant('lang_objc', default=True, description="Include support for the Objective-C language") - variant('lang_objcxx', default=True, description="Include support for the Objective-C++ language") - variant('jit', default=False, description="Include support for JIT") - def install(self, spec, prefix): # libjava/configure needs a minor fix to install into spack paths. filter_file(r"'@.*@'", "'@[[:alnum:]]*@'", 'libjava/configure', string=True) - enabled_languages = set()#('c', 'c++', 'fortran', 'java', 'objc') - if '+lang_fortran' in spec: - enabled_languages.add('fortran') - if '+lang_c' in spec: - enabled_languages.add('c') - if '+lang_cxx' in spec: - enabled_languages.add('c++') - if '+lang_java' in spec: - enabled_languages.add('java') - if '+lang_objc' in spec: - enabled_languages.add('objc') - if '+lang_objcxx' in spec: - enabled_languages.add('obj-c++') - if '+lang_ada' in spec: - enabled_languages.add('ada') - if '+jit' in spec: - enabled_languages.add('jit') - if spec.satisfies("@4.7.1:") and '+lang_go' in spec: + enabled_languages = set(('c', 'c++', 'fortran', 'java', 'objc')) + if spec.satisfies("@4.7.1:"): enabled_languages.add('go') # Generic options to compile GCC diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index 8a36a873675..a3307584e08 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -3,9 +3,8 @@ # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. -# Written by David Beckingsale, david@llnl.gov -# Heavily modified by others, including Tom Scogland, scogland1@llnl.gov -# All rights reserved. LLNL-CODE-647188 +# Written by David Beckingsale, david@llnl.gov, All rights reserved. +# LLNL-CODE-647188 # # For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. @@ -24,7 +23,6 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os class Llvm(Package): @@ -42,147 +40,33 @@ class Llvm(Package): version('3.5.1', '2d3d8004f38852aa679e5945b8ce0b14', url='http://llvm.org/releases/3.5.1/llvm-3.5.1.src.tar.xz') version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package - variant('debug', default=False, description="Build a debug version of LLVM, this increases binary size considerably") - variant('clang', default=True, description="Build the LLVM C/C++/Objective-C compiler frontend") - variant('clang_extra', default=True, description="Build the clang extra tools") - variant('lldb', default=True, description="Build the LLVM debugger") - variant('internal_unwind', default=True, description="Build and use the libcxxabi built-in unwinder") - variant('compiler-rt', default=False, description="Compiler runtime including sanitizers and unwinding libraries") - variant('gold', default=True, description="Add support for the gold linker and build the gold linker plugin for LTO") - - # Universal dependency depends_on('python@2.7:') - # lldb dependencies - depends_on('ncurses', when='+lldb') - depends_on('swig', when='+lldb') - depends_on('libedit', when='+lldb') + variant('libcxx', default=False, description="Builds the LLVM Standard C++ library targeting C++11") - # gold support - depends_on('binutils+gold', when='+gold') - - base_url = 'http://llvm.org/releases/%%(version)s/%(pkg)s-%%(version)s.src.tar.xz' - llvm_url = base_url % { 'pkg' : 'llvm'} - - resources = { - 'compiler-rt' : { - 'url' : base_url % { 'pkg' : 'compiler-rt'}, - 'destination' : 'projects', - 'placement' : 'compiler-rt', - }, - 'openmp' : { - 'url' : base_url % { 'pkg' : 'openmp'}, - 'destination' : 'projects', - 'placement' : 'openmp', - }, - 'libcxx' : { - 'url' : base_url % { 'pkg' : 'libcxx'}, - 'destination' : 'projects', - 'placement' : 'libcxx', - }, - 'libcxxabi' : { - 'url' : base_url % { 'pkg' : 'libcxxabi'}, - 'destination' : 'projects', - 'placement' : 'libcxxabi', - }, - 'clang' : { - 'url' : base_url % { 'pkg' : 'cfe'}, - 'destination' : 'tools', - 'placement' : 'clang', - }, - 'clang-tools-extra' : { - 'url' : base_url % { 'pkg' : 'clang-tools-extra'}, - 'destination' : 'tools/clang/tools', - 'placement' : 'clang-tools-extra', - }, - 'lldb' : { - 'url' : base_url % { 'pkg' : 'lldb'}, - 'destination' : 'tools', - 'placement' : 'lldb', - }, - 'polly' : { - 'url' : base_url % { 'pkg' : 'polly'}, - 'destination' : 'tools', - 'placement' : 'polly', - }, - 'llvm-libunwind' : { - 'url' : base_url % { 'pkg' : 'libunwind'}, - 'destination' : 'projects', - 'placement' : 'libunwind', - }, - } - releases = [ - { - 'version' : '3.7.0', - 'md5':'b98b9495e5655a672d6cb83e1a180f8e', - 'resources' : { - # FIXME: compiler-rt - # 'compiler-rt' : '383c10affd513026f08936b5525523f5', - 'openmp' : 'f482c86fdead50ba246a1a2b0bbf206f', - 'polly' : '32f93ffc9cc7e042df22089761558f8b', - 'libcxx' : '46aa5175cbe1ad42d6e9c995968e56dd', - 'libcxxabi' : '5aa769e2fca79fa5335cfae8f6258772', - 'clang' : '8f9d27335e7331cf0a4711e952f21f01', - 'clang-tools-extra' : 'd5a87dacb65d981a427a536f6964642e', - 'lldb' : 'e5931740400d1dc3e7db4c7ba2ceff68', - 'llvm-libunwind' : '9a75392eb7eb8ed5c0840007e212baf5', - } - }, - { - 'version' : '3.6.2', - 'md5':'0c1ee3597d75280dee603bae9cbf5cc2', - 'resources' : { - 'compiler-rt' : 'e3bc4eb7ba8c39a6fe90d6c988927f3c', - 'openmp' : '65dd5863b9b270960a96817e9152b123', - 'libcxx' : '22214c90697636ef960a49aef7c1823a', - 'libcxxabi' : '17518e361e4e228f193dd91e8ef54ba2', - 'clang' : 'ff862793682f714bb7862325b9c06e20', - 'clang-tools-extra' : '3ebc1dc41659fcec3db1b47d81575e06', - 'lldb' : '51e5eb552f777b950bb0ff326e60d5f0', - } - }, - { - 'version' : '3.5.1', - 'md5':'2d3d8004f38852aa679e5945b8ce0b14', - 'resources' : { - 'compiler-rt' : 'd626cfb8a9712cb92b820798ab5bc1f8', - 'openmp' : '121ddb10167d7fc38b1f7e4b029cf059', - 'libcxx' : '406f09b1dab529f3f7879f4d548329d2', - 'libcxxabi' : 'b22c707e8d474a99865ad3c521c3d464', - 'clang' : '93f9532f8f7e6f1d8e5c1116907051cb', - 'clang-tools-extra' : 'f13f31ed3038acadc6fa63fef812a246', - 'lldb' : 'cc5ea8a414c62c33e760517f8929a204', - } - }, - ] - - for release in releases: - version(release['version'], release['md5'], url=llvm_url % release) - - for name, md5 in release['resources'].items(): - resource(name=name, - url=resources[name]['url'] % release, - md5=md5, - destination=resources[name]['destination'], - when='@%(version)s' % release, - placement=resources[name].get('placement', None)) + ########## + # @3.7.0 + resource(name='compiler-rt', + url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5', + destination='projects', when='@3.7.0') + resource(name='openmp', + url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f', + destination='projects', when='@3.7.0') + resource(name='libcxx', + url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd', + destination='projects', placement='libcxx', when='+libcxx@3.7.0') + resource(name='libcxxabi', + url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772', + destination='projects', placement='libcxxabi', when='+libcxx@3.7.0') + ########## def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag - build_type = 'Debug' if '+debug' in spec else 'Release' - - if '+clang' not in spec: - if '+clang_extra' in spec: - raise SpackException('The clang_extra variant requires the clang variant to be selected') - if '+lldb' in spec: - raise SpackException('The lldb variant requires the clang variant to be selected') with working_dir('spack-build', create=True): cmake('..', - '-DCMAKE_RELEASE_TYPE=' + build_type, '-DLLVM_REQUIRES_RTTI:BOOL=ON', '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix, *std_cmake_args) - make("VERBOSE=1") + make() make("install") - From 8726d2fe2e1e6c7b16fafb398fdcea851392c2cd Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 10:29:11 -0800 Subject: [PATCH 228/910] removing ill-fated array check for non-portability --- share/spack/setup-env.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 4faae0d2c9d..6503728d8f2 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -55,7 +55,6 @@ # avoids the need to come up with a user-friendly naming scheme for # spack dotfiles. ######################################################################## -arrtest[0]='test' || (echo 'Failure: arrays not supported in this version of bash.' && exit 2) function spack { # save raw arguments into an array before butchering them From 926efed31c6b8280060188e18466ad7cf7be4ebf Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 10:55:52 -0800 Subject: [PATCH 229/910] another portability fix, this time for zsh/non-bash sh Yay for non-portable declaration syntax. After the previous screwiness I ran this through a number of shells, and found that this is the most portable version I coudl seem to get. --- share/spack/setup-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 6503728d8f2..586a5b836b1 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -58,7 +58,7 @@ function spack { # save raw arguments into an array before butchering them - declare -a args=( "$@" ) + args=( "$@" ) # accumulate initial flags for main spack command _sp_flags="" From 31f674da3c8676fc4ac27327ef735c8a2c12dd48 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 11:13:09 -0800 Subject: [PATCH 230/910] python libraries, libedit, and ninja A pile of libraries and tools, libedit is actually important as a replacement of readline for non-GPL projects. Also ninja may be worthwhile for some of the larger CMake projects, like llvm/clang. --- var/spack/packages/libedit/package.py | 14 ++++++++++++++ var/spack/packages/ninja/package.py | 22 ++++++++++++++++++++++ var/spack/packages/py-blessings/package.py | 15 +++++++++++++++ var/spack/packages/py-coverage/package.py | 16 ++++++++++++++++ var/spack/packages/py-tappy/package.py | 15 +++++++++++++++ var/spack/packages/py-urwid/package.py | 16 ++++++++++++++++ 6 files changed, 98 insertions(+) create mode 100644 var/spack/packages/libedit/package.py create mode 100644 var/spack/packages/ninja/package.py create mode 100644 var/spack/packages/py-blessings/package.py create mode 100644 var/spack/packages/py-coverage/package.py create mode 100644 var/spack/packages/py-tappy/package.py create mode 100644 var/spack/packages/py-urwid/package.py diff --git a/var/spack/packages/libedit/package.py b/var/spack/packages/libedit/package.py new file mode 100644 index 00000000000..bcd5212b9ea --- /dev/null +++ b/var/spack/packages/libedit/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Libedit(Package): + """An autotools compatible port of the NetBSD editline library""" + homepage = "http://thrysoee.dk/editline/" + url = "http://thrysoee.dk/editline/libedit-20150325-3.1.tar.gz" + + version('3.1', '43cdb5df3061d78b5e9d59109871b4f6', url="http://thrysoee.dk/editline/libedit-20150325-3.1.tar.gz") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/packages/ninja/package.py b/var/spack/packages/ninja/package.py new file mode 100644 index 00000000000..9e6bf4e358d --- /dev/null +++ b/var/spack/packages/ninja/package.py @@ -0,0 +1,22 @@ +from spack import * +import os + +class Ninja(Package): + """ A small, fast Make alternative """ + homepage = "https://martine.github.io/ninja/" + url = "https://github.com/martine/ninja/archive/v1.6.0.tar.gz" + + version('1.6.0', '254133059f2da79d8727f654d7198f43') + + extends('python') + + def install(self, spec, prefix): + sh = which('sh') + python('configure.py', '--bootstrap') + + cp = which('cp') + + bindir = os.path.join(prefix, 'bin') + mkdir(bindir) + cp('-a', '-t', bindir, 'ninja') + cp('-ra', 'misc', prefix) diff --git a/var/spack/packages/py-blessings/package.py b/var/spack/packages/py-blessings/package.py new file mode 100644 index 00000000000..f2475a0efd8 --- /dev/null +++ b/var/spack/packages/py-blessings/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyBlessings(Package): + """A nicer, kinder way to write to the terminal """ + homepage = "https://github.com/erikrose/blessings" + url = "https://pypi.python.org/packages/source/b/blessings/blessings-1.6.tar.gz" + + version('1.6', '4f552a8ebcd4982693c92571beb99394') + + depends_on('py-setuptools') + + extends("python") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-coverage/package.py b/var/spack/packages/py-coverage/package.py new file mode 100644 index 00000000000..39b2ac3b016 --- /dev/null +++ b/var/spack/packages/py-coverage/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyCoverage(Package): + """ Testing coverage checker for python """ + # FIXME: add a proper url for your package's homepage here. + homepage = "http://nedbatchelder.com/code/coverage/" + url = "https://pypi.python.org/packages/source/c/coverage/coverage-4.0a6.tar.gz" + + version('4.0a6', '1bb4058062646148965bef0796b61efc') + + depends_on('py-setuptools') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-tappy/package.py b/var/spack/packages/py-tappy/package.py new file mode 100644 index 00000000000..df61a909da0 --- /dev/null +++ b/var/spack/packages/py-tappy/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyTappy(Package): + """Python TAP interface module for unit tests""" + homepage = "https://github.com/mblayman/tappy" + # base https://pypi.python.org/pypi/cffi + url = "https://pypi.python.org/packages/source/t/tap.py/tap.py-1.6.tar.gz" + + version('1.6', 'c8bdb93ad66e05f939905172a301bedf') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-urwid/package.py b/var/spack/packages/py-urwid/package.py new file mode 100644 index 00000000000..aaa11c681d1 --- /dev/null +++ b/var/spack/packages/py-urwid/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyUrwid(Package): + """A full-featured console UI library""" + homepage = "http://urwid.org/" + url = "https://pypi.python.org/packages/source/u/urwid/urwid-1.3.0.tar.gz" + + version('1.3.0', 'a989acd54f4ff1a554add464803a9175') + + depends_on('py-setuptools') + + extends("python") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + From 01f811d5469c8cc059dae7846aa660aa0c5a4ab7 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 11:18:13 -0800 Subject: [PATCH 231/910] legacy mysql bindings for python --- var/spack/packages/py-mysqldb1/package.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 var/spack/packages/py-mysqldb1/package.py diff --git a/var/spack/packages/py-mysqldb1/package.py b/var/spack/packages/py-mysqldb1/package.py new file mode 100644 index 00000000000..fda02b49822 --- /dev/null +++ b/var/spack/packages/py-mysqldb1/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyMysqldb1(Package): + """Legacy mysql bindings for python""" + homepage = "https://github.com/farcepest/MySQLdb1" + url = "https://github.com/farcepest/MySQLdb1/archive/MySQLdb-1.2.5.tar.gz" + + version('1.2.5', '332c8f4955b6bc0c79ea15170bf7321b') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + From 30f5ccb80d62608b3006c0f2521c37d3bff6ec32 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Tue, 29 Dec 2015 12:14:52 -0800 Subject: [PATCH 232/910] create leading directories for resources This allows resources to be placed into subdirectory trees that may not exist in the base package, and may depend on other resources to be staged later. --- lib/spack/spack/package.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 84bcb15f7fe..69b263db0c0 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -705,10 +705,17 @@ def _expand_archive(stage, name=self.name): placement = {'': placement} # Make the paths in the dictionary absolute and link for key, value in placement.iteritems(): - link_path = join_path(self.stage.source_path, resource.destination, value) + target_path = join_path(self.stage.source_path, resource.destination) + link_path = join_path(target_path, value) source_path = join_path(stage.source_path, key) if not os.path.exists(link_path): # Create a symlink + try: + os.makedirs(target_path) + except OSError as err: + if err.errno == errno.EEXIST and os.path.isdir(target_path): + pass + else: raise os.symlink(source_path, link_path) ########## self.stage.chdir_to_source() From cc8c783bb29e037e3343698e7a958bce4c17add5 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Tue, 29 Dec 2015 12:27:35 -0800 Subject: [PATCH 233/910] adding errno import for the deep directory creation patch --- lib/spack/spack/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 69b263db0c0..999b5e125ee 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -34,6 +34,7 @@ README. """ import os +import errno import re import time import itertools From fcdf08e4d7da824afe95445496d8b14f7bc08724 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 12:26:55 -0800 Subject: [PATCH 234/910] allow nested resources This solution doesn't really make me happy, but does seem to work. It sorts the resources by the length of the string representing their destination. Since any nested resource must contain another resource's name in its path, it seems that should work, but there should be a better way to do this. --- lib/spack/spack/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 999b5e125ee..118069a0a7c 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -697,7 +697,9 @@ def _expand_archive(stage, name=self.name): ########## # Stage resources in appropriate path resources = self._get_resources() - for resource in resources: + # TODO: this is to allow nested resources, a better solution would be + # good + for resource in sorted(resources, key=lambda res: len(res.destination)): stage = resource.fetcher.stage _expand_archive(stage, resource.name) # Turn placement into a dict with relative paths From 4ae98f8b214ea491a50aa2ddf3d995625d1b35f7 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 15:37:06 -0800 Subject: [PATCH 235/910] significant llvm update This update significantly reworks the llvm and clang packages. The llvm package now includes variants allowing it to build and install any and all of: * clang * lldb * llvm's libunwind (why, WHY did they name it this?!?) * polly (including building it directly into the clang tools, 3.7.0 only) * clang extra tools * compiler-rt (sanitizers) * clang lto (the gold linker plugin that allows same to work) * libcxx/libcxxabi * libopenmp, also setting the default openmp runtime to same, when parameters happen this shoudl be an option of libomp or libgomp Ideally, this should have rpath setup like the gcc package does, but clang's driver has no support for specs as such, and no clearly equivalent mechanism either. If anyone has ideas on this, they would be welcome. One significant note related to gcc though, if you test this on LLNL systems, or anywhere that has multiple GCCs straddling the dwarf2 boundary and sharing a libstdc++, build a gcc with spack and use that to build clang. If you use a gcc4.8+ to build this with an older libstdc++ it will fail on missing unwind symbols because of the discrepancy. Resource handling has been changed slightly to move the unpacked archive into the target rather than use symlinks, because symlinks break certain kinds of relative paths, and orders resource staging such that nested resources are unpacked after outer ones. --- lib/spack/spack/package.py | 22 ++-- var/spack/packages/clang/package.py | 95 -------------- var/spack/packages/llvm/package.py | 184 ++++++++++++++++++++++++---- 3 files changed, 175 insertions(+), 126 deletions(-) delete mode 100644 var/spack/packages/clang/package.py diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 118069a0a7c..fe82d583949 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -36,6 +36,7 @@ import os import errno import re +import shutil import time import itertools import subprocess @@ -711,15 +712,20 @@ def _expand_archive(stage, name=self.name): target_path = join_path(self.stage.source_path, resource.destination) link_path = join_path(target_path, value) source_path = join_path(stage.source_path, key) + + try: + os.makedirs(target_path) + except OSError as err: + if err.errno == errno.EEXIST and os.path.isdir(target_path): + pass + else: raise + + # NOTE: a reasonable fix for the TODO above might be to have + # these expand in place, but expand_archive does not offer + # this + if not os.path.exists(link_path): - # Create a symlink - try: - os.makedirs(target_path) - except OSError as err: - if err.errno == errno.EEXIST and os.path.isdir(target_path): - pass - else: raise - os.symlink(source_path, link_path) + shutil.move(source_path, link_path) ########## self.stage.chdir_to_source() diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py deleted file mode 100644 index e46e08d5f11..00000000000 --- a/var/spack/packages/clang/package.py +++ /dev/null @@ -1,95 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## - - -from spack import * - -import os -import os.path - -class Clang(Package): - """The goal of the Clang project is to create a new C, C++, - Objective C and Objective C++ front-end for the LLVM compiler. - """ - homepage = 'http://clang.llvm.org' - url = 'http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz' - - depends_on('llvm@3.7.0', when='@3.7.0') - depends_on('llvm@3.6.2', when='@3.6.2') - depends_on('llvm@3.5.1', when='@3.5.1') - - version('3.7.0', '8f9d27335e7331cf0a4711e952f21f01', url='http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz') - version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz') - version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz') - - ########## - # @3.7.0 - resource(name='clang-tools-extra', - url='http://llvm.org/releases/3.7.0/clang-tools-extra-3.7.0.src.tar.xz', - md5='d5a87dacb65d981a427a536f6964642e', destination='tools', when='@3.7.0') - ########## - - def install(self, spec, prefix): - env['CXXFLAGS'] = self.compiler.cxx11_flag - - with working_dir('spack-build', create=True): - - options = [] - if '@3.7.0:' in spec: - options.append('-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp') - options.extend(std_cmake_args) - - cmake('..', - '-DCLANG_PATH_TO_LLVM_BUILD:PATH=%s' % spec['llvm'].prefix, - '-DLLVM_MAIN_SRC_DIR:PATH=%s' % spec['llvm'].prefix, - *options) - make() - make("install") - # CLang doesn't look in llvm folders for system headers... - self.link_llvm_directories(spec) - - def link_llvm_directories(self, spec): - - def clang_include_dir_at(root): - return join_path(root, 'include') - - def clang_lib_dir_at(root): - return join_path(root, 'lib/clang/', str(self.version), 'include') - - def do_link(source_dir, destination_dir): - if os.path.exists(source_dir): - for name in os.listdir(source_dir): - source = join_path(source_dir, name) - link = join_path(destination_dir, name) - os.symlink(source, link) - - # Link folder and files in include - llvm_dir = clang_include_dir_at(spec['llvm'].prefix) - clang_dir = clang_include_dir_at(self.prefix) - do_link(llvm_dir, clang_dir) - # Link folder and files in lib - llvm_dir = clang_lib_dir_at(spec['llvm'].prefix) - clang_dir = clang_lib_dir_at(self.prefix) - do_link(llvm_dir, clang_dir) \ No newline at end of file diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index a3307584e08..eeb2c11fe24 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -35,38 +35,176 @@ class Llvm(Package): homepage = 'http://llvm.org/' url = 'http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz' - version('3.7.0', 'b98b9495e5655a672d6cb83e1a180f8e', url='http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz') - version('3.6.2', '0c1ee3597d75280dee603bae9cbf5cc2', url='http://llvm.org/releases/3.6.2/llvm-3.6.2.src.tar.xz') - version('3.5.1', '2d3d8004f38852aa679e5945b8ce0b14', url='http://llvm.org/releases/3.5.1/llvm-3.5.1.src.tar.xz') version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package + variant('debug', default=False, description="Build a debug version of LLVM, this increases binary size by an order of magnitude, make sure you have 20-30gb of space available to build this") + variant('clang', default=True, description="Build the LLVM C/C++/Objective-C compiler frontend") + variant('lldb', default=True, description="Build the LLVM debugger") + variant('internal_unwind', default=True, description="Build the libcxxabi libunwind") + variant('polly', default=True, description="Build the LLVM polyhedral optimization plugin, only builds for 3.7.0+") + variant('libcxx', default=True, description="Build the LLVM C++ standard library") + variant('compiler-rt', default=True, description="Build the LLVM compiler runtime, including sanitizers") + variant('lto', default=True, description="Add support for LTO with the gold linker plugin") + + + # Universal dependency depends_on('python@2.7:') - variant('libcxx', default=False, description="Builds the LLVM Standard C++ library targeting C++11") + # lldb dependencies + depends_on('ncurses', when='+lldb') + depends_on('swig', when='+lldb') + depends_on('libedit', when='+lldb') - ########## - # @3.7.0 - resource(name='compiler-rt', - url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5', - destination='projects', when='@3.7.0') - resource(name='openmp', - url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f', - destination='projects', when='@3.7.0') - resource(name='libcxx', - url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd', - destination='projects', placement='libcxx', when='+libcxx@3.7.0') - resource(name='libcxxabi', - url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772', - destination='projects', placement='libcxxabi', when='+libcxx@3.7.0') - ########## + # gold support + depends_on('binutils+gold', when='+lto') + + # polly plugin + depends_on('gmp', when='+polly') + depends_on('isl', when='+polly') + + base_url = 'http://llvm.org/releases/%%(version)s/%(pkg)s-%%(version)s.src.tar.xz' + llvm_url = base_url % { 'pkg' : 'llvm'} + + resources = { + 'compiler-rt' : { + 'url' : base_url % { 'pkg' : 'compiler-rt'}, + 'destination' : 'projects', + 'placement' : 'compiler-rt', + }, + 'openmp' : { + 'url' : base_url % { 'pkg' : 'openmp'}, + 'destination' : 'projects', + 'placement' : 'openmp', + }, + 'libcxx' : { + 'url' : base_url % { 'pkg' : 'libcxx'}, + 'destination' : 'projects', + 'placement' : 'libcxx', + }, + 'libcxxabi' : { + 'url' : base_url % { 'pkg' : 'libcxxabi'}, + 'destination' : 'projects', + 'placement' : 'libcxxabi', + }, + 'clang' : { + 'url' : base_url % { 'pkg' : 'cfe'}, + 'destination' : 'tools', + 'placement' : 'clang', + }, + 'clang-tools-extra' : { + 'url' : base_url % { 'pkg' : 'clang-tools-extra'}, + 'destination' : 'tools/clang/tools', + 'placement' : 'extra', + }, + 'lldb' : { + 'url' : base_url % { 'pkg' : 'lldb'}, + 'destination' : 'tools', + 'placement' : 'lldb', + }, + 'polly' : { + 'url' : base_url % { 'pkg' : 'polly'}, + 'destination' : 'tools', + 'placement' : 'polly', + }, + 'llvm-libunwind' : { + 'url' : base_url % { 'pkg' : 'libunwind'}, + 'destination' : 'projects', + 'placement' : 'libunwind', + }, + } + releases = [ + { + 'version' : '3.7.0', + 'md5':'b98b9495e5655a672d6cb83e1a180f8e', + 'resources' : { + 'compiler-rt' : '383c10affd513026f08936b5525523f5', + 'openmp' : 'f482c86fdead50ba246a1a2b0bbf206f', + 'polly' : '32f93ffc9cc7e042df22089761558f8b', + 'libcxx' : '46aa5175cbe1ad42d6e9c995968e56dd', + 'libcxxabi' : '5aa769e2fca79fa5335cfae8f6258772', + 'clang' : '8f9d27335e7331cf0a4711e952f21f01', + 'clang-tools-extra' : 'd5a87dacb65d981a427a536f6964642e', + 'lldb' : 'e5931740400d1dc3e7db4c7ba2ceff68', + 'llvm-libunwind' : '9a75392eb7eb8ed5c0840007e212baf5', + } + }, + { + 'version' : '3.6.2', + 'md5':'0c1ee3597d75280dee603bae9cbf5cc2', + 'resources' : { + 'compiler-rt' : 'e3bc4eb7ba8c39a6fe90d6c988927f3c', + 'openmp' : '65dd5863b9b270960a96817e9152b123', + 'libcxx' : '22214c90697636ef960a49aef7c1823a', + 'libcxxabi' : '17518e361e4e228f193dd91e8ef54ba2', + 'clang' : 'ff862793682f714bb7862325b9c06e20', + 'clang-tools-extra' : '3ebc1dc41659fcec3db1b47d81575e06', + 'lldb' : '51e5eb552f777b950bb0ff326e60d5f0', + } + }, + { + 'version' : '3.5.1', + 'md5':'2d3d8004f38852aa679e5945b8ce0b14', + 'resources' : { + 'compiler-rt' : 'd626cfb8a9712cb92b820798ab5bc1f8', + 'openmp' : '121ddb10167d7fc38b1f7e4b029cf059', + 'libcxx' : '406f09b1dab529f3f7879f4d548329d2', + 'libcxxabi' : 'b22c707e8d474a99865ad3c521c3d464', + 'clang' : '93f9532f8f7e6f1d8e5c1116907051cb', + 'clang-tools-extra' : 'f13f31ed3038acadc6fa63fef812a246', + 'lldb' : 'cc5ea8a414c62c33e760517f8929a204', + } + }, + ] + + for release in releases: + version(release['version'], release['md5'], url=llvm_url % release) + + for name, md5 in release['resources'].items(): + resource(name=name, + url=resources[name]['url'] % release, + md5=md5, + destination=resources[name]['destination'], + when='@%(version)s' % release, + placement=resources[name].get('placement', None)) def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag + cmake_args = [ arg for arg in std_cmake_args if 'BUILD_TYPE' not in arg ] + + build_type = 'RelWithDebInfo' if '+debug' in spec else 'Release' + cmake_args.extend([ + '..', + '-DCMAKE_BUILD_TYPE=' + build_type, + '-DLLVM_REQUIRES_RTTI:BOOL=ON', + '-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp', + '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix ]) + + if '+lto' in spec: + cmake_args.append('-DLLVM_BINUTILS_INCDIR=' + os.path.join( spec['binutils'].prefix, 'include')) + if '+polly' in spec: + cmake_args.append('-DPOLLY_LINK_INTO_TOOLS:Bool=ON') + else: + cmake_args.append('-DLLVM_EXTERNAL_POLLY_BUILD:Bool=OFF') + + if '+clang' not in spec: + cmake_args.append('-DLLVM_EXTERNAL_CLANG_BUILD:Bool=OFF') + if '+lldb' not in spec: + cmake_args.append('-DLLVM_EXTERNAL_LLDB_BUILD:Bool=OFF') + if '+internal_unwind' not in spec: + cmake_args.append('-DLLVM_EXTERNAL_LIBUNWIND_BUILD:Bool=OFF') + if '+libcxx' not in spec: + cmake_args.append('-DLLVM_EXTERNAL_LIBCXX_BUILD:Bool=OFF') + cmake_args.append('-DLLVM_EXTERNAL_LIBCXXABI_BUILD:Bool=OFF') + if '+compiler-rt' not in spec: + cmake_args.append('-DLLVM_EXTERNAL_COMPILER_RT_BUILD:Bool=OFF') + + if '+clang' not in spec: + if '+clang_extra' in spec: + raise SpackException('The clang_extra variant requires the clang variant to be selected') + if '+lldb' in spec: + raise SpackException('The lldb variant requires the clang variant to be selected') with working_dir('spack-build', create=True): - cmake('..', - '-DLLVM_REQUIRES_RTTI:BOOL=ON', - '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix, - *std_cmake_args) + cmake(*cmake_args) make() make("install") From 5d89fb8dfa200072fbbd2d2945c0a38917477ae1 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 16:56:00 -0800 Subject: [PATCH 236/910] ensure that clang-query gets installed --- var/spack/packages/llvm/package.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index eeb2c11fe24..19d82d81b71 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * +import os, shutil class Llvm(Package): @@ -182,7 +183,7 @@ def install(self, spec, prefix): if '+lto' in spec: cmake_args.append('-DLLVM_BINUTILS_INCDIR=' + os.path.join( spec['binutils'].prefix, 'include')) if '+polly' in spec: - cmake_args.append('-DPOLLY_LINK_INTO_TOOLS:Bool=ON') + cmake_args.append('-DLINK_POLLY_INTO_TOOLS:Bool=ON') else: cmake_args.append('-DLLVM_EXTERNAL_POLLY_BUILD:Bool=OFF') @@ -205,6 +206,10 @@ def install(self, spec, prefix): raise SpackException('The lldb variant requires the clang variant to be selected') with working_dir('spack-build', create=True): - cmake(*cmake_args) - make() + # cmake(*cmake_args) + # make() make("install") + query_path = os.path.join('bin', 'clang-query') + # Manually install clang-query, because llvm doesn't... + if os.path.exists(query_path): + shutil.copy(query_path, os.path.join(prefix, 'bin')) From 48f19b5fdec848339314c456e69fbd761fd277dd Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 16:56:25 -0800 Subject: [PATCH 237/910] add the gold linker to binutils --- var/spack/packages/binutils/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py index 123f4598f69..1de796bee0b 100644 --- a/var/spack/packages/binutils/package.py +++ b/var/spack/packages/binutils/package.py @@ -11,6 +11,7 @@ class Binutils(Package): # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell variant('krellpatch', default=False, description="build with openspeedshop based patch.") + variant('gold', default=True, description="build the gold linker") patch('binutilskrell-2.24.patch', when='@2.24+krellpatch') variant('libiberty', default=False, description='Also install libiberty.') @@ -26,6 +27,9 @@ def install(self, spec, prefix): '--enable-targets=all', '--with-sysroot=/'] + if '+gold' in spec: + configure_args.append('--enable-gold') + if '+libiberty' in spec: configure_args.append('--enable-install-libiberty') From dcddb19e5b55c8c61279a084bdbaa95047236ccb Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 1 Jan 2016 17:35:01 +0100 Subject: [PATCH 238/910] added class decorator to define composite classes --- lib/spack/spack/util/pattern.py | 98 +++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 lib/spack/spack/util/pattern.py diff --git a/lib/spack/spack/util/pattern.py b/lib/spack/spack/util/pattern.py new file mode 100644 index 00000000000..8d1584b4c29 --- /dev/null +++ b/lib/spack/spack/util/pattern.py @@ -0,0 +1,98 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import inspect +import collections +import functools + + +def composite(interface=None, method_list=None, container=list): + """ + Returns a class decorator that patches a class adding all the methods it needs to be a composite for a given + interface. + + :param interface: class exposing the interface to which the composite object must conform. Only non-private and + non-special methods will be taken into account + + :param method_list: names of methods that should be part of the composite + + :param container: container for the composite object (default = list). Must fulfill the MutableSequence contract. + The composite class will expose the container API to manage object composition + + :return: class decorator + """ + # Check if container fulfills the MutableSequence contract and raise an exception if it doesn't + # The patched class returned by the decorator will inherit from the container class to expose the + # interface needed to manage objects composition + if not issubclass(container, collections.MutableSequence): + raise TypeError("Container must fulfill the MutableSequence contract") + + # Check if at least one of the 'interface' or the 'method_list' arguments are defined + if interface is None and method_list is None: + raise TypeError("Either 'interface' or 'method_list' must be defined on a call to composite") + + def cls_decorator(cls): + # Retrieve the base class of the composite. Inspect its the methods and decide which ones will be overridden + def no_special_no_private(x): + return inspect.ismethod(x) and not x.__name__.startswith('_') + + # Patch the behavior of each of the methods in the previous list. This is done associating an instance of the + # descriptor below to any method that needs to be patched. + class IterateOver(object): + """ + Decorator used to patch methods in a composite. It iterates over all the items in the instance containing the + associated attribute and calls for each of them an attribute with the same name + """ + def __init__(self, name, func=None): + self.name = name + self.func = func + + def __get__(self, instance, owner): + def getter(*args, **kwargs): + for item in instance: + getattr(item, self.name)(*args, **kwargs) + # If we are using this descriptor to wrap a method from an interface, then we must conditionally + # use the `functools.wraps` decorator to set the appropriate fields. + if self.func is not None: + getter = functools.wraps(self.func)(getter) + return getter + + dictionary_for_type_call = {} + # Construct a dictionary with the methods explicitly passed as name + if method_list is not None: + method_list_dict = {name: IterateOver(name) for name in method_list} + dictionary_for_type_call.update(method_list_dict) + # Construct a dictionary with the methods inspected from the interface + if interface is not None: + interface_methods = {name: method for name, method in inspect.getmembers(interface, predicate=no_special_no_private)} + interface_methods_dict = {name: IterateOver(name, method) for name, method in interface_methods.iteritems()} + dictionary_for_type_call.update(interface_methods_dict) + # Get the methods that are defined in the scope of the composite class and override any previous definition + cls_method = {name: method for name, method in inspect.getmembers(cls, predicate=inspect.ismethod)} + dictionary_for_type_call.update(cls_method) + # Generate the new class on the fly and return it + wrapper_class = type(cls.__name__, (cls, container), dictionary_for_type_call) + return wrapper_class + + return cls_decorator From f5a49280c3b9063c6deb29307cd6356bf75cedd5 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 1 Jan 2016 17:26:49 -0500 Subject: [PATCH 239/910] Enable C++ and Fortran languages by default --- var/spack/packages/hdf5/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/hdf5/package.py b/var/spack/packages/hdf5/package.py index fdc77c121a9..9a401643411 100644 --- a/var/spack/packages/hdf5/package.py +++ b/var/spack/packages/hdf5/package.py @@ -15,8 +15,8 @@ class Hdf5(Package): version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') version('1.8.13', 'c03426e9e77d7766944654280b467289') - variant('cxx', default=False, description='Enable C++ support') - variant('fortran', default=False, description='Enable Fortran support') + variant('cxx', default=True, description='Enable C++ support') + variant('fortran', default=True, description='Enable Fortran support') variant('mpi', default=False, description='Enable MPI support') variant('threadsafe', default=False, description='Enable multithreading') From 03537a6be414f92114e240435b6ecbff200d8717 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 1 Jan 2016 17:38:42 -0500 Subject: [PATCH 240/910] Support Darwin for Boost Ensure that builds on Darwin use the "darwin" toolset. Also update Boost to 1.60.0. --- var/spack/packages/boost/package.py | 60 +++++++++++++++++------------ 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index 81dadbbf611..9b7c22c53d9 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -14,6 +14,7 @@ class Boost(Package): list_url = "http://sourceforge.net/projects/boost/files/boost/" list_depth = 2 + version('1.60.0', '65a840e1a0b13a558ff19eeb2c4f0cbe') version('1.59.0', '6aa9a5c6a4ca1016edd0ed1178e3cb87') version('1.58.0', 'b8839650e61e9c1c0a89f371dd475546') version('1.57.0', '1be49befbdd9a5ce9def2983ba3e7b76') @@ -48,11 +49,11 @@ class Boost(Package): variant('mpi', default=False, description='Activate the component Boost.MPI') variant('compression', default=True, description='Activate the compression Boost.iostreams') - depends_on('mpi', when='+mpi') depends_on('python', when='+python') - depends_on('zlib', when='+compression') + depends_on('mpi', when='+mpi') depends_on('bzip2', when='+compression') - + depends_on('zlib', when='+compression') + def url_for_version(self, version): """Handle Boost's weird URLs, which write the version two different ways.""" parts = [str(p) for p in Version(version)] @@ -61,20 +62,23 @@ def url_for_version(self, version): return "http://downloads.sourceforge.net/project/boost/boost/%s/boost_%s.tar.bz2" % ( dots, underscores) - def determine_toolset(self): - toolsets = {'gcc': 'gcc', + def determine_toolset(self, spec): + if spec.satisfies("=darwin-x86_64"): + return 'darwin' + + toolsets = {'g++': 'gcc', 'icpc': 'intel', 'clang++': 'clang'} for cc, toolset in toolsets.iteritems(): - if(cc in self.compiler.cxx_names): + if cc in self.compiler.cxx_names: return toolset # fallback to gcc if no toolset found return 'gcc' def determine_bootstrap_options(self, spec, options): - options.append('--with-toolset=%s' % self.determine_toolset()) + options.append('--with-toolset=%s' % self.determine_toolset(spec)) without_libs = [] if '~mpi' in spec: @@ -82,17 +86,20 @@ def determine_bootstrap_options(self, spec, options): if '~python' in spec: without_libs.append('python') else: - options.append('--with-python=%s' % (spec['python'].prefix.bin + '/python')) + options.append('--with-python=%s' % + join_path(spec['python'].prefix.bin, 'python')) if without_libs: options.append('--without-libraries=%s' % ','.join(without_libs)) with open('user-config.jam', 'w') as f: if '+mpi' in spec: - f.write('using mpi : %s ;\n' % (spec['mpi'].prefix.bin + '/mpicxx')) + f.write('using mpi : %s ;\n' % + joinpath(spec['mpi'].prefix.bin, 'mpicxx')) if '+python' in spec: - f.write('using python : %s : %s ;\n' % (spec['python'].version, - (spec['python'].prefix.bin + '/python'))) + f.write('using python : %s : %s ;\n' % + (spec['python'].version, + joinpath(spec['python'].prefix.bin, 'python'))) def determine_b2_options(self, spec, options): if '+debug' in spec: @@ -101,22 +108,26 @@ def determine_b2_options(self, spec, options): options.append('variant=release') if '~compression' in spec: - options.extend(['-s NO_BZIP2=1', - '-s NO_ZLIB=1', - ]) + options.extend([ + '-s', 'NO_BZIP2=1', + '-s', 'NO_ZLIB=1']) if '+compression' in spec: - options.extend(['-s BZIP2_INCLUDE=%s' % spec['bzip2'].prefix.include, - '-s BZIP2_LIBPATH=%s' % spec['bzip2'].prefix.lib, - '-s ZLIB_INCLUDE=%s' % spec['zlib'].prefix.include, - '-s ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib]) + options.extend([ + '-s', 'BZIP2_INCLUDE=%s' % spec['bzip2'].prefix.include, + '-s', 'BZIP2_LIBPATH=%s' % spec['bzip2'].prefix.lib, + '-s', 'ZLIB_INCLUDE=%s' % spec['zlib'].prefix.include, + '-s', 'ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib, + ]) - options.extend(['toolset=%s' % self.determine_toolset(), - 'link=static,shared', - '--layout=tagged']) + options.extend([ + 'toolset=%s' % self.determine_toolset(spec), + 'link=static,shared', + 'threading=single,multi', + '--layout=tagged']) def install(self, spec, prefix): - # to make him find the user-config.jam + # to make Boost find the user-config.jam env['BOOST_BUILD_PATH'] = './' bootstrap = Executable('./bootstrap.sh') @@ -130,9 +141,8 @@ def install(self, spec, prefix): b2name = './b2' if spec.satisfies('@1.47:') else './bjam' b2 = Executable(b2name) - b2_options = ['-j %s' % make_jobs] + b2_options = ['-j', '%s' % make_jobs] self.determine_b2_options(spec, b2_options) - b2('install', 'threading=single', *b2_options) - b2('install', 'threading=multi', *b2_options) + b2('install', *b2_options) From 356836ccbb6a187588ea6454bf1b7f7a679c5706 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Fri, 1 Jan 2016 22:30:02 -0800 Subject: [PATCH 241/910] removed extraneous comment lines --- var/spack/packages/llvm/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index 19d82d81b71..61e9b68542d 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -206,8 +206,6 @@ def install(self, spec, prefix): raise SpackException('The lldb variant requires the clang variant to be selected') with working_dir('spack-build', create=True): - # cmake(*cmake_args) - # make() make("install") query_path = os.path.join('bin', 'clang-query') # Manually install clang-query, because llvm doesn't... From f499f71f64994d1914ca3e8195a79828f442f6e8 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 1 Jan 2016 17:36:58 +0100 Subject: [PATCH 242/910] Package : factored out code in do_stage and do_fetch, changed mirror command accordingly --- lib/spack/spack/fetch_strategy.py | 116 ++++++++++++++---------------- lib/spack/spack/mirror.py | 80 ++++++++------------- lib/spack/spack/package.py | 113 +++++++++++++---------------- lib/spack/spack/stage.py | 51 ++++++++++++- 4 files changed, 181 insertions(+), 179 deletions(-) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index a9374fb34b9..72b6abe2707 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -55,53 +55,59 @@ from spack.version import Version, ver from spack.util.compression import decompressor_for, extension +import spack.util.pattern as pattern + """List of all fetch strategies, created by FetchStrategy metaclass.""" all_strategies = [] + def _needs_stage(fun): """Many methods on fetch strategies require a stage to be set using set_stage(). This decorator adds a check for self.stage.""" + @wraps(fun) def wrapper(self, *args, **kwargs): if not self.stage: raise NoStageError(fun) return fun(self, *args, **kwargs) + return wrapper class FetchStrategy(object): """Superclass of all fetch strategies.""" - enabled = False # Non-abstract subclasses should be enabled. + enabled = False # Non-abstract subclasses should be enabled. required_attributes = None # Attributes required in version() args. class __metaclass__(type): """This metaclass registers all fetch strategies in a list.""" + def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) if cls.enabled: all_strategies.append(cls) - def __init__(self): # The stage is initialized late, so that fetch strategies can be constructed # at package construction time. This is where things will be fetched. self.stage = None - def set_stage(self, stage): """This is called by Stage before any of the fetching methods are called on the stage.""" self.stage = stage - # Subclasses need to implement these methods def fetch(self): pass # Return True on success, False on fail. + def check(self): pass # Do checksum. + def expand(self): pass # Expand archive. + def reset(self): pass # Revert to freshly downloaded state. def archive(self, destination): pass # Used to create tarball for mirror. - def __str__(self): # Should be human readable URL. + def __str__(self): # Should be human readable URL. return "FetchStrategy.__str___" # This method is used to match fetch strategies to version() @@ -111,6 +117,15 @@ def matches(cls, args): return any(k in args for k in cls.required_attributes) +@pattern.composite(interface=FetchStrategy) +class FetchStrategyComposite(object): + """ + Composite for a FetchStrategy object. Implements the GoF composite pattern. + """ + matches = FetchStrategy.matches + set_stage = FetchStrategy.set_stage + + class URLFetchStrategy(FetchStrategy): """FetchStrategy that pulls source code from a URL for an archive, checks the archive against a checksum,and decompresses the archive. @@ -142,19 +157,19 @@ def fetch(self): tty.msg("Trying to fetch from %s" % self.url) - curl_args = ['-O', # save file to disk - '-f', # fail on >400 errors - '-D', '-', # print out HTML headers - '-L', self.url,] + curl_args = ['-O', # save file to disk + '-f', # fail on >400 errors + '-D', '-', # print out HTML headers + '-L', self.url, ] if sys.stdout.isatty(): curl_args.append('-#') # status bar when using a tty else: - curl_args.append('-sS') # just errors when not. + curl_args.append('-sS') # just errors when not. # Run curl but grab the mime type from the http headers headers = spack.curl( - *curl_args, return_output=True, fail_on_error=False) + *curl_args, return_output=True, fail_on_error=False) if spack.curl.returncode != 0: # clean up archive on failure. @@ -164,24 +179,23 @@ def fetch(self): if spack.curl.returncode == 22: # This is a 404. Curl will print the error. raise FailedDownloadError( - self.url, "URL %s was not found!" % self.url) + self.url, "URL %s was not found!" % self.url) elif spack.curl.returncode == 60: # This is a certificate error. Suggest spack -k raise FailedDownloadError( - self.url, - "Curl was unable to fetch due to invalid certificate. " - "This is either an attack, or your cluster's SSL configuration " - "is bad. If you believe your SSL configuration is bad, you " - "can try running spack -k, which will not check SSL certificates." - "Use this at your own risk.") + self.url, + "Curl was unable to fetch due to invalid certificate. " + "This is either an attack, or your cluster's SSL configuration " + "is bad. If you believe your SSL configuration is bad, you " + "can try running spack -k, which will not check SSL certificates." + "Use this at your own risk.") else: # This is some other curl error. Curl will print the # error, but print a spack message too raise FailedDownloadError( - self.url, "Curl failed with error %d" % spack.curl.returncode) - + self.url, "Curl failed with error %d" % spack.curl.returncode) # Check if we somehow got an HTML file rather than the archive we # asked for. We only look at the last content type, to handle @@ -196,7 +210,6 @@ def fetch(self): if not self.archive_file: raise FailedDownloadError(self.url) - @property def archive_file(self): """Path to the source archive within this stage directory.""" @@ -209,7 +222,7 @@ def expand(self): self.stage.chdir() if not self.archive_file: raise NoArchiveFileError("URLFetchStrategy couldn't find archive file", - "Failed on expand() for URL %s" % self.url) + "Failed on expand() for URL %s" % self.url) decompress = decompressor_for(self.archive_file) @@ -241,7 +254,6 @@ def expand(self): # Set the wd back to the stage when done. self.stage.chdir() - def archive(self, destination): """Just moves this archive to the destination.""" if not self.archive_file: @@ -252,7 +264,6 @@ def archive(self, destination): shutil.move(self.archive_file, destination) - @_needs_stage def check(self): """Check the downloaded archive against a checksum digest. @@ -263,9 +274,8 @@ def check(self): checker = crypto.Checker(self.digest) if not checker.check(self.archive_file): raise ChecksumError( - "%s checksum failed for %s." % (checker.hash_name, self.archive_file), - "Expected %s but got %s." % (self.digest, checker.sum)) - + "%s checksum failed for %s." % (checker.hash_name, self.archive_file), + "Expected %s but got %s." % (self.digest, checker.sum)) @_needs_stage def reset(self): @@ -277,12 +287,10 @@ def reset(self): shutil.rmtree(self.stage.source_path, ignore_errors=True) self.expand() - def __repr__(self): url = self.url if self.url else "no url" return "URLFetchStrategy<%s>" % url - def __str__(self): if self.url: return self.url @@ -298,33 +306,30 @@ def __init__(self, name, *rev_types, **kwargs): # Set a URL based on the type of fetch strategy. self.url = kwargs.get(name, None) if not self.url: raise ValueError( - "%s requires %s argument." % (self.__class__, name)) + "%s requires %s argument." % (self.__class__, name)) # Ensure that there's only one of the rev_types if sum(k in kwargs for k in rev_types) > 1: raise FetchStrategyError( - "Supply only one of %s to fetch with %s." % ( - comma_or(rev_types), name)) + "Supply only one of %s to fetch with %s." % ( + comma_or(rev_types), name)) # Set attributes for each rev type. for rt in rev_types: setattr(self, rt, kwargs.get(rt, None)) - @_needs_stage def check(self): tty.msg("No checksum needed when fetching with %s." % self.name) - @_needs_stage def expand(self): tty.debug("Source fetched with %s is already expanded." % self.name) - @_needs_stage def archive(self, destination, **kwargs): - assert(extension(destination) == 'tar.gz') - assert(self.stage.source_path.startswith(self.stage.path)) + assert (extension(destination) == 'tar.gz') + assert (self.stage.source_path.startswith(self.stage.path)) tar = which('tar', required=True) @@ -338,16 +343,13 @@ def archive(self, destination, **kwargs): self.stage.chdir() tar('-czf', destination, os.path.basename(self.stage.source_path)) - def __str__(self): return "VCS: %s" % self.url - def __repr__(self): return "%s<%s>" % (self.__class__, self.url) - class GitFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a git repository. Use like this in a package: @@ -369,23 +371,20 @@ class GitFetchStrategy(VCSFetchStrategy): def __init__(self, **kwargs): super(GitFetchStrategy, self).__init__( - 'git', 'tag', 'branch', 'commit', **kwargs) + 'git', 'tag', 'branch', 'commit', **kwargs) self._git = None - @property def git_version(self): vstring = self.git('--version', return_output=True).lstrip('git version ') return Version(vstring) - @property def git(self): if not self._git: self._git = which('git', required=True) return self._git - @_needs_stage def fetch(self): self.stage.chdir() @@ -418,7 +417,7 @@ def fetch(self): if self.branch: args.extend(['--branch', self.branch]) elif self.tag and self.git_version >= ver('1.8.5.2'): - args.extend(['--branch', self.tag]) + args.extend(['--branch', self.tag]) # Try to be efficient if we're using a new enough git. # This checks out only one branch's history @@ -429,7 +428,7 @@ def fetch(self): # Yet more efficiency, only download a 1-commit deep tree if self.git_version >= ver('1.7.1'): try: - self.git(*(args + ['--depth','1', self.url])) + self.git(*(args + ['--depth', '1', self.url])) cloned = True except spack.error.SpackError: # This will fail with the dumb HTTP transport @@ -452,18 +451,15 @@ def fetch(self): self.git('pull', '--tags', ignore_errors=1) self.git('checkout', self.tag) - def archive(self, destination): super(GitFetchStrategy, self).archive(destination, exclude='.git') - @_needs_stage def reset(self): self.stage.chdir_to_source() self.git('checkout', '.') self.git('clean', '-f') - def __str__(self): return "[git] %s" % self.url @@ -484,19 +480,17 @@ class SvnFetchStrategy(VCSFetchStrategy): def __init__(self, **kwargs): super(SvnFetchStrategy, self).__init__( - 'svn', 'revision', **kwargs) + 'svn', 'revision', **kwargs) self._svn = None if self.revision is not None: self.revision = str(self.revision) - @property def svn(self): if not self._svn: self._svn = which('svn', required=True) return self._svn - @_needs_stage def fetch(self): self.stage.chdir() @@ -515,7 +509,6 @@ def fetch(self): self.svn(*args) self.stage.chdir_to_source() - def _remove_untracked_files(self): """Removes untracked files in an svn repository.""" status = self.svn('status', '--no-ignore', return_output=True) @@ -529,23 +522,19 @@ def _remove_untracked_files(self): elif os.path.isdir(path): shutil.rmtree(path, ignore_errors=True) - def archive(self, destination): super(SvnFetchStrategy, self).archive(destination, exclude='.svn') - @_needs_stage def reset(self): self.stage.chdir_to_source() self._remove_untracked_files() self.svn('revert', '.', '-R') - def __str__(self): return "[svn] %s" % self.url - class HgFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a Mercurial repository. Use like this in a package: @@ -568,10 +557,9 @@ class HgFetchStrategy(VCSFetchStrategy): def __init__(self, **kwargs): super(HgFetchStrategy, self).__init__( - 'hg', 'revision', **kwargs) + 'hg', 'revision', **kwargs) self._hg = None - @property def hg(self): if not self._hg: @@ -597,11 +585,9 @@ def fetch(self): self.hg(*args) - def archive(self, destination): super(HgFetchStrategy, self).archive(destination, exclude='.hg') - @_needs_stage def reset(self): self.stage.chdir() @@ -619,7 +605,6 @@ def reset(self): shutil.move(scrubbed, source_path) self.stage.chdir_to_source() - def __str__(self): return "[hg] %s" % self.url @@ -693,9 +678,10 @@ def __init__(self, msg, long_msg): class FailedDownloadError(FetchError): """Raised wen a download fails.""" + def __init__(self, url, msg=""): super(FailedDownloadError, self).__init__( - "Failed to fetch file from URL: %s" % url, msg) + "Failed to fetch file from URL: %s" % url, msg) self.url = url @@ -718,12 +704,14 @@ def __init__(self, pkg, version): class ChecksumError(FetchError): """Raised when archive fails to checksum.""" + def __init__(self, message, long_msg=None): super(ChecksumError, self).__init__(message, long_msg) class NoStageError(FetchError): """Raised when fetch operations are called before set_stage().""" + def __init__(self, method): super(NoStageError, self).__init__( - "Must call FetchStrategy.set_stage() before calling %s" % method.__name__) + "Must call FetchStrategy.set_stage() before calling %s" % method.__name__) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 1d9b0e7ef25..44e03a8a450 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -45,12 +45,11 @@ from spack.util.compression import extension, allowed_archive -def mirror_archive_filename(spec): +def mirror_archive_filename(spec, fetcher): """Get the name of the spec's archive in the mirror.""" if not spec.version.concrete: raise ValueError("mirror.path requires spec with concrete version.") - fetcher = spec.package.fetcher if isinstance(fetcher, fs.URLFetchStrategy): # If we fetch this version with a URLFetchStrategy, use URL's archive type ext = url.downloaded_file_extension(fetcher.url) @@ -61,9 +60,9 @@ def mirror_archive_filename(spec): return "%s-%s.%s" % (spec.package.name, spec.version, ext) -def mirror_archive_path(spec): +def mirror_archive_path(spec, fetcher): """Get the relative path to the spec's archive within a mirror.""" - return join_path(spec.name, mirror_archive_filename(spec)) + return join_path(spec.name, mirror_archive_filename(spec, fetcher)) def get_matching_versions(specs, **kwargs): @@ -158,68 +157,47 @@ def create(path, specs, **kwargs): everything_already_exists = True for spec in version_specs: pkg = spec.package - - stage = None + tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) try: - # create a subdirectory for the current package@version - archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec))) - subdir = os.path.dirname(archive_path) - mkdirp(subdir) + for ii, stage in enumerate(pkg.stage): + fetcher = stage.fetcher + if ii == 0: + # create a subdirectory for the current package@version + archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) + name = spec.format("$_$@") + else: + resource = stage.resource + archive_path = join_path(subdir, suggest_archive_basename(resource)) + name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) + subdir = os.path.dirname(archive_path) + mkdirp(subdir) - if os.path.exists(archive_path): - tty.msg("Already added %s" % spec.format("$_$@")) - else: - everything_already_exists = False - # Set up a stage and a fetcher for the download - unique_fetch_name = spec.format("$_$@") - fetcher = fs.for_package_version(pkg, pkg.version) - stage = Stage(fetcher, name=unique_fetch_name) - fetcher.set_stage(stage) + if os.path.exists(archive_path): + tty.msg("{name} : already added".format(name=name)) + else: + everything_already_exists = False + fetcher.fetch() + if not kwargs.get('no_checksum', False): + fetcher.check() + tty.msg("{name} : checksum passed".format(name=name)) - # Do the fetch and checksum if necessary - fetcher.fetch() - if not kwargs.get('no_checksum', False): - fetcher.check() - tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version)) - - # Fetchers have to know how to archive their files. Use - # that to move/copy/create an archive in the mirror. - fetcher.archive(archive_path) - tty.msg("Added %s." % spec.format("$_$@")) - - # Fetch resources if they are associated with the spec - resources = pkg._get_resources() - for resource in resources: - resource_archive_path = join_path(subdir, suggest_archive_basename(resource)) - if os.path.exists(resource_archive_path): - tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version)) - continue - everything_already_exists = False - resource_stage_folder = pkg._resource_stage(resource) - resource_stage = Stage(resource.fetcher, name=resource_stage_folder) - resource.fetcher.set_stage(resource_stage) - resource.fetcher.fetch() - if not kwargs.get('no_checksum', False): - resource.fetcher.check() - tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version)) - resource.fetcher.archive(resource_archive_path) - tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version)) + # Fetchers have to know how to archive their files. Use + # that to move/copy/create an archive in the mirror. + fetcher.archive(archive_path) + tty.msg("{name} : added".format(name=name)) if everything_already_exists: present.append(spec) else: mirrored.append(spec) - except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) else: tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) error.append(spec) - finally: - if stage: - stage.destroy() + pkg.stage.destroy() return (present, mirrored, error) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 6673e4f3929..3ccedaa458f 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -61,7 +61,7 @@ import spack.util.web import spack.fetch_strategy as fs from spack.version import * -from spack.stage import Stage +from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive, extension from spack.util.executable import ProcessError @@ -431,23 +431,47 @@ def url_for_version(self, version): return spack.url.substitute_version(self.nearest_url(version), self.url_version(version)) + def _make_resource_stage(self, root_stage, fetcher, resource): + resource_stage_folder = self._resource_stage(resource) + # FIXME : works only for URLFetchStrategy + resource_mirror = join_path(self.name, os.path.basename(fetcher.url)) + stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource, + name=resource_stage_folder, mirror_path=resource_mirror) + return stage + + def _make_root_stage(self, fetcher): + # Construct a mirror path (TODO: get this out of package.py) + mp = spack.mirror.mirror_archive_path(self.spec, fetcher) + # Construct a path where the stage should build.. + s = self.spec + stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash()) + # Build the composite stage + stage = Stage(fetcher, mirror_path=mp, name=stage_name) + return stage + + def _make_stage(self): + # Construct a composite stage on top of the composite FetchStrategy + composite_fetcher = self.fetcher + composite_stage = StageComposite() + resources = self._get_resources() + for ii, fetcher in enumerate(composite_fetcher): + if ii == 0: + # Construct root stage first + stage = self._make_root_stage(fetcher) + else: + # Construct resource stage + resource = resources[ii - 1] # ii == 0 is root! + stage = self._make_resource_stage(composite_stage[0], fetcher, resource) + # Append the item to the composite + composite_stage.append(stage) + return composite_stage @property def stage(self): if not self.spec.concrete: raise ValueError("Can only get a stage for a concrete package.") - if self._stage is None: - # Construct a mirror path (TODO: get this out of package.py) - mp = spack.mirror.mirror_archive_path(self.spec) - - # Construct a path where the stage should build.. - s = self.spec - stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash()) - - # Build the stage - self._stage = Stage(self.fetcher, mirror_path=mp, name=stage_name) - + self._stage = self._make_stage() return self._stage @@ -457,17 +481,25 @@ def stage(self, stage): self._stage = stage + def _make_fetcher(self): + # Construct a composite fetcher that always contains at least one element (the root package). In case there + # are resources associated with the package, append their fetcher to the composite. + root_fetcher = fs.for_package_version(self, self.version) + fetcher = fs.FetchStrategyComposite() # Composite fetcher + fetcher.append(root_fetcher) # Root fetcher is always present + resources = self._get_resources() + for resource in resources: + fetcher.append(resource.fetcher) + return fetcher + @property def fetcher(self): if not self.spec.versions.concrete: - raise ValueError( - "Can only get a fetcher for a package with concrete versions.") - + raise ValueError("Can only get a fetcher for a package with concrete versions.") if not self._fetcher: - self._fetcher = fs.for_package_version(self, self.version) + self._fetcher = self._make_fetcher() return self._fetcher - @fetcher.setter def fetcher(self, f): self._fetcher = f @@ -630,7 +662,7 @@ def remove_prefix(self): def do_fetch(self): - """Creates a stage directory and downloads the taball for this package. + """Creates a stage directory and downloads the tarball for this package. Working directory will be set to the stage directory. """ if not self.spec.concrete: @@ -656,20 +688,6 @@ def do_fetch(self): self.stage.fetch() - ########## - # Fetch resources - resources = self._get_resources() - for resource in resources: - resource_stage_folder = self._resource_stage(resource) - # FIXME : works only for URLFetchStrategy - resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url)) - resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror) - resource.fetcher.set_stage(resource_stage) - # Delegate to stage object to trigger mirror logic - resource_stage.fetch() - resource_stage.check() - ########## - self._fetch_time = time.time() - start_time if spack.do_checksum and self.version in self.versions: @@ -681,39 +699,10 @@ def do_stage(self): if not self.spec.concrete: raise ValueError("Can only stage concrete packages.") - def _expand_archive(stage, name=self.name): - archive_dir = stage.source_path - if not archive_dir: - stage.expand_archive() - tty.msg("Created stage in %s." % stage.path) - else: - tty.msg("Already staged %s in %s." % (name, stage.path)) - - self.do_fetch() - _expand_archive(self.stage) - - ########## - # Stage resources in appropriate path - resources = self._get_resources() - for resource in resources: - stage = resource.fetcher.stage - _expand_archive(stage, resource.name) - # Turn placement into a dict with relative paths - placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement - if not isinstance(placement, dict): - placement = {'': placement} - # Make the paths in the dictionary absolute and link - for key, value in placement.iteritems(): - link_path = join_path(self.stage.source_path, resource.destination, value) - source_path = join_path(stage.source_path, key) - if not os.path.exists(link_path): - # Create a symlink - os.symlink(source_path, link_path) - ########## + self.stage.expand_archive() self.stage.chdir_to_source() - def do_patch(self): """Calls do_stage(), then applied patches to the expanded tarball if they haven't been applied already.""" diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 754344fc016..9b4c8fdc1d4 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -30,6 +30,8 @@ import llnl.util.tty as tty from llnl.util.filesystem import * +import spack.util.pattern as pattern + import spack import spack.config import spack.fetch_strategy as fs @@ -40,7 +42,7 @@ class Stage(object): - """A Stage object manaages a directory where some source code is + """A Stage object manages a directory where some source code is downloaded and built before being installed. It handles fetching the source code, either as an archive to be expanded or by checking it out of a repository. A stage's lifecycle @@ -276,7 +278,12 @@ def expand_archive(self): archive. Fail if the stage is not set up or if the archive is not yet downloaded. """ - self.fetcher.expand() + archive_dir = self.source_path + if not archive_dir: + self.fetcher.expand() + tty.msg("Created stage in %s." % self.path) + else: + tty.msg("Already staged %s in %s." % (self.name, self.path)) def chdir_to_source(self): @@ -310,6 +317,46 @@ def destroy(self): os.chdir(os.path.dirname(self.path)) +class ResourceStage(Stage): + def __init__(self, url_or_fetch_strategy, root, resource, **kwargs): + super(ResourceStage, self).__init__(url_or_fetch_strategy, **kwargs) + self.root_stage = root + self.resource = resource + + def expand_archive(self): + super(ResourceStage, self).expand_archive() + root_stage = self.root_stage + resource = self.resource + placement = os.path.basename(self.source_path) if resource.placement is None else resource.placement + if not isinstance(placement, dict): + placement = {'': placement} + # Make the paths in the dictionary absolute and link + for key, value in placement.iteritems(): + link_path = join_path(root_stage.source_path, resource.destination, value) + source_path = join_path(self.source_path, key) + if not os.path.exists(link_path): + # Create a symlink + os.symlink(source_path, link_path) + + +@pattern.composite(method_list=['fetch', 'check', 'expand_archive', 'restage', 'destroy']) +class StageComposite: + """ + Composite for Stage type objects. The first item in this composite is considered to be the root package, and + operations that return a value are forwarded to it. + """ + @property + def source_path(self): + return self[0].source_path + + @property + def path(self): + return self[0].path + + def chdir_to_source(self): + return self[0].chdir_to_source() + + class DIYStage(object): """Simple class that allows any directory to be a spack stage.""" def __init__(self, path): From d95d169ac50a3c253ef9712782626af1e1610ca7 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 2 Jan 2016 19:04:06 +0100 Subject: [PATCH 243/910] fixed broken unit tests --- lib/spack/spack/test/install.py | 7 +++++-- lib/spack/spack/util/pattern.py | 26 ++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 1ef4171fb2f..bb7d3f4fd4c 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -31,7 +31,7 @@ import spack from spack.stage import Stage -from spack.fetch_strategy import URLFetchStrategy +from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite from spack.directory_layout import YamlDirectoryLayout from spack.util.executable import which from spack.test.mock_packages_test import * @@ -81,7 +81,10 @@ def test_install_and_uninstall(self): pkg = spack.db.get(spec) # Fake the URL for the package so it downloads from a file. - pkg.fetcher = URLFetchStrategy(self.repo.url) + + fetcher = FetchStrategyComposite() + fetcher.append(URLFetchStrategy(self.repo.url)) + pkg.fetcher = fetcher try: pkg.do_install() diff --git a/lib/spack/spack/util/pattern.py b/lib/spack/spack/util/pattern.py index 8d1584b4c29..73c1e26aa5d 100644 --- a/lib/spack/spack/util/pattern.py +++ b/lib/spack/spack/util/pattern.py @@ -81,17 +81,35 @@ def getter(*args, **kwargs): dictionary_for_type_call = {} # Construct a dictionary with the methods explicitly passed as name if method_list is not None: - method_list_dict = {name: IterateOver(name) for name in method_list} + # python@2.7: method_list_dict = {name: IterateOver(name) for name in method_list} + method_list_dict = {} + for name in method_list: + method_list_dict[name] = IterateOver(name) dictionary_for_type_call.update(method_list_dict) # Construct a dictionary with the methods inspected from the interface if interface is not None: - interface_methods = {name: method for name, method in inspect.getmembers(interface, predicate=no_special_no_private)} - interface_methods_dict = {name: IterateOver(name, method) for name, method in interface_methods.iteritems()} + ########## + # python@2.7: interface_methods = {name: method for name, method in inspect.getmembers(interface, predicate=no_special_no_private)} + interface_methods = {} + for name, method in inspect.getmembers(interface, predicate=no_special_no_private): + interface_methods[name] = method + ########## + # python@2.7: interface_methods_dict = {name: IterateOver(name, method) for name, method in interface_methods.iteritems()} + interface_methods_dict = {} + for name, method in interface_methods.iteritems(): + interface_methods_dict[name] = IterateOver(name, method) + ########## dictionary_for_type_call.update(interface_methods_dict) # Get the methods that are defined in the scope of the composite class and override any previous definition - cls_method = {name: method for name, method in inspect.getmembers(cls, predicate=inspect.ismethod)} + ########## + # python@2.7: cls_method = {name: method for name, method in inspect.getmembers(cls, predicate=inspect.ismethod)} + cls_method = {} + for name, method in inspect.getmembers(cls, predicate=inspect.ismethod): + cls_method[name] = method + ########## dictionary_for_type_call.update(cls_method) # Generate the new class on the fly and return it + # FIXME : inherit from interface if we start to use ABC classes? wrapper_class = type(cls.__name__, (cls, container), dictionary_for_type_call) return wrapper_class From 31803d10aac4dc163eb168de4028357aa23b68c3 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Sat, 2 Jan 2016 11:59:33 -0800 Subject: [PATCH 244/910] bringing back configuration and build stages --- var/spack/packages/llvm/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index 61e9b68542d..419aef1ac99 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -206,6 +206,8 @@ def install(self, spec, prefix): raise SpackException('The lldb variant requires the clang variant to be selected') with working_dir('spack-build', create=True): + cmake(*cmake_args) + make() make("install") query_path = os.path.join('bin', 'clang-query') # Manually install clang-query, because llvm doesn't... From bf4231dd4b906f20d522b9df5f2cae2ba6291174 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 2 Jan 2016 22:48:14 +0100 Subject: [PATCH 245/910] depends_on and when multimethod support boolean --- lib/spack/spack/directives.py | 6 +++- lib/spack/spack/multimethod.py | 9 ++++-- lib/spack/spack/util/architecture.py | 46 +++++++++++++++++++++++++++ var/spack/packages/openssl/package.py | 15 +++++++++ 4 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 lib/spack/spack/util/architecture.py diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index aa9fbd8d339..02477bb904a 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -174,7 +174,11 @@ def version(pkg, ver, checksum=None, **kwargs): def _depends_on(pkg, spec, when=None): - if when is None: + # If when is False do nothing + if when is False: + return + # If when is None or True make sure the condition is always satisfied + if when is None or when is True: when = pkg.name when_spec = parse_anonymous_spec(when, pkg.name) diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index df9b9b2ab14..51c6a8e89d9 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -193,10 +193,11 @@ def install(self, prefix): platform-specific versions. There's not much we can do to get around this because of the way decorators work. """ -class when(object): def __init__(self, spec): pkg = get_calling_module_name() - self.spec = parse_anonymous_spec(spec, pkg) + if spec is True: + spec = pkg + self.spec = parse_anonymous_spec(spec, pkg) if spec is not False else None def __call__(self, method): # Get the first definition of the method in the calling scope @@ -207,7 +208,9 @@ def __call__(self, method): if not type(original_method) == SpecMultiMethod: original_method = SpecMultiMethod(original_method) - original_method.register(self.spec, method) + if self.spec is not None: + original_method.register(self.spec, method) + return original_method diff --git a/lib/spack/spack/util/architecture.py b/lib/spack/spack/util/architecture.py new file mode 100644 index 00000000000..a020c74a7ac --- /dev/null +++ b/lib/spack/spack/util/architecture.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import sys + + +def os_is_in(*args): + """ + Return True if any element in the list is equal to sys.platform, False otherwise + + :param args: list of names to be checked + :return: True or False + """ + return any(map(lambda x: x == sys.platform, args)) + + +def os_is_not_in(*args): + """ + Return True if none of the elements in the list is equal to sys.platform, False otherwise + + :param args: list of names to be checked + :return: True or False + """ + return not os_is_in(*args) diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index 40648fca492..1fa288a36dc 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -1,5 +1,7 @@ from spack import * +import spack.util.architecture as arch + class Openssl(Package): """The OpenSSL Project is a collaborative effort to develop a robust, commercial-grade, full-featured, and Open Source @@ -26,3 +28,16 @@ def install(self, spec, prefix): make() make("install") + + @when(arch.os_is_in('darwin')) + def install(self, spec, prefix): + perl = which('perl') + perl("./Configure", + "--prefix=%s" % prefix, + "--openssldir=%s/etc/openssl" % prefix, + "zlib", + "no-krb5", + "shared", + "darwin64-x86_64-cc") + filter_file(r'-arch x86_64', '', 'Makefile') + From 20b7f8a8e0ea186cde1e641c154566f279e67cd6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 2 Jan 2016 14:25:10 -0800 Subject: [PATCH 246/910] Fix bug in tests. --- lib/spack/spack/compilers/__init__.py | 29 +++++++++++----------- lib/spack/spack/test/mock_packages_test.py | 2 ++ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 67dfaa3ac9c..2a3d67c731f 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -46,6 +46,7 @@ _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] +# TODO: customize order in config file _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] def _auto_compiler_spec(function): @@ -132,7 +133,7 @@ def remove_compiler_from_config(compiler_spec, arch=None, scope=None): spack.config.update_config('compilers', update, scope) -def all_compilers(arch=None, scope=None): +def all_compilers_config(arch=None, scope=None): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ @@ -144,25 +145,25 @@ def all_compilers(arch=None, scope=None): merged_config = get_compiler_config('all', scope=scope) merged_config = spack.config._merge_yaml(merged_config, arch_config) - # Return compiler specs for the result. - return [spack.spec.CompilerSpec(s) for s in merged_config] + return merged_config + + +def all_compilers(arch=None, scope=None): + # Return compiler specs from the merged config. + return [spack.spec.CompilerSpec(s) + for s in all_compilers_config(arch, scope)] -_cached_default_compiler = None def default_compiler(): - global _cached_default_compiler - if _cached_default_compiler: - return _cached_default_compiler versions = [] - for name in _default_order: # TODO: customize order. + for name in _default_order: versions = find(name) - if versions: break - - if not versions: + if versions: + break + else: raise NoCompilersError() - _cached_default_compiler = sorted(versions)[-1] - return _cached_default_compiler + return sorted(versions)[-1] def find_compilers(*path): @@ -224,7 +225,7 @@ def compilers_for_spec(compiler_spec, arch=None, scope=None): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ - config = get_compiler_config(arch, scope) + config = all_compilers_config(arch, scope) def get_compiler(cspec): items = config[str(cspec)] diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 7642edcf4b7..e9f1f95df55 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -24,6 +24,7 @@ ############################################################################## import sys import os +import shutil import unittest import tempfile from ordereddict_backport import OrderedDict @@ -103,6 +104,7 @@ def cleanmock(self): """Restore the real packages path after any test.""" spack.repo.swap(self.db) spack.config.config_scopes = self.real_scopes + shutil.rmtree(self.temp_config, ignore_errors=True) spack.config.clear_config_caches() # Restore dependency changes that happened during the test From 56937416a92787ba712bbc73c0089cc7eb0c5f78 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 2 Jan 2016 18:18:00 -0800 Subject: [PATCH 247/910] All tests that call concretize() need to be MockPackagesTests. --- lib/spack/spack/test/directory_layout.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 107a7a6412b..925cb648edd 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -36,21 +36,25 @@ from spack.spec import Spec from spack.repository import RepoPath from spack.directory_layout import YamlDirectoryLayout +from spack.test.mock_packages_test import * + # number of packages to test (to reduce test time) max_packages = 10 -class DirectoryLayoutTest(unittest.TestCase): +class DirectoryLayoutTest(MockPackagesTest): """Tests that a directory layout works correctly and produces a consistent install path.""" def setUp(self): + super(DirectoryLayoutTest, self).setUp() self.tmpdir = tempfile.mkdtemp() self.layout = YamlDirectoryLayout(self.tmpdir) def tearDown(self): + super(DirectoryLayoutTest, self).tearDown() shutil.rmtree(self.tmpdir, ignore_errors=True) self.layout = None From 21fae634a54b3aa0f0d8f22fcf1e9d1429636c5f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 3 Jan 2016 01:19:03 -0800 Subject: [PATCH 248/910] Add Python 3 aliases to our argparse backport. --- lib/spack/external/argparse.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/spack/external/argparse.py b/lib/spack/external/argparse.py index 394e5da1524..ec9a9ee738c 100644 --- a/lib/spack/external/argparse.py +++ b/lib/spack/external/argparse.py @@ -1067,9 +1067,13 @@ class _SubParsersAction(Action): class _ChoicesPseudoAction(Action): - def __init__(self, name, help): + def __init__(self, name, aliases, help): + metavar = dest = name + if aliases: + metavar += ' (%s)' % ', '.join(aliases) sup = super(_SubParsersAction._ChoicesPseudoAction, self) - sup.__init__(option_strings=[], dest=name, help=help) + sup.__init__(option_strings=[], dest=dest, help=help, + metavar=metavar) def __init__(self, option_strings, @@ -1097,15 +1101,22 @@ def add_parser(self, name, **kwargs): if kwargs.get('prog') is None: kwargs['prog'] = '%s %s' % (self._prog_prefix, name) + aliases = kwargs.pop('aliases', ()) + # create a pseudo-action to hold the choice help if 'help' in kwargs: help = kwargs.pop('help') - choice_action = self._ChoicesPseudoAction(name, help) + choice_action = self._ChoicesPseudoAction(name, aliases, help) self._choices_actions.append(choice_action) # create the parser and add it to the map parser = self._parser_class(**kwargs) self._name_parser_map[name] = parser + + # make parser available under aliases also + for alias in aliases: + self._name_parser_map[alias] = parser + return parser def _get_subactions(self): @@ -1123,8 +1134,9 @@ def __call__(self, parser, namespace, values, option_string=None): try: parser = self._name_parser_map[parser_name] except KeyError: - tup = parser_name, ', '.join(self._name_parser_map) - msg = _('unknown parser %r (choices: %s)' % tup) + args = {'parser_name': parser_name, + 'choices': ', '.join(self._name_parser_map)} + msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args raise ArgumentError(self, msg) # parse all the remaining options into the namespace From b02faf56411c734b91a7f51b60f8921a31e12c16 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 3 Jan 2016 02:27:50 -0800 Subject: [PATCH 249/910] add/remove/list working for new config format. - mirrors.yaml now uses dict order for precedence, instead of lists of dicts. - spack.cmd now specifies default scope for add/remove and for list with `default_modify_scope` and `default_list_scope`. - commands that only read or list default to all scopes (merged) - commands that modify configs modify user scope (highest precedence) by default - These vars are used in setup_paraser for mirror/repo/compiler. - Spack's argparse supports aliases now. - added 'rm' alias for `spack [repo|compiler|mirror] remove` --- lib/spack/spack/__init__.py | 11 ++-- lib/spack/spack/cmd/__init__.py | 9 +++ lib/spack/spack/cmd/compiler.py | 17 ++++-- lib/spack/spack/cmd/mirror.py | 61 +++++++++++-------- lib/spack/spack/cmd/repo.py | 105 +++++++++++++++++++++++++++++--- lib/spack/spack/config.py | 57 ++++++----------- lib/spack/spack/repository.py | 63 +++++++++++++------ lib/spack/spack/stage.py | 3 +- 8 files changed, 219 insertions(+), 107 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 973ba64b96a..6c4a15aaabb 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -56,15 +56,12 @@ # Set up the default packages database. # import spack.repository -_repo_paths = spack.config.get_repos_config() -if not _repo_paths: - tty.die("Spack configuration contains no package repositories.") - try: - repo = spack.repository.RepoPath(*_repo_paths) + repo = spack.repository.RepoPath() sys.meta_path.append(repo) -except spack.repository.BadRepoError, e: - tty.die('Bad repository. %s' % e.message) +except spack.repository.RepoError, e: + tty.error('while initializing Spack RepoPath:') + tty.die(e.message) # # Set up the installed packages database diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 926e7ac14ab..6c635a1e6c7 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -31,6 +31,15 @@ import spack import spack.spec +import spack.config + +# +# Settings for commands that modify configuration +# +# Commands that modify confguration By default modify the *highest* priority scope. +default_modify_scope = spack.config.highest_precedence_scope().name +# Commands that list confguration list *all* scopes by default. +default_list_scope = None # cmd has a submodule called "list" so preserve the python list module python_list = list diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index a3860abf769..75b51f6b495 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -42,25 +42,31 @@ def setup_parser(subparser): sp = subparser.add_subparsers( metavar='SUBCOMMAND', dest='compiler_command') + scopes = spack.config.config_scopes + + # Add add_parser = sp.add_parser('add', help='Add compilers to the Spack configuration.') add_parser.add_argument('add_paths', nargs=argparse.REMAINDER) - add_parser.add_argument('--scope', choices=spack.config.config_scopes, + add_parser.add_argument('--scope', choices=scopes, default=spack.cmd.default_modify_scope, help="Configuration scope to modify.") - remove_parser = sp.add_parser('remove', help='Remove compiler by spec.') + # Remove + remove_parser = sp.add_parser('remove', aliases=['rm'], help='Remove compiler by spec.') remove_parser.add_argument( '-a', '--all', action='store_true', help='Remove ALL compilers that match spec.') remove_parser.add_argument('compiler_spec') - remove_parser.add_argument('--scope', choices=spack.config.config_scopes, + remove_parser.add_argument('--scope', choices=scopes, default=spack.cmd.default_modify_scope, help="Configuration scope to modify.") + # List list_parser = sp.add_parser('list', help='list available compilers') - list_parser.add_argument('--scope', choices=spack.config.config_scopes, + list_parser.add_argument('--scope', choices=scopes, default=spack.cmd.default_list_scope, help="Configuration scope to read from.") + # Info info_parser = sp.add_parser('info', help='Show compiler paths.') info_parser.add_argument('compiler_spec') - info_parser.add_argument('--scope', choices=spack.config.config_scopes, + info_parser.add_argument('--scope', choices=scopes, default=spack.cmd.default_list_scope, help="Configuration scope to read from.") @@ -132,6 +138,7 @@ def compiler_list(args): def compiler(parser, args): action = { 'add' : compiler_add, 'remove' : compiler_remove, + 'rm' : compiler_remove, 'info' : compiler_info, 'list' : compiler_list } action[args.compiler_command](args) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 946b50350ba..885483a8406 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -36,6 +36,7 @@ import spack.mirror from spack.spec import Spec from spack.error import SpackError +from spack.util.spack_yaml import syaml_dict description = "Manage mirrors." @@ -47,6 +48,7 @@ def setup_parser(subparser): sp = subparser.add_subparsers( metavar='SUBCOMMAND', dest='mirror_command') + # Create create_parser = sp.add_parser('create', help=mirror_create.__doc__) create_parser.add_argument('-d', '--directory', default=None, help="Directory in which to create mirror.") @@ -60,22 +62,29 @@ def setup_parser(subparser): '-o', '--one-version-per-spec', action='store_const', const=1, default=0, help="Only fetch one 'preferred' version per spec, not all known versions.") + scopes = spack.config.config_scopes + # Add add_parser = sp.add_parser('add', help=mirror_add.__doc__) add_parser.add_argument('name', help="Mnemonic name for mirror.") add_parser.add_argument( 'url', help="URL of mirror directory created by 'spack mirror create'.") - add_parser.add_argument('--scope', choices=spack.config.config_scopes, - help="Configuration scope to modify.") + add_parser.add_argument( + '--scope', choices=scopes, default=spack.cmd.default_modify_scope, + help="Configuration scope to modify.") - remove_parser = sp.add_parser('remove', help=mirror_remove.__doc__) + # Remove + remove_parser = sp.add_parser('remove', aliases=['rm'], help=mirror_remove.__doc__) remove_parser.add_argument('name') - remove_parser.add_argument('--scope', choices=spack.config.config_scopes, - help="Configuration scope to modify.") + remove_parser.add_argument( + '--scope', choices=scopes, default=spack.cmd.default_modify_scope, + help="Configuration scope to modify.") + # List list_parser = sp.add_parser('list', help=mirror_list.__doc__) - list_parser.add_argument('--scope', choices=spack.config.config_scopes, - help="Configuration scope to read from.") + list_parser.add_argument( + '--scope', choices=scopes, default=spack.cmd.default_list_scope, + help="Configuration scope to read from.") def mirror_add(args): @@ -86,17 +95,18 @@ def mirror_add(args): mirrors = spack.config.get_config('mirrors', scope=args.scope) if not mirrors: - mirrors = [] + mirrors = syaml_dict() - for m in mirrors: - for name, u in m.items(): - if name == args.name: - tty.die("Mirror with name %s already exists." % name) - if u == url: - tty.die("Mirror with url %s already exists." % url) - # should only be one item per mirror dict. + for name, u in mirrors.items(): + if name == args.name: + tty.die("Mirror with name %s already exists." % name) + if u == url: + tty.die("Mirror with url %s already exists." % url) + # should only be one item per mirror dict. - mirrors.insert(0, { args.name : url }) + items = [(n,u) for n,u in mirrors.items()] + items.insert(0, (args.name, url)) + mirrors = syaml_dict(items) spack.config.update_config('mirrors', mirrors, scope=args.scope) @@ -106,15 +116,14 @@ def mirror_remove(args): mirrors = spack.config.get_config('mirrors', scope=args.scope) if not mirrors: - mirrors = [] + mirrors = syaml_dict() - names = [n for m in mirrors for n,u in m.items()] - if not name in names: + if not name in mirrors: tty.die("No mirror with name %s" % name) - old_mirror = mirrors.pop(names.index(name)) + old_value = mirrors.pop(name) spack.config.update_config('mirrors', mirrors, scope=args.scope) - tty.msg("Removed mirror %s with url %s." % old_mirror.popitem()) + tty.msg("Removed mirror %s with url %s." % (name, old_value)) def mirror_list(args): @@ -124,14 +133,11 @@ def mirror_list(args): tty.msg("No mirrors configured.") return - names = [n for m in mirrors for n,u in m.items()] - max_len = max(len(n) for n in names) + max_len = max(len(n) for n in mirrors.keys()) fmt = "%%-%ds%%s" % (max_len + 4) - for m in mirrors: - for name, url in m.items(): - print fmt % (name, url) - # should only be one item per mirror dict. + for name in mirrors: + print fmt % (name, mirrors[name]) def _read_specs_from_file(filename): @@ -205,6 +211,7 @@ def mirror(parser, args): action = { 'create' : mirror_create, 'add' : mirror_add, 'remove' : mirror_remove, + 'rm' : mirror_remove, 'list' : mirror_list } action[args.mirror_command](args) diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 991d306c047..ebe42d0138e 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. @@ -33,12 +33,13 @@ import spack.spec import spack.config from spack.util.environment import get_path -from spack.repository import packages_dir_name, repo_config_name, Repo +from spack.repository import * description = "Manage package source repositories." def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='repo_command') + scopes = spack.config.config_scopes # Create create_parser = sp.add_parser('create', help=repo_create.__doc__) @@ -49,6 +50,25 @@ def setup_parser(subparser): # List list_parser = sp.add_parser('list', help=repo_list.__doc__) + list_parser.add_argument( + '--scope', choices=scopes, default=spack.cmd.default_list_scope, + help="Configuration scope to read from.") + + # Add + add_parser = sp.add_parser('add', help=repo_add.__doc__) + add_parser.add_argument('path', help="Path to a Spack package repository directory.") + add_parser.add_argument( + '--scope', choices=scopes, default=spack.cmd.default_modify_scope, + help="Configuration scope to modify.") + + # Remove + remove_parser = sp.add_parser('remove', help=repo_remove.__doc__, aliases=['rm']) + remove_parser.add_argument( + 'path_or_namespace', + help="Path or namespace of a Spack package repository.") + remove_parser.add_argument( + '--scope', choices=scopes, default=spack.cmd.default_modify_scope, + help="Configuration scope to modify.") def repo_create(args): @@ -104,25 +124,87 @@ def repo_create(args): def repo_add(args): - """Remove a package source from the Spack configuration""" - # FIXME: how to deal with this with the current config architecture? - # FIXME: Repos do not have mnemonics, which I assumed would be simpler... should they have them after all? + """Add a package source to the Spack configuration""" + path = args.path + + # check if the path is relative to the spack directory. + real_path = path + if path.startswith('$spack'): + real_path = spack.repository.substitute_spack_prefix(path) + elif not os.path.isabs(real_path): + real_path = os.path.abspath(real_path) + path = real_path + + # check if the path exists + if not os.path.exists(real_path): + tty.die("No such file or directory: '%s'." % path) + + # Make sure the path is a directory. + if not os.path.isdir(real_path): + tty.die("Not a Spack repository: '%s'." % path) + + # Make sure it's actually a spack repository by constructing it. + repo = Repo(real_path) + + # If that succeeds, finally add it to the configuration. + repos = spack.config.get_config('repos', args.scope) + if not repos: repos = [] + + if repo.root in repos or path in repos: + tty.die("Repository is already registered with Spack: '%s'" % path) + + repos.insert(0, path) + spack.config.update_config('repos', repos, args.scope) + tty.msg("Created repo with namespace '%s'." % repo.namespace) def repo_remove(args): - """Remove a package source from the Spack configuration""" - # FIXME: see above. + """Remove a repository from the Spack configuration.""" + repos = spack.config.get_config('repos', args.scope) + path_or_namespace = args.path_or_namespace + + # If the argument is a path, remove that repository from config. + path = os.path.abspath(path_or_namespace) + if path in repos: + repos.remove(path) + spack.config.update_config('repos', repos, args.scope) + tty.msg("Removed repository '%s'." % path) + return + + # If it is a namespace, remove corresponding repo + for path in repos: + try: + repo = Repo(path) + if repo.namespace == path_or_namespace: + repos.remove(repo.root) + spack.config.update_config('repos', repos, args.scope) + tty.msg("Removed repository '%s' with namespace %s." + % (repo.root, repo.namespace)) + return + except RepoError as e: + continue + + tty.die("No repository with path or namespace: '%s'" + % path_or_namespace) def repo_list(args): """List package sources and their mnemoics""" - roots = spack.config.get_repos_config() - repos = [Repo(r) for r in roots] + roots = spack.config.get_config('repos', args.scope) + repos = [] + for r in roots: + try: + repos.append(Repo(r)) + except RepoError as e: + continue msg = "%d package repositor" % len(repos) msg += "y." if len(repos) == 1 else "ies." tty.msg(msg) + if not repos: + return + max_ns_len = max(len(r.namespace) for r in repos) for repo in repos: fmt = "%%-%ds%%s" % (max_ns_len + 4) @@ -131,5 +213,8 @@ def repo_list(args): def repo(parser, args): action = { 'create' : repo_create, - 'list' : repo_list } + 'list' : repo_list, + 'add' : repo_add, + 'remove' : repo_remove, + 'rm' : repo_remove} action[args.repo_command](args) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index d266d2e23ff..c53dcbc4050 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -204,6 +204,11 @@ def clear(self): ConfigScope('user', os.path.expanduser('~/.spack')) +def highest_precedence_scope(): + """Get the scope with highest precedence (prefs will override others).""" + return config_scopes.values()[-1] + + def validate_scope(scope): """Ensure that scope is valid, and return a valid scope if it is None. @@ -214,7 +219,7 @@ def validate_scope(scope): """ if scope is None: # default to the scope with highest precedence. - return config_scopes.values()[-1] + return highest_precedence_scope() elif scope in config_scopes: return config_scopes[scope] @@ -287,15 +292,10 @@ def they_are(t): dest[:] = source + [x for x in dest if x not in seen] return dest - # Source dict is merged into dest. Extra ':' means overwrite. + # Source dict is merged into dest. elif they_are(dict): for sk, sv in source.iteritems(): - # allow total override with, e.g., repos:: - override = sk.endswith(':') - if override: - sk = sk.rstrip(':') - - if override or not sk in dest: + if not sk in dest: dest[sk] = copy.copy(sv) else: dest[sk] = _merge_yaml(dest[sk], source[sk]) @@ -306,18 +306,13 @@ def they_are(t): return copy.copy(source) -def substitute_spack_prefix(path): - """Replaces instances of $spack with Spack's prefix.""" - return path.replace('$spack', spack.prefix) - - def get_config(section, scope=None): """Get configuration settings for a section. Strips off the top-level section name from the YAML dict. """ validate_section(section) - merged_section = {} + merged_section = syaml.syaml_dict() if scope is None: scopes = config_scopes.values() @@ -327,37 +322,25 @@ def get_config(section, scope=None): for scope in scopes: # read potentially cached data from the scope. data = scope.get_section(section) - if not data or not section in data: + + # Skip empty configs + if not data or not isinstance(data, dict): continue - # extract data under the section name header - data = data[section] - - # ignore empty sections for easy commenting of single-line configs. - if not data: + # Allow complete override of site config with '
::' + override_key = section + ':' + if not (section in data or override_key in data): + tty.warn("Skipping bad configuration file: '%s'" % scope.path) continue - # merge config data from scopes. - merged_section = _merge_yaml(merged_section, data) + if override_key in data: + merged_section = data[override_key] + else: + merged_section = _merge_yaml(merged_section, data[section]) return merged_section -def get_repos_config(): - repo_list = get_config('repos') - if repo_list is None: - return [] - - if not isinstance(repo_list, list): - tty.die("Bad repository configuration. 'repos' element does not contain a list.") - - def expand_repo_path(path): - path = substitute_spack_prefix(path) - path = os.path.expanduser(path) - return path - return [expand_repo_path(repo) for repo in repo_list] - - def get_config_filename(scope, section): """For some scope and section, get the name of the configuration file""" scope = validate_scope(scope) diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 4e91855db0c..3367572ef50 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -36,6 +36,7 @@ from llnl.util.filesystem import join_path import spack.error +import spack.config import spack.spec from spack.virtual import ProviderIndex from spack.util.naming import * @@ -53,6 +54,7 @@ packages_dir_name = 'packages' # Top-level repo directory containing pkgs. package_file_name = 'package.py' # Filename for packages in a repository. + def _autospec(function): """Decorator that automatically converts the argument of a single-arg function to a Spec.""" @@ -71,6 +73,11 @@ def _make_namespace_module(ns): return module +def substitute_spack_prefix(path): + """Replaces instances of $spack with Spack's prefix.""" + return path.replace('$spack', spack.prefix) + + class RepoPath(object): """A RepoPath is a list of repos that function as one. @@ -89,10 +96,20 @@ def __init__(self, *repo_dirs, **kwargs): self._all_package_names = [] self._provider_index = None + # If repo_dirs is empty, just use the configuration + if not repo_dirs: + repo_dirs = spack.config.get_config('repos') + if not repo_dirs: + raise NoRepoConfiguredError( + "Spack configuration contains no package repositories.") + # Add each repo to this path. for root in repo_dirs: - repo = Repo(root, self.super_namespace) - self.put_last(repo) + try: + repo = Repo(root, self.super_namespace) + self.put_last(repo) + except RepoError as e: + tty.warn("Failed to initialize repository at '%s'." % root, e.message) def swap(self, other): @@ -121,12 +138,12 @@ def _add(self, repo): """ if repo.root in self.by_path: - raise DuplicateRepoError("Package repos are the same", - repo, self.by_path[repo.root]) + raise DuplicateRepoError("Duplicate repository: '%s'" % repo.root) if repo.namespace in self.by_namespace: - raise DuplicateRepoError("Package repos cannot provide the same namespace", - repo, self.by_namespace[repo.namespace]) + raise DuplicateRepoError( + "Package repos '%s' and '%s' both provide namespace %s." + % (repo.root, self.by_namespace[repo.namespace].root, repo.namespace)) # Add repo to the pkg indexes self.by_namespace[repo.full_namespace] = repo @@ -292,7 +309,8 @@ def __init__(self, root, namespace=repo_namespace): """ # Root directory, containing _repo.yaml and package dirs - self.root = root + # Allow roots to by spack-relative by starting with '$spack' + self.root = substitute_spack_prefix(root) # super-namespace for all packages in the Repo self.super_namespace = namespace @@ -629,13 +647,27 @@ def __contains__(self, pkg_name): return self.exists(pkg_name) -class BadRepoError(spack.error.SpackError): +class RepoError(spack.error.SpackError): + """Superclass for repository-related errors.""" + + +class NoRepoConfiguredError(RepoError): + """Raised when there are no repositories configured.""" + + +class BadRepoError(RepoError): """Raised when repo layout is invalid.""" - def __init__(self, msg): - super(BadRepoError, self).__init__(msg) -class UnknownPackageError(spack.error.SpackError): +class DuplicateRepoError(RepoError): + """Raised when duplicate repos are added to a RepoPath.""" + + +class PackageLoadError(spack.error.SpackError): + """Superclass for errors related to loading packages.""" + + +class UnknownPackageError(PackageLoadError): """Raised when we encounter a package spack doesn't have.""" def __init__(self, name, repo=None): msg = None @@ -647,14 +679,7 @@ def __init__(self, name, repo=None): self.name = name -class DuplicateRepoError(spack.error.SpackError): - """Raised when duplicate repos are added to a RepoPath.""" - def __init__(self, msg, repo1, repo2): - super(UnknownPackageError, self).__init__( - "%s: %s, %s" % (msg, repo1, repo2)) - - -class FailedConstructorError(spack.error.SpackError): +class FailedConstructorError(PackageLoadError): """Raised when a package's class constructor fails.""" def __init__(self, name, exc_type, exc_obj, exc_tb): super(FailedConstructorError, self).__init__( diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index a9631d4b620..61f98468352 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -244,8 +244,7 @@ def fetch(self): # TODO: move mirror logic out of here and clean it up! if self.mirror_path: mirrors = spack.config.get_config('mirrors') - mirrors = [(n,u) for m in mirrors for n,u in m.items()] - urls = [urljoin(u, self.mirror_path) for name, u in mirrors] + urls = [urljoin(u, self.mirror_path) for name, u in mirrors.items()] digest = None if isinstance(self.fetcher, fs.URLFetchStrategy): From 884f317d50344101246f2892af492663a45be261 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Sun, 3 Jan 2016 17:22:13 -0800 Subject: [PATCH 250/910] tweak gold support, added to gcc The gold linker support and gold plugin variants now use the same name. Trying to apply use-flag-style discipline here despite the fact gold has other implications for clang, this way globally enabling gold will have a more consistent effect if that becomes possible. The gold support in gcc could use more testing to ensure it works consistently, but as long as a binutils including gold is used the gcc configure tends to pick it up, and it seems to work with 5.3.0 at least. --- var/spack/packages/gcc/package.py | 5 ++++- var/spack/packages/llvm/package.py | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py index 7ec160d5957..61b16f3fd8d 100644 --- a/var/spack/packages/gcc/package.py +++ b/var/spack/packages/gcc/package.py @@ -48,11 +48,14 @@ class Gcc(Package): version('4.7.4', '4c696da46297de6ae77a82797d2abe28') version('4.6.4', 'b407a3d1480c11667f293bfb1f17d1a4') version('4.5.4', '27e459c2566b8209ab064570e1b378f7') + + variant('gold', default=True, description="Build the gold linker plugin for ld-based LTO") depends_on("mpfr") depends_on("gmp") depends_on("mpc") # when @4.5: - depends_on("binutils~libiberty") + depends_on("binutils~libiberty", when='~gold') + depends_on("binutils~libiberty+gold", when='+gold') # Save these until we can do optional deps. depends_on("isl", when=DEPENDS_ON_ISL_PREDICATE) diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index 19d82d81b71..10e899437fb 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -45,7 +45,7 @@ class Llvm(Package): variant('polly', default=True, description="Build the LLVM polyhedral optimization plugin, only builds for 3.7.0+") variant('libcxx', default=True, description="Build the LLVM C++ standard library") variant('compiler-rt', default=True, description="Build the LLVM compiler runtime, including sanitizers") - variant('lto', default=True, description="Add support for LTO with the gold linker plugin") + variant('gold', default=True, description="Add support for LTO with the gold linker plugin") # Universal dependency @@ -57,7 +57,7 @@ class Llvm(Package): depends_on('libedit', when='+lldb') # gold support - depends_on('binutils+gold', when='+lto') + depends_on('binutils+gold', when='+gold') # polly plugin depends_on('gmp', when='+polly') @@ -180,7 +180,7 @@ def install(self, spec, prefix): '-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp', '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix ]) - if '+lto' in spec: + if '+gold' in spec: cmake_args.append('-DLLVM_BINUTILS_INCDIR=' + os.path.join( spec['binutils'].prefix, 'include')) if '+polly' in spec: cmake_args.append('-DLINK_POLLY_INTO_TOOLS:Bool=ON') From 96e5905bce302007e8be6e3debf6d3efdf77a8cd Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 4 Jan 2016 12:14:42 -0500 Subject: [PATCH 251/910] Build multiple FFTW variants simultaneously FFTW can build only one floating point precision (float, double, long double, quad) at once, but they can all be installed simultaneously as the libraries have different names. It is common packages to decide only at run time which precision FFTW they need, and thus FFTW should offer all precisions at once. --- var/spack/packages/fftw/package.py | 55 +++++++++--------------------- 1 file changed, 17 insertions(+), 38 deletions(-) diff --git a/var/spack/packages/fftw/package.py b/var/spack/packages/fftw/package.py index 5f71762c4ff..4d2b9642426 100644 --- a/var/spack/packages/fftw/package.py +++ b/var/spack/packages/fftw/package.py @@ -39,54 +39,21 @@ class Fftw(Package): version('3.3.4', '2edab8c06b24feeb3b82bbb3ebf3e7b3') - ########## - # Floating point precision - FLOAT = 'float' - LONG_DOUBLE = 'long_double' - QUAD_PRECISION = 'quad' - PRECISION_OPTIONS = { - FLOAT: '--enable-float', - LONG_DOUBLE: '--enable--long-double', - QUAD_PRECISION: '--enable-quad-precision' - } - variant(FLOAT, default=False, description='Produces a single precision version of the library') - variant(LONG_DOUBLE, default=False, description='Produces a long double precision version of the library') - variant(QUAD_PRECISION, default=False, description='Produces a quad precision version of the library (works only with GCC and libquadmath)') - ########## + variant('float', default=True, description='Produces a single precision version of the library') + variant('long_double', default=True, description='Produces a long double precision version of the library') + variant('quad', default=False, description='Produces a quad precision version of the library (works only with GCC and libquadmath)') variant('mpi', default=False, description='Activate MPI support') depends_on('mpi', when='+mpi') - @staticmethod - def enabled(x): - """ - Given a variant name returns the string that means the variant is enabled - - :param x: variant name - """ - # FIXME : duplicated from MVAPICH2 - return '+' + x - - def check_fortran_availability(self, options): - if not self.compiler.f77 or not self.compiler.fc: - options.append("--disable-fortran") - - def set_floating_point_precision(self, spec, options): - l = [option for variant, option in Fftw.PRECISION_OPTIONS.iteritems() if self.enabled(variant) in spec] - if len(l) > 1: - raise RuntimeError('At most one floating point precision variant may activated per build.') - options.extend(l) - def install(self, spec, prefix): - options = ['--prefix=%s' % prefix, '--enable-shared', '--enable-threads', '--enable-openmp'] - self.check_fortran_availability(options) - self.set_floating_point_precision(spec, options) - + if not self.compiler.f77 or not self.compiler.fc: + options.append("--disable-fortran") if '+mpi' in spec: options.append('--enable-mpi') @@ -94,3 +61,15 @@ def install(self, spec, prefix): make() make("install") + if '+float' in spec: + configure('--enable-float', *options) + make() + make("install") + if '+long_double' in spec: + configure('--enable-long-double', *options) + make() + make("install") + if '+quad' in spec: + configure('--enable-quad-precision', *options) + make() + make("install") From 24ced90913d7b819d0bec5375ab6cc0c54ceecd8 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 5 Jan 2016 09:40:30 -0500 Subject: [PATCH 252/910] Correct `join_path` in package boost --- var/spack/packages/boost/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index 9b7c22c53d9..3427b74ad68 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -95,11 +95,11 @@ def determine_bootstrap_options(self, spec, options): with open('user-config.jam', 'w') as f: if '+mpi' in spec: f.write('using mpi : %s ;\n' % - joinpath(spec['mpi'].prefix.bin, 'mpicxx')) + join_path(spec['mpi'].prefix.bin, 'mpicxx')) if '+python' in spec: f.write('using python : %s : %s ;\n' % (spec['python'].version, - joinpath(spec['python'].prefix.bin, 'python'))) + join_path(spec['python'].prefix.bin, 'python'))) def determine_b2_options(self, spec, options): if '+debug' in spec: From 12632f16b8a5f51fd35ad81c9a8625fe1e30cd7c Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 5 Jan 2016 13:17:17 -0500 Subject: [PATCH 253/910] Combine with other OpenSSL Darwin patch Set KERNEL_BITS. Don't use Darwin-specific configure script. --- var/spack/packages/openssl/package.py | 29 +++++++++++---------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index e1039fcbe12..779c31af120 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -17,23 +17,18 @@ class Openssl(Package): parallel = False def install(self, spec, prefix): - if spec.satisfies("=darwin-x86_64"): - perl = which('perl') - perl("./Configure", - "--prefix=%s" % prefix, - "--openssldir=%s/etc/openssl" % prefix, - "zlib", - "no-krb5", - "shared", - "darwin64-x86_64-cc") - filter_file(r'-arch x86_64', '', 'Makefile') - else: - config = Executable("./config") - config("--prefix=%s" % prefix, - "--openssldir=%s/etc/openssl" % prefix, - "zlib", - "no-krb5", - "shared") + if spec.satisfies("=darwin-x86_64") or spec.satisfies("=ppc64"): + # This needs to be done for all 64-bit architectures (except Linux, + # where it happens automatically?) + env['KERNEL_BITS'] = '64' + config = Executable("./config") + config("--prefix=%s" % prefix, + "--openssldir=%s" % join_path(prefix, 'etc', 'openssl'), + "zlib", + "no-krb5", + "shared") + # Remove non-standard compiler options if present + filter_file(r'-arch x86_64', '', 'Makefile') make() make("install") From c18dfa058a208627063d069566a728d8715f984c Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 5 Jan 2016 14:09:54 -0500 Subject: [PATCH 254/910] Add details to comment --- var/spack/packages/openssl/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index 779c31af120..a24de20cc37 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -27,7 +27,9 @@ def install(self, spec, prefix): "zlib", "no-krb5", "shared") - # Remove non-standard compiler options if present + # Remove non-standard compiler options if present. These options are + # present e.g. on Darwin. They are non-standard, i.e. most compilers + # (e.g. gcc) will not accept them. filter_file(r'-arch x86_64', '', 'Makefile') make() From 2b801e20866152055f1cb4845b22da66a0020968 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Tue, 22 Dec 2015 20:24:17 -0500 Subject: [PATCH 255/910] new package: py-astropy --- var/spack/packages/py-astropy/package.py | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 var/spack/packages/py-astropy/package.py diff --git a/var/spack/packages/py-astropy/package.py b/var/spack/packages/py-astropy/package.py new file mode 100644 index 00000000000..d138a514f68 --- /dev/null +++ b/var/spack/packages/py-astropy/package.py @@ -0,0 +1,28 @@ +from spack import * + +class PyAstropy(Package): + """ + The Astropy Project is a community effort to develop a single core + package for Astronomy in Python and foster interoperability between + Python astronomy packages. + """ + homepage = 'http://www.astropy.org/' + + version('1.1.post1', 'b52919f657a37d45cc45f5cb0f58c44d') + + def url_for_version(self, version): + return 'https://pypi.python.org/packages/source/a/astropy/astropy-{0}.tar.gz'.format(version) + + extends('python') + + depends_on('cfitsio') + depends_on('expat') + depends_on('py-h5py') + depends_on('py-numpy') + depends_on('py-scipy') + + def install(self, spec, prefix): + python('setup.py', 'build', '--use-system-cfitsio', + '--use-system-expat') + python('setup.py', 'install', '--prefix=' + prefix) + From 60d8413940119c64db89ded7854850912947e135 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 5 Jan 2016 19:21:57 -0500 Subject: [PATCH 256/910] Support mbedtls, an alternative SSL library --- var/spack/packages/mbedtls/package.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 var/spack/packages/mbedtls/package.py diff --git a/var/spack/packages/mbedtls/package.py b/var/spack/packages/mbedtls/package.py new file mode 100644 index 00000000000..77455229568 --- /dev/null +++ b/var/spack/packages/mbedtls/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Mbedtls(Package): + """ + mbed TLS (formerly known as PolarSSL) makes it trivially easy for developers to include cryptographic and SSL/TLS capabilities in their (embedded) products, facilitating this functionality with a minimal coding footprint. + """ + homepage = "https://tls.mbed.org" + url = "https://github.com/ARMmbed/mbedtls/archive/mbedtls-2.2.1.tar.gz" + + version('2.2.1' , '73a38f96898d6d03e32f55dd9f9a67be') + version('2.2.0' , 'eaf4586c1ef93ae872e606b6c1203942') + version('2.1.4' , '40cdf67b6c6d92c9cbcfd552d39ea3ae') + version('2.1.3' , '7eb4cf1dfa68578a2c8dbd0b6fa752dd') + version('1.3.16', '4144d7320c691f721aeb9e67a1bc38e0') + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") From cfe54b201d0b563214f068c93c05acad116c7361 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 5 Jan 2016 20:09:22 -0500 Subject: [PATCH 257/910] New package atop --- var/spack/packages/atop/package.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 var/spack/packages/atop/package.py diff --git a/var/spack/packages/atop/package.py b/var/spack/packages/atop/package.py new file mode 100644 index 00000000000..83894477f40 --- /dev/null +++ b/var/spack/packages/atop/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Atop(Package): + """Atop is an ASCII full-screen performance monitor for Linux""" + homepage = "http://www.atoptool.nl/index.php" + url = "http://www.atoptool.nl/download/atop-2.2-3.tar.gz" + + version('2.2-3', '034dc1544f2ec4e4d2c739d320dc326d') + + def install(self, spec, prefix): + make() + mkdirp(prefix.bin) + install("atop", join_path(prefix.bin, "atop")) + mkdirp(join_path(prefix.man, "man1")) + install(join_path("man", "atop.1"), + join_path(prefix.man, "man1", "atop.1")) From 24ba60168d8c0f7a681deca81c4a9bb8ea4cd5d1 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 6 Jan 2016 09:49:10 +0100 Subject: [PATCH 258/910] added package : glpk --- var/spack/packages/glpk/package.py | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 var/spack/packages/glpk/package.py diff --git a/var/spack/packages/glpk/package.py b/var/spack/packages/glpk/package.py new file mode 100644 index 00000000000..ab1eb1983d6 --- /dev/null +++ b/var/spack/packages/glpk/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Glpk(Package): + """ + The GLPK (GNU Linear Programming Kit) package is intended for solving large-scale linear programming (LP), mixed + integer programming (MIP), and other related problems. It is a set of routines written in ANSI C and organized in + the form of a callable library + """ + homepage = "https://www.gnu.org/software/glpk" + url = "http://ftp.gnu.org/gnu/glpk/glpk-4.57.tar.gz" + + version('4.57', '237531a54f73155842f8defe51aedb0f') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") From 7670dc7efcee7b16c7b08b3c2ba6669bcbbcf49b Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 6 Jan 2016 10:05:48 +0100 Subject: [PATCH 259/910] glpk : added support for gmp --- var/spack/packages/glpk/package.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/glpk/package.py b/var/spack/packages/glpk/package.py index ab1eb1983d6..855f459fb38 100644 --- a/var/spack/packages/glpk/package.py +++ b/var/spack/packages/glpk/package.py @@ -37,7 +37,17 @@ class Glpk(Package): version('4.57', '237531a54f73155842f8defe51aedb0f') + variant('gmp', default=False, description='Activates support for GMP library') + + depends_on('gmp', when='+gmp') + def install(self, spec, prefix): - configure('--prefix=%s' % prefix) + + options = ['--prefix=%s' % prefix] + + if '+gmp' in spec: + options.append('--with-gmp') + + configure(*options) make() make("install") From fd7c1fe58481a211267a67e5ec3c08499dab8b10 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 6 Jan 2016 10:10:15 -0500 Subject: [PATCH 260/910] Avoid using tabs --- var/spack/packages/atop/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/atop/package.py b/var/spack/packages/atop/package.py index 83894477f40..346ab0763c2 100644 --- a/var/spack/packages/atop/package.py +++ b/var/spack/packages/atop/package.py @@ -12,5 +12,5 @@ def install(self, spec, prefix): mkdirp(prefix.bin) install("atop", join_path(prefix.bin, "atop")) mkdirp(join_path(prefix.man, "man1")) - install(join_path("man", "atop.1"), - join_path(prefix.man, "man1", "atop.1")) + install(join_path("man", "atop.1"), + join_path(prefix.man, "man1", "atop.1")) From e7901466273884b680d2575d54561bd73d0c73b4 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Tue, 5 Jan 2016 17:53:55 -0500 Subject: [PATCH 261/910] paraview: add some extra arguments on OS X X shouldn't be used on OS X and spack expects a Unix-style install, not a .app bundle. --- var/spack/packages/paraview/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/packages/paraview/package.py b/var/spack/packages/paraview/package.py index 1d99b348997..aaab352e662 100644 --- a/var/spack/packages/paraview/package.py +++ b/var/spack/packages/paraview/package.py @@ -60,6 +60,10 @@ def nfeature_to_bool(feature): feature_args.extend(std_cmake_args) + if 'darwin' in self.spec.architecture: + feature_args.append('-DVTK_USE_X:BOOL=OFF') + feature_args.append('-DPARAVIEW_DO_UNIX_STYLE_INSTALLS:BOOL=ON') + cmake('..', '-DCMAKE_INSTALL_PREFIX:PATH=%s' % prefix, '-DBUILD_TESTING:BOOL=OFF', From 1646bec1a4fd612abcb408dc169dd189b14756ac Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 7 Jan 2016 13:54:47 -0500 Subject: [PATCH 262/910] Make PETSc build with OpenMPI --- var/spack/packages/petsc/package.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/var/spack/packages/petsc/package.py b/var/spack/packages/petsc/package.py index f3ed3d72ec5..43c7f8a9b6c 100644 --- a/var/spack/packages/petsc/package.py +++ b/var/spack/packages/petsc/package.py @@ -23,9 +23,6 @@ class Petsc(Package): def install(self, spec, prefix): configure("--prefix=%s" % prefix, - "CC=cc", - "CXX=c++", - "FC=f90", "--with-blas-lib=%s/libblas.a" % spec['blas'].prefix.lib, "--with-lapack-lib=%s/liblapack.a" % spec['lapack'].prefix.lib, "--with-boost-dir=%s" % spec['boost'].prefix, @@ -33,6 +30,7 @@ def install(self, spec, prefix): "--with-parmetis-dir=%s" % spec['parmetis'].prefix, "--with-metis-dir=%s" % spec['metis'].prefix, "--with-hdf5-dir=%s" % spec['hdf5'].prefix, + "--with-mpi-dir=%s" % spec['mpi'].prefix, "--with-shared-libraries=0") # PETSc has its own way of doing parallel make. From 175f20a9dce00bf2129923aae50abe36fdd25bf4 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Thu, 7 Jan 2016 11:12:47 -0800 Subject: [PATCH 263/910] new and updated python packages --- var/spack/packages/py-funcsigs/package.py | 19 +++++++++++++++++++ var/spack/packages/py-matplotlib/package.py | 5 ++++- var/spack/packages/py-mock/package.py | 1 + var/spack/packages/py-pbr/package.py | 18 ++++++++++++++++++ .../packages/py-periodictable/package.py | 17 +++++++++++++++++ var/spack/packages/py-setuptools/package.py | 1 + 6 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 var/spack/packages/py-funcsigs/package.py create mode 100644 var/spack/packages/py-pbr/package.py create mode 100644 var/spack/packages/py-periodictable/package.py diff --git a/var/spack/packages/py-funcsigs/package.py b/var/spack/packages/py-funcsigs/package.py new file mode 100644 index 00000000000..a4288902887 --- /dev/null +++ b/var/spack/packages/py-funcsigs/package.py @@ -0,0 +1,19 @@ +from spack import * +import os + +class PyFuncsigs(Package): + """Python function signatures from PEP362 for Python 2.6, 2.7 and 3.2.""" + homepage = "https://pypi.python.org/pypi/funcsigs" + url = "https://pypi.python.org/packages/source/f/funcsigs/funcsigs-0.4.tar.gz" + + version('0.4', 'fb1d031f284233e09701f6db1281c2a5') + + extends('python') + + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + + + diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py index a5fee39d42e..4776c581ee6 100644 --- a/var/spack/packages/py-matplotlib/package.py +++ b/var/spack/packages/py-matplotlib/package.py @@ -12,7 +12,7 @@ class PyMatplotlib(Package): variant('gui', default=False, description='Enable GUI') variant('ipython', default=False, description='Enable ipython support') - extends('python', ignore=r'bin/nosetests.*$') + extends('python', ignore=r'bin/nosetests.*$|bin/pbr$') depends_on('py-pyside', when='+gui') depends_on('py-ipython', when='+ipython') @@ -22,6 +22,9 @@ class PyMatplotlib(Package): depends_on('py-pytz') depends_on('py-nose') depends_on('py-numpy') + depends_on('py-mock') + depends_on('py-pbr') + depends_on('py-funcsigs') depends_on('qt', when='+gui') depends_on('bzip2') diff --git a/var/spack/packages/py-mock/package.py b/var/spack/packages/py-mock/package.py index 3b08428ba0c..e89af8802aa 100644 --- a/var/spack/packages/py-mock/package.py +++ b/var/spack/packages/py-mock/package.py @@ -11,6 +11,7 @@ class PyMock(Package): version('1.3.0', '73ee8a4afb3ff4da1b4afa287f39fdeb') extends('python') + depends_on('py-pbr') depends_on('py-setuptools@17.1:') def install(self, spec, prefix): diff --git a/var/spack/packages/py-pbr/package.py b/var/spack/packages/py-pbr/package.py new file mode 100644 index 00000000000..02957483d4e --- /dev/null +++ b/var/spack/packages/py-pbr/package.py @@ -0,0 +1,18 @@ +from spack import * +import os + +class PyPbr(Package): + """PBR is a library that injects some useful and sensible default behaviors into your setuptools run.""" + homepage = "https://pypi.python.org/pypi/pbr" + url = "https://pypi.python.org/packages/source/p/pbr/pbr-1.8.1.tar.gz" + + version('1.8.1', 'c8f9285e1a4ca6f9654c529b158baa3a') + + extends('python') + + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + + diff --git a/var/spack/packages/py-periodictable/package.py b/var/spack/packages/py-periodictable/package.py new file mode 100644 index 00000000000..6a495a1cc83 --- /dev/null +++ b/var/spack/packages/py-periodictable/package.py @@ -0,0 +1,17 @@ +from spack import * + +class PyPeriodictable(Package): + """nose extends the test loading and running features of unittest, + making it easier to write, find and run tests.""" + + homepage = "https://pypi.python.org/pypi/periodictable" + url = "https://pypi.python.org/packages/source/p/periodictable/periodictable-1.4.1.tar.gz" + + version('1.4.1', '7246b63cc0b6b1be6e86b6616f9e866e') + + depends_on('py-numpy') + depends_on('py-pyparsing') + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-setuptools/package.py b/var/spack/packages/py-setuptools/package.py index 760ad4d6dbf..26c048bfd42 100644 --- a/var/spack/packages/py-setuptools/package.py +++ b/var/spack/packages/py-setuptools/package.py @@ -8,6 +8,7 @@ class PySetuptools(Package): version('11.3.1', '01f69212e019a2420c1693fb43593930') version('16.0', '0ace0b96233516fc5f7c857d086aa3ad') version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06') + version('19.2', '78353b1f80375ca5e088f4b4627ffe03') extends('python') From 7ad3e2b56bca4aca3a1ed5a157380070ea17514f Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Fri, 8 Jan 2016 11:09:11 -0800 Subject: [PATCH 264/910] new python packages --- var/spack/packages/py-numexpr/package.py | 15 +++++++++++++++ var/spack/packages/py-pytables/package.py | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 var/spack/packages/py-numexpr/package.py create mode 100644 var/spack/packages/py-pytables/package.py diff --git a/var/spack/packages/py-numexpr/package.py b/var/spack/packages/py-numexpr/package.py new file mode 100644 index 00000000000..89f8a525b12 --- /dev/null +++ b/var/spack/packages/py-numexpr/package.py @@ -0,0 +1,15 @@ +from spack import * +import re + +class PyNumexpr(Package): + """Fast numerical expression evaluator for NumPy""" + homepage = "https://pypi.python.org/pypi/numexpr" + url = "https://pypi.python.org/packages/source/n/numexpr/numexpr-2.4.6.tar.gz" + + version('2.4.6', '17ac6fafc9ea1ce3eb970b9abccb4fbd') + + extends('python') + depends_on('py-numpy') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pytables/package.py b/var/spack/packages/py-pytables/package.py new file mode 100644 index 00000000000..a5b1e78ab3b --- /dev/null +++ b/var/spack/packages/py-pytables/package.py @@ -0,0 +1,19 @@ +from spack import * +import re + +class PyPytables(Package): + """PyTables is a package for managing hierarchical datasets and designed to efficiently and easily cope with extremely large amounts of data.""" + homepage = "http://www.pytables.org/" + url = "https://github.com/PyTables/PyTables/archive/v.3.2.2.tar.gz" + + version('3.2.2', '7cbb0972e4d6580f629996a5bed92441') + + extends('python') + depends_on('hdf5') + depends_on('py-numpy') + depends_on('py-numexpr') + depends_on('py-cython') + + def install(self, spec, prefix): + env["HDF5_DIR"] = spec['hdf5'].prefix + python('setup.py', 'install', '--prefix=%s' % prefix) From 6214148d35845f2a15d23b4c197aa4cc91ebe306 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 8 Jan 2016 17:16:30 -0500 Subject: [PATCH 265/910] Use .tar.gz instead of .tar.xz for backward compatibility --- var/spack/packages/gdb/package.py | 12 ++++++------ var/spack/packages/git/package.py | 14 +++++++------- var/spack/packages/python/package.py | 13 +++++++------ var/spack/packages/texinfo/package.py | 10 +++++----- var/spack/packages/wget/package.py | 5 +++-- 5 files changed, 28 insertions(+), 26 deletions(-) diff --git a/var/spack/packages/gdb/package.py b/var/spack/packages/gdb/package.py index fd567f346b4..85ed3215e26 100644 --- a/var/spack/packages/gdb/package.py +++ b/var/spack/packages/gdb/package.py @@ -32,13 +32,13 @@ class Gdb(Package): -- or what another program was doing at the moment it crashed. """ homepage = "https://www.gnu.org/software/gdb" - url = "http://ftp.gnu.org/gnu/gdb/gdb-7.10.tar.xz" + url = "http://ftp.gnu.org/gnu/gdb/gdb-7.10.tar.gz" - version('7.10.1', '39e654460c9cdd80200a29ac020cfe11') - version('7.10', '2a35bac41fa8e10bf04f3a0dd7f7f363') - version('7.9.1', '35374c77a70884eb430c97061053a36e') - version('7.9', 'e6279f26559d839f0b4218a482bcb43e') - version('7.8.2', 'a80cf252ed2e775d4e4533341bbf2459') + version('7.10.1', 'b93a2721393e5fa226375b42d567d90b') + version('7.10', 'fa6827ad0fd2be1daa418abb11a54d86') + version('7.9.1', 'f3b97de919a9dba84490b2e076ec4cb0') + version('7.9', '8f8ced422fe462a00e0135a643544f17') + version('7.8.2', '8b0ea8b3559d3d90b3ff4952f0aeafbc') depends_on('texinfo') diff --git a/var/spack/packages/git/package.py b/var/spack/packages/git/package.py index 28c7aa8161f..ddc5078c4d3 100644 --- a/var/spack/packages/git/package.py +++ b/var/spack/packages/git/package.py @@ -5,14 +5,14 @@ class Git(Package): system designed to handle everything from small to very large projects with speed and efficiency.""" homepage = "http://git-scm.com" - url = "https://www.kernel.org/pub/software/scm/git/git-2.2.1.tar.xz" + url = "https://www.kernel.org/pub/software/scm/git/git-2.2.1.tar.gz" - version('2.6.3', '5a6375349c3f13c8dbbabfc327bae429') - version('2.6.2', '32ae5ad29763fc927bfcaeab55385fd9') - version('2.6.1', 'dd4a3a7fe96598c553edd39d40c9c290') - version('2.6.0', '6b7d43d615fb3f0dfecf4d131e23f438') - version('2.5.4', 'ec118fcd1cf984edc17eb6588b78e81b') - version('2.2.1', '43e01f9d96ba8c11611e0eef0d9f9f28') + version('2.6.3', 'b711be7628a4a2c25f38d859ee81b423') + version('2.6.2', 'da293290da69f45a86a311ad3cd43dc8') + version('2.6.1', '4c62ee9c5991fe93d99cf2a6b68397fd') + version('2.6.0', 'eb76a07148d94802a1745d759716a57e') + version('2.5.4', '3eca2390cf1fa698b48e2a233563a76b') + version('2.2.1', 'ff41fdb094eed1ec430aed8ee9b9849c') # Git compiles with curl support by default on but if your system diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index eae2566b573..a1ce06feb03 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -11,15 +11,16 @@ class Python(Package): """The Python programming language.""" homepage = "http://www.python.org" - url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tar.xz" + url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz" extendable = True - version('2.7.8', 'd235bdfa75b8396942e360a70487ee00') - version('2.7.10', 'c685ef0b8e9f27b5e3db5db12b268ac6') - version('2.7.11', '1dbcc848b4cd8399a8199d000f9f823c', preferred=True) - version('3.5.0', 'd149d2812f10cbe04c042232e7964171') - version('3.5.1', 'e9ea6f2623fffcdd871b7b19113fde80') + version('3.5.1', 'be78e48cdfc1a7ad90efff146dce6cfe') + version('3.5.0', 'a56c0c0b45d75a0ec9c6dee933c41c36') + version('2.7.11', '6b6076ec9e93f05dd63e47eb9c15728b', preferred=True) + version('2.7.10', 'd7547558fd673bd9d38e2108c6b42521') + version('2.7.9', '5eebcaa0030dc4061156d3429657fb83') + version('2.7.8', 'd4bca0159acb0b44a781292b5231936f') depends_on("openssl") depends_on("bzip2") diff --git a/var/spack/packages/texinfo/package.py b/var/spack/packages/texinfo/package.py index 460db65b3ec..1b22c72e722 100644 --- a/var/spack/packages/texinfo/package.py +++ b/var/spack/packages/texinfo/package.py @@ -33,12 +33,12 @@ class Texinfo(Package): used by many non-GNU projects as well.FIXME: put a proper description of your package here. """ homepage = "https://www.gnu.org/software/texinfo/" - url = "http://ftp.gnu.org/gnu/texinfo/texinfo-6.0.tar.xz" + url = "http://ftp.gnu.org/gnu/texinfo/texinfo-6.0.tar.gz" - version('6.0', '02818e62a5b8ae0213a7ff572991bb50') - version('5.2', 'cb489df8a7ee9d10a236197aefdb32c5') - version('5.1', '52ee905a3b705020d2a1b6ec36d53ca6') - version('5.0', 'ef2fad34c71ddc95b20c7d6a08c0d7a6') + version('6.0', 'e1a2ef5dce5018b53f0f6eed45b247a7') + version('5.2', '1b8f98b80a8e6c50422125e07522e8db') + version('5.1', '54e250014fe698fb4832016158747c03') + version('5.0', '918432285abe6fe96c98355594c5656a') def install(self, spec, prefix): configure('--prefix=%s' % prefix) diff --git a/var/spack/packages/wget/package.py b/var/spack/packages/wget/package.py index c8fd0251229..55728b05157 100644 --- a/var/spack/packages/wget/package.py +++ b/var/spack/packages/wget/package.py @@ -8,9 +8,10 @@ class Wget(Package): etc.""" homepage = "http://www.gnu.org/software/wget/" - url = "http://ftp.gnu.org/gnu/wget/wget-1.16.tar.xz" + url = "http://ftp.gnu.org/gnu/wget/wget-1.16.tar.gz" - version('1.16', 'fe102975ab3a6c049777883f1bb9ad07') + version('1.17', 'c4c4727766f24ac716936275014a0536') + version('1.16', '293a37977c41b5522f781d3a3a078426') depends_on("openssl") From 8da1dd2b4526241e70df3bb17c93439b26f389dd Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 8 Jan 2016 21:32:17 -0500 Subject: [PATCH 266/910] Require cmake --- var/spack/packages/julia/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/julia/package.py b/var/spack/packages/julia/package.py index 9fd946c9054..6e81564e170 100644 --- a/var/spack/packages/julia/package.py +++ b/var/spack/packages/julia/package.py @@ -9,7 +9,7 @@ class Julia(Package): version('0.4.2', 'ccfeb4f4090c8b31083f5e1ccb03eb06') # Build-time dependencies - # depends_on("cmake") + depends_on("cmake") # depends_on("awk") # depends_on("m4") # depends_on("pkg-config") From 04f908a957c69ae03b82482d37b87d74bd4a0064 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 8 Jan 2016 21:41:35 -0500 Subject: [PATCH 267/910] Require cmake @2.8: --- var/spack/packages/julia/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/julia/package.py b/var/spack/packages/julia/package.py index 6e81564e170..40f94269abb 100644 --- a/var/spack/packages/julia/package.py +++ b/var/spack/packages/julia/package.py @@ -9,7 +9,7 @@ class Julia(Package): version('0.4.2', 'ccfeb4f4090c8b31083f5e1ccb03eb06') # Build-time dependencies - depends_on("cmake") + depends_on("cmake @2.8:") # depends_on("awk") # depends_on("m4") # depends_on("pkg-config") From da8fd5b30a696e5e7db91b929e840cb5e1bd070b Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 8 Jan 2016 21:56:08 -0500 Subject: [PATCH 268/910] Require cmake @2.8: for metis --- var/spack/packages/metis/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/metis/package.py b/var/spack/packages/metis/package.py index 7ce5ae1925d..9d6d7fefe1a 100644 --- a/var/spack/packages/metis/package.py +++ b/var/spack/packages/metis/package.py @@ -12,6 +12,7 @@ class Metis(Package): version('5.1.0', '5465e67079419a69e0116de24fce58fe') + depends_on("cmake @2.8:") # build-time dependency depends_on('mpi') def install(self, spec, prefix): From e628896c2be37ae974b787611609e91cee2dcef4 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 8 Jan 2016 22:12:19 -0500 Subject: [PATCH 269/910] parmetis requires cmake @2.8: --- var/spack/packages/parmetis/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/parmetis/package.py b/var/spack/packages/parmetis/package.py index d8cd3373047..8c61cc00c83 100644 --- a/var/spack/packages/parmetis/package.py +++ b/var/spack/packages/parmetis/package.py @@ -10,6 +10,7 @@ class Parmetis(Package): version('4.0.3', 'f69c479586bf6bb7aff6a9bc0c739628') + depends_on('cmake @2.8:') # build dependency depends_on('mpi') def install(self, spec, prefix): From 2ac2985cf20f385e062227fc8f5bf7e4f330409a Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 8 Jan 2016 22:24:42 -0500 Subject: [PATCH 270/910] Require Python for PETSc PETSc requires Python for building. I have a case where an HPC system has a very old default Python version, too old for Spack. So I load a module for Python, which makes Spack work. This module relies on LD_LIBRARY_PATH. When building PETSc, Spack unsets LD_LIBRARY_PATH, breaking the Python that PETSc finds. Explicitly requiring Python for PETSc makes building PETSc work. --- var/spack/packages/petsc/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/packages/petsc/package.py b/var/spack/packages/petsc/package.py index f3ed3d72ec5..50533dbe541 100644 --- a/var/spack/packages/petsc/package.py +++ b/var/spack/packages/petsc/package.py @@ -12,6 +12,8 @@ class Petsc(Package): version('3.5.2', 'ad170802b3b058b5deb9cd1f968e7e13') version('3.5.1', 'a557e029711ebf425544e117ffa44d8f') + depends_on("python @2.6:@2.9") # requires Python for building + depends_on("boost") depends_on("blas") depends_on("lapack") From 0c9acb0f6256389a1fd8266542e0857d6cdeb774 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 8 Jan 2016 22:32:00 -0500 Subject: [PATCH 271/910] Update gperftools to version 2.4 --- var/spack/packages/gperftools/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/gperftools/package.py b/var/spack/packages/gperftools/package.py index 0ba44c93296..3ee0de1bc43 100644 --- a/var/spack/packages/gperftools/package.py +++ b/var/spack/packages/gperftools/package.py @@ -30,6 +30,7 @@ class Gperftools(Package): homepage = "https://code.google.com/p/gperftools" url = "https://googledrive.com/host/0B6NtGsLhIcf7MWxMMF9JdTN3UVk/gperftools-2.3.tar.gz" + version('2.4', '2171cea3bbe053036fb5d5d25176a160', url="https://github.com/gperftools/gperftools/releases/download/gperftools-2.4/gperftools-2.4.tar.gz") version('2.3', 'f54dd119f0e46ac1f13264f8d97adf90', url="https://googledrive.com/host/0B6NtGsLhIcf7MWxMMF9JdTN3UVk/gperftools-2.3.tar.gz") def install(self, spec, prefix): From 3d7e7c53c5c2809df74a8f8d61a7f929e02a1ce6 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 8 Jan 2016 22:33:55 -0500 Subject: [PATCH 272/910] Require libunwind --- var/spack/packages/gperftools/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/packages/gperftools/package.py b/var/spack/packages/gperftools/package.py index 3ee0de1bc43..22b2e6c424e 100644 --- a/var/spack/packages/gperftools/package.py +++ b/var/spack/packages/gperftools/package.py @@ -33,6 +33,8 @@ class Gperftools(Package): version('2.4', '2171cea3bbe053036fb5d5d25176a160', url="https://github.com/gperftools/gperftools/releases/download/gperftools-2.4/gperftools-2.4.tar.gz") version('2.3', 'f54dd119f0e46ac1f13264f8d97adf90', url="https://googledrive.com/host/0B6NtGsLhIcf7MWxMMF9JdTN3UVk/gperftools-2.3.tar.gz") + depends_on("libunwind") + def install(self, spec, prefix): configure("--prefix=" + prefix) make() From d5ff7fda6ec26df1e574ef9fc38ffc082ec2daca Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 8 Jan 2016 22:41:18 -0500 Subject: [PATCH 273/910] Ensure the APPS environment variable is not set when building OpenSSL --- var/spack/packages/openssl/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index a24de20cc37..36b63e1262f 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -17,6 +17,10 @@ class Openssl(Package): parallel = False def install(self, spec, prefix): + # OpenSSL uses a variable APPS in its Makefile. If it happens to be set + # in the environment, then this will override what is set in the + # Makefile, leading to build errors. + del env['APPS'] if spec.satisfies("=darwin-x86_64") or spec.satisfies("=ppc64"): # This needs to be done for all 64-bit architectures (except Linux, # where it happens automatically?) From d9f1a55c902ca4544e99d946a81df33331f989dc Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 8 Jan 2016 22:48:37 -0500 Subject: [PATCH 274/910] Ensure than OpenBLAS provides a file liblapack.a OpenBLAS implements the virtual package "lapack", and its consumers apparently expect a "liblapack.a". --- var/spack/packages/openblas/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/packages/openblas/package.py b/var/spack/packages/openblas/package.py index e01467c05ae..9c8fa1c694b 100644 --- a/var/spack/packages/openblas/package.py +++ b/var/spack/packages/openblas/package.py @@ -19,3 +19,7 @@ def install(self, spec, prefix): with working_dir(prefix.lib): symlink('libopenblas.a', 'blas.a') symlink('libopenblas.a', 'libblas.a') + + # Lapack virtual package should provide liblapack.a + with working_dir(prefix.lib): + symlink('libopenblas.a', 'liblapack.a') From b18732173a22aa431165fa321abac7924f66b1e1 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 8 Jan 2016 23:21:54 -0500 Subject: [PATCH 275/910] Require Python to build Julia --- var/spack/packages/julia/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/julia/package.py b/var/spack/packages/julia/package.py index 40f94269abb..80a58ab7457 100644 --- a/var/spack/packages/julia/package.py +++ b/var/spack/packages/julia/package.py @@ -13,6 +13,7 @@ class Julia(Package): # depends_on("awk") # depends_on("m4") # depends_on("pkg-config") + depends_on("python @2.6:2.9") # I think that Julia requires the dependencies above, but it builds find (on # my system) without these. We should enable them as necessary. From 79a1de2f17094e7e5592bed8cd9040bff9f43cf5 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 9 Jan 2016 10:47:31 -0500 Subject: [PATCH 276/910] Add a patch to prevent Julia running out of virtual memory --- var/spack/packages/julia/gc.patch | 11 +++++++++++ var/spack/packages/julia/package.py | 2 ++ 2 files changed, 13 insertions(+) create mode 100644 var/spack/packages/julia/gc.patch diff --git a/var/spack/packages/julia/gc.patch b/var/spack/packages/julia/gc.patch new file mode 100644 index 00000000000..6db69c6c1b0 --- /dev/null +++ b/var/spack/packages/julia/gc.patch @@ -0,0 +1,11 @@ +--- julia/src/gc.c ++++ julia/src/gc.c +@@ -162,7 +162,7 @@ + // A region is contiguous storage for up to REGION_PG_COUNT naturally aligned GC_PAGE_SZ pages + // It uses a very naive allocator (see malloc_page & free_page) + #if defined(_P64) && !defined(_COMPILER_MICROSOFT_) +-#define REGION_PG_COUNT 16*8*4096 // 8G because virtual memory is cheap ++#define REGION_PG_COUNT 8*4096 // 512M + #else + #define REGION_PG_COUNT 8*4096 // 512M + #endif diff --git a/var/spack/packages/julia/package.py b/var/spack/packages/julia/package.py index 80a58ab7457..d978842d9c9 100644 --- a/var/spack/packages/julia/package.py +++ b/var/spack/packages/julia/package.py @@ -8,6 +8,8 @@ class Julia(Package): version('0.4.2', 'ccfeb4f4090c8b31083f5e1ccb03eb06') + patch('gc.patch') + # Build-time dependencies depends_on("cmake @2.8:") # depends_on("awk") From 992b9c46dd432ad409025a3cbaeb1c06f880526c Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 10 Jan 2016 13:44:53 -0800 Subject: [PATCH 277/910] Resolve readline/ncurses dependency when building Lua --- var/spack/packages/lua/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/lua/package.py b/var/spack/packages/lua/package.py index 6d8f7806d98..f0c949687e9 100644 --- a/var/spack/packages/lua/package.py +++ b/var/spack/packages/lua/package.py @@ -18,11 +18,12 @@ class Lua(Package): version('5.1.3', 'a70a8dfaa150e047866dc01a46272599') depends_on('ncurses') + depends_on('readline') def install(self, spec, prefix): make('INSTALL_TOP=%s' % prefix, - 'MYLDFLAGS="-L%s/lib -Wl,-rpath,%s"' % (spec['ncurses'].prefix,spec['ncurses'].prefix), + 'MYLDFLAGS=-L%s -lncurses' % spec['ncurses'].prefix.lib, 'linux') make('INSTALL_TOP=%s' % prefix, - 'MYLDFLAGS="-L%s/lib -Wl,-rpath,%s"' % (spec['ncurses'].prefix,spec['ncurses'].prefix), + 'MYLDFLAGS=-L%s -lncurses' % spec['ncurses'].prefix.lib, 'install') From 9f2dbc9df43ca0ebb4409c61d0a60bb5c6eccfa7 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 10 Jan 2016 19:34:02 -0500 Subject: [PATCH 278/910] Update Lua to 5.3.2 --- var/spack/packages/lua/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/lua/package.py b/var/spack/packages/lua/package.py index f0c949687e9..e1a1ae95063 100644 --- a/var/spack/packages/lua/package.py +++ b/var/spack/packages/lua/package.py @@ -6,6 +6,7 @@ class Lua(Package): homepage = "http://www.lua.org" url = "http://www.lua.org/ftp/lua-5.1.5.tar.gz" + version('5.3.2', '33278c2ab5ee3c1a875be8d55c1ca2a1') version('5.3.1', '797adacada8d85761c079390ff1d9961') version('5.3.0', 'a1b0a7e92d0c85bbff7a8d27bf29f8af') version('5.2.4', '913fdb32207046b273fdb17aad70be13') From 23c5a1e6e9b6f07ee3a97eeca3e786703bc1a58d Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 10 Jan 2016 19:37:44 -0500 Subject: [PATCH 279/910] Build Lua on Darwin --- var/spack/packages/lua/package.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/lua/package.py b/var/spack/packages/lua/package.py index e1a1ae95063..ca8cfc53657 100644 --- a/var/spack/packages/lua/package.py +++ b/var/spack/packages/lua/package.py @@ -22,9 +22,13 @@ class Lua(Package): depends_on('readline') def install(self, spec, prefix): + if spec.satisfies("=darwin-i686") or spec.satisfies("=darwin-x86_64"): + target = 'macosx' + else: + target = 'linux' make('INSTALL_TOP=%s' % prefix, 'MYLDFLAGS=-L%s -lncurses' % spec['ncurses'].prefix.lib, - 'linux') + target) make('INSTALL_TOP=%s' % prefix, 'MYLDFLAGS=-L%s -lncurses' % spec['ncurses'].prefix.lib, 'install') From 625570e730e0aa699426240c657829e6ac5f3284 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 11 Jan 2016 12:50:47 +0100 Subject: [PATCH 280/910] added package : cgal --- var/spack/packages/cgal/package.py | 73 ++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 var/spack/packages/cgal/package.py diff --git a/var/spack/packages/cgal/package.py b/var/spack/packages/cgal/package.py new file mode 100644 index 00000000000..97356433be1 --- /dev/null +++ b/var/spack/packages/cgal/package.py @@ -0,0 +1,73 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Cgal(Package): + """ + CGAL is a software project that provides easy access to efficient and reliable geometric algorithms in the form of + a C++ library. CGAL is used in various areas needing geometric computation, such as geographic information systems, + computer aided design, molecular biology, medical imaging, computer graphics, and robotics. + """ + homepage = 'http://www.cgal.org/' + url = 'https://github.com/CGAL/cgal/archive/releases/CGAL-4.7.tar.gz' + + version('4.7', '4826714810f3b4c65cac96b90fb03b67') + version('4.6.3', 'e8ee2ecc8d2b09b94a121c09257b576d') + + # Installation instructions : http://doc.cgal.org/latest/Manual/installation.html + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Builds a debug version of the libraries') + + depends_on('boost') + depends_on('mpfr') + depends_on('gmp') + depends_on('zlib') + + # FIXME : Qt5 dependency missing (needs Qt5 and OpenGL) + # FIXME : Optional third party libraries missing + + def install(self, spec, prefix): + + options = [] + options.extend(std_cmake_args) + # CGAL supports only Release and Debug build type. Any other build type will raise an error at configure time + if '+debug' in spec: + options.append('-DCMAKE_BUILD_TYPE:STRING=Debug') + else: + options.append('-DCMAKE_BUILD_TYPE:STRING=Release') + + if '+shared' in spec: + options.append('-DBUILD_SHARED_LIBS:BOOL=ON') + else: + options.append('-DBUILD_SHARED_LIBS:BOOL=OFF') + + build_directory = join_path(self.stage.path, 'spack-build') + source_directory = self.stage.source_path + with working_dir(build_directory, create=True): + cmake(source_directory, *options) + make() + make("install") From 81a389151f58a362f9dc447d5af8dfc13fdd8b3b Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 11 Jan 2016 13:00:23 +0100 Subject: [PATCH 281/910] cmake : added variant that activates the build of the ncurses gui --- var/spack/packages/cmake/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/cmake/package.py b/var/spack/packages/cmake/package.py index cb54c92d692..173f72742c4 100644 --- a/var/spack/packages/cmake/package.py +++ b/var/spack/packages/cmake/package.py @@ -38,8 +38,9 @@ class Cmake(Package): version('3.4.0', 'cd3034e0a44256a0917e254167217fc8', url = 'https://cmake.org/files/v3.4/cmake-3.4.0.tar.gz') -# version('3.0.1', 'e2e05d84cb44a42f1371d9995631dcf5') -# version('3.0.0', '21a1c85e1a3b803c4b48e7ff915a863e') + variant('ncurses', default=True, description='Enables the build of the ncurses gui') + + depends_on('ncurses', when='+ncurses') def install(self, spec, prefix): configure('--prefix=' + prefix, From 6a51b1d5705096264a1280a35185fd7293aafde3 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 11 Jan 2016 13:43:31 -0600 Subject: [PATCH 282/910] Add SZip package --- var/spack/packages/szip/package.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 var/spack/packages/szip/package.py diff --git a/var/spack/packages/szip/package.py b/var/spack/packages/szip/package.py new file mode 100644 index 00000000000..c48c5b431e1 --- /dev/null +++ b/var/spack/packages/szip/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Szip(Package): + """Szip is an implementation of the extended-Rice lossless compression algorithm. + It provides lossless compression of scientific data, and is provided with HDF + software products.""" + + homepage = "https://www.hdfgroup.org/doc_resource/SZIP/" + url = "http://www.hdfgroup.org/ftp/lib-external/szip/2.1/src/szip-2.1.tar.gz" + + version('2.1', '902f831bcefb69c6b635374424acbead') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, + '--enable-production', + '--enable-shared', + '--enable-static', + '--enable-encoding') + + make() + make("install") From adbe1d4f06028ba13e21386f7d62939d4b2eb740 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 11 Jan 2016 13:54:26 -0600 Subject: [PATCH 283/910] Add PatchELF package --- var/spack/packages/patchelf/package.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 var/spack/packages/patchelf/package.py diff --git a/var/spack/packages/patchelf/package.py b/var/spack/packages/patchelf/package.py new file mode 100644 index 00000000000..036dc6bd17b --- /dev/null +++ b/var/spack/packages/patchelf/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Patchelf(Package): + """PatchELF is a small utility to modify the dynamic linker and RPATH of ELF executables.""" + + homepage = "https://nixos.org/patchelf.html" + url = "http://nixos.org/releases/patchelf/patchelf-0.8/patchelf-0.8.tar.gz" + list_url = "http://nixos.org/releases/patchelf/" + list_depth = 2 + + version('0.8', '407b229e6a681ffb0e2cdd5915cb2d01') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") From 85220dd77d696e11bd39cdd467084f579a10a138 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 11 Jan 2016 13:57:21 -0800 Subject: [PATCH 284/910] Revert "Require Python for PETSc" --- var/spack/packages/petsc/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/var/spack/packages/petsc/package.py b/var/spack/packages/petsc/package.py index dc4d6bb4129..43c7f8a9b6c 100644 --- a/var/spack/packages/petsc/package.py +++ b/var/spack/packages/petsc/package.py @@ -12,8 +12,6 @@ class Petsc(Package): version('3.5.2', 'ad170802b3b058b5deb9cd1f968e7e13') version('3.5.1', 'a557e029711ebf425544e117ffa44d8f') - depends_on("python @2.6:@2.9") # requires Python for building - depends_on("boost") depends_on("blas") depends_on("lapack") From 8af8c105597f3bdeb999e288f8201612d6756788 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 11 Jan 2016 17:33:44 -0500 Subject: [PATCH 285/910] Avoid key error when env var is unset --- var/spack/packages/openssl/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index 36b63e1262f..bbb169ec6bc 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -20,7 +20,7 @@ def install(self, spec, prefix): # OpenSSL uses a variable APPS in its Makefile. If it happens to be set # in the environment, then this will override what is set in the # Makefile, leading to build errors. - del env['APPS'] + env.pop('APPS', None) if spec.satisfies("=darwin-x86_64") or spec.satisfies("=ppc64"): # This needs to be done for all 64-bit architectures (except Linux, # where it happens automatically?) From a5588e4bb76892286fadf486d15e96f16a107a68 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 8 Jan 2016 14:20:48 -0500 Subject: [PATCH 286/910] Download package version with a configure script This avoids having to use autoconf, which requires many m4 scripts that are not always available. --- var/spack/packages/libpciaccess/package.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/var/spack/packages/libpciaccess/package.py b/var/spack/packages/libpciaccess/package.py index 403bafbbd2c..0c0847d3232 100644 --- a/var/spack/packages/libpciaccess/package.py +++ b/var/spack/packages/libpciaccess/package.py @@ -5,12 +5,10 @@ class Libpciaccess(Package): """Generic PCI access library.""" homepage = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" - url = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" + url = "http://xorg.freedesktop.org/archive/individual/lib/libpciaccess-0.13.4.tar.bz2" - version('0.13.4', git='http://anongit.freedesktop.org/git/xorg/lib/libpciaccess.git', - tag='libpciaccess-0.13.4') + version('0.13.4', 'ace78aec799b1cf6dfaea55d3879ed9f') - depends_on('autoconf') depends_on('libtool') def install(self, spec, prefix): @@ -20,9 +18,6 @@ def install(self, spec, prefix): mkdir(prefix.lib) return - from subprocess import call - call(["./autogen.sh"]) configure("--prefix=%s" % prefix) - make() make("install") From 36d74d2a97b4b00fb9abc7fb07a067dac157f038 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 11 Jan 2016 17:39:39 -0500 Subject: [PATCH 287/910] Correct Python version constraint syntax --- var/spack/packages/petsc/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/petsc/package.py b/var/spack/packages/petsc/package.py index 50533dbe541..1463f360c2f 100644 --- a/var/spack/packages/petsc/package.py +++ b/var/spack/packages/petsc/package.py @@ -12,7 +12,7 @@ class Petsc(Package): version('3.5.2', 'ad170802b3b058b5deb9cd1f968e7e13') version('3.5.1', 'a557e029711ebf425544e117ffa44d8f') - depends_on("python @2.6:@2.9") # requires Python for building + depends_on("python @2.6:2.9") # requires Python for building depends_on("boost") depends_on("blas") From 50727527bc69a1b8ed26aa84a40039086db33827 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 11 Jan 2016 15:51:02 -0800 Subject: [PATCH 288/910] This commit makes the following changes: There are two sensible defaults for building boost libraries: build all of them or build none of them. Previously the Spack boost package took the first approach. This commit changes to building no libraries by default. The user can specify which libraries they need using variants (e.g. +iostreams to compile the boost iostreams library). If no libraries are built then a header-only install is performed (no compilation, just copy header files to prefix). The consequence of this change is that packages which specify a dependency on boost may now fail (until they are updated to specify exactly which boost libraries they need compiled). The user may now specify whether to build shared libraries (static libraries are always built) and whether to build libraries with/out multi-threading support (default is to only build with multi-threading support). The executable on the user-config.jam toolset line is set to Spack's cc script. Before, without this, the desired toolset was used but Spack deferred to the boost build system to choose the compiler version. bzip2 and zlib are always specified as dependencies when iostreams is built (before this could be controlled with the +compression variant). --- var/spack/packages/boost/package.py | 90 ++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 26 deletions(-) diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index 3427b74ad68..5fb6c9dc049 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -1,4 +1,5 @@ from spack import * +import spack class Boost(Package): """Boost provides free peer-reviewed portable C++ source @@ -44,15 +45,35 @@ class Boost(Package): version('1.34.1', '2d938467e8a448a2c9763e0a9f8ca7e5') version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') - variant('debug', default=False, description='Switch to the debug version of Boost') - variant('python', default=False, description='Activate the component Boost.Python') - variant('mpi', default=False, description='Activate the component Boost.MPI') - variant('compression', default=True, description='Activate the compression Boost.iostreams') + libs = ['chrono', + 'date_time', + 'filesystem', + 'iostreams', + 'random', + 'regex', + 'serialization', + 'signals', + 'system', + 'thread', + 'wave', + 'mpi', + 'python'] + for lib in libs: + variant(lib, default=False, description="Compile with {0} library" + .format(lib)) + + variant('debug', default=False, description='Switch to the debug version of Boost') + variant('shared', default=True, description="Additionally build shared libraries") + variant('multithreaded', default=True, description="Build multi-threaded versions of libraries") + variant('singlethreaded', default=False, description="Build single-threaded versions of libraries") + variant('regex_icu', default=False, description="Include regex ICU support (by default false even if regex library is compiled)") + + depends_on('icu', when='+regex_icu') depends_on('python', when='+python') depends_on('mpi', when='+mpi') - depends_on('bzip2', when='+compression') - depends_on('zlib', when='+compression') + depends_on('bzip2', when='+iostreams') + depends_on('zlib', when='+iostreams') def url_for_version(self, version): """Handle Boost's weird URLs, which write the version two different ways.""" @@ -77,22 +98,20 @@ def determine_toolset(self, spec): # fallback to gcc if no toolset found return 'gcc' - def determine_bootstrap_options(self, spec, options): - options.append('--with-toolset=%s' % self.determine_toolset(spec)) + def determine_bootstrap_options(self, spec, withLibs, options): + boostToolsetId = self.determine_toolset(spec) + options.append('--with-toolset=%s' % boostToolsetId) + options.append("--with-libraries=%s" % ','.join(withLibs)) - without_libs = [] - if '~mpi' in spec: - without_libs.append('mpi') - if '~python' in spec: - without_libs.append('python') - else: + if '+python' in spec: options.append('--with-python=%s' % join_path(spec['python'].prefix.bin, 'python')) - if without_libs: - options.append('--without-libraries=%s' % ','.join(without_libs)) - with open('user-config.jam', 'w') as f: + compiler_wrapper = join_path(spack.build_env_path, 'c++') + f.write("using {0} : : {1} ;\n".format(boostToolsetId, + compiler_wrapper)) + if '+mpi' in spec: f.write('using mpi : %s ;\n' % join_path(spec['mpi'].prefix.bin, 'mpicxx')) @@ -107,12 +126,7 @@ def determine_b2_options(self, spec, options): else: options.append('variant=release') - if '~compression' in spec: - options.extend([ - '-s', 'NO_BZIP2=1', - '-s', 'NO_ZLIB=1']) - - if '+compression' in spec: + if '+iostreams' in spec: options.extend([ '-s', 'BZIP2_INCLUDE=%s' % spec['bzip2'].prefix.include, '-s', 'BZIP2_LIBPATH=%s' % spec['bzip2'].prefix.lib, @@ -120,20 +134,44 @@ def determine_b2_options(self, spec, options): '-s', 'ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib, ]) + linkTypes = ['static'] + if '+shared' in spec: + linkTypes.append('shared') + + #TODO: at least one of these two options must be active + threadingOpts = [] + if '+multithreaded' in spec: + threadingOpts.append('multi') + if '+singlethreaded' in spec: + threadingOpts.append('single') + options.extend([ 'toolset=%s' % self.determine_toolset(spec), - 'link=static,shared', - 'threading=single,multi', + 'link=%s' % ','.join(linkTypes), + 'threading=%s' % ','.join(threadingOpts), '--layout=tagged']) def install(self, spec, prefix): + withLibs = list() + for lib in Boost.libs: + if "+{0}".format(lib) in spec: + withLibs.append(lib) + if not withLibs: + # if no libraries are specified for compilation, then you dont have + # to configure/build anything, just copy over to the prefix directory. + src = join_path(self.stage.source_path, 'boost') + mkdirp(join_path(prefix, 'include')) + dst = join_path(prefix, 'include', 'boost') + install_tree(src, dst) + return + # to make Boost find the user-config.jam env['BOOST_BUILD_PATH'] = './' bootstrap = Executable('./bootstrap.sh') bootstrap_options = ['--prefix=%s' % prefix] - self.determine_bootstrap_options(spec, bootstrap_options) + self.determine_bootstrap_options(spec, withLibs, bootstrap_options) bootstrap(*bootstrap_options) From 30da85c3d8ecf293e791820880fd29306ad07d94 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 12 Jan 2016 11:26:29 +0100 Subject: [PATCH 289/910] metis : added variant specified in BUILD.txt. Fixed wrong dependency on MPI. parmetis : added variants as in metis --- var/spack/packages/metis/package.py | 93 +++++++++++++++++----- var/spack/packages/parmetis/package.py | 105 ++++++++++++++++++++----- 2 files changed, 161 insertions(+), 37 deletions(-) diff --git a/var/spack/packages/metis/package.py b/var/spack/packages/metis/package.py index 9d6d7fefe1a..bbfc4de7d1d 100644 --- a/var/spack/packages/metis/package.py +++ b/var/spack/packages/metis/package.py @@ -1,28 +1,83 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + from spack import * -class Metis(Package): - """METIS is a set of serial programs for partitioning graphs, - partitioning finite element meshes, and producing fill reducing - orderings for sparse matrices. The algorithms implemented in - METIS are based on the multilevel recursive-bisection, - multilevel k-way, and multi-constraint partitioning schemes.""" - homepage = "http://glaros.dtc.umn.edu/gkhome/metis/metis/overview" - url = "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz" +class Metis(Package): + """ + METIS is a set of serial programs for partitioning graphs, partitioning finite element meshes, and producing fill + reducing orderings for sparse matrices. The algorithms implemented in METIS are based on the multilevel + recursive-bisection, multilevel k-way, and multi-constraint partitioning schemes. + """ + + homepage = 'http://glaros.dtc.umn.edu/gkhome/metis/metis/overview' + url = "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz" version('5.1.0', '5465e67079419a69e0116de24fce58fe') - depends_on("cmake @2.8:") # build-time dependency - depends_on('mpi') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Builds the library in debug mode') + variant('gdb', default=False, description='Enables gdb support') + + variant('idx64', default=False, description='Use int64_t as default index type') + variant('double', default=False, description='Use double precision floating point types') + + depends_on('cmake @2.8:') # build-time dependency + + depends_on('gdb', when='+gdb') def install(self, spec, prefix): - cmake(".", - '-DGKLIB_PATH=%s/GKlib' % pwd(), - '-DSHARED=1', - '-DCMAKE_C_COMPILER=mpicc', - '-DCMAKE_CXX_COMPILER=mpicxx', - '-DSHARED=1', - *std_cmake_args) - make() - make("install") + options = [] + options.extend(std_cmake_args) + + build_directory = join_path(self.stage.path, 'spack-build') + source_directory = self.stage.source_path + + options.append('-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=source_directory)) + + if '+shared' in spec: + options.append('-DSHARED:BOOL=ON') + + if '+debug' in spec: + options.extend(['-DDEBUG:BOOL=ON', + '-DCMAKE_BUILD_TYPE:STRING=Debug']) + + if '+gdb' in spec: + options.append('-DGDB:BOOL=ON') + + metis_header = join_path(source_directory, 'include', 'metis.h') + + if '+idx64' in spec: + filter_file('IDXTYPEWIDTH 32', 'IDXTYPEWIDTH 64', metis_header) + + if '+double' in spec: + filter_file('REALTYPEWIDTH 32', 'REALTYPEWIDTH 64', metis_header) + + with working_dir(build_directory, create=True): + cmake(source_directory, *options) + make() + make("install") \ No newline at end of file diff --git a/var/spack/packages/parmetis/package.py b/var/spack/packages/parmetis/package.py index 8c61cc00c83..fa1b317eebf 100644 --- a/var/spack/packages/parmetis/package.py +++ b/var/spack/packages/parmetis/package.py @@ -1,27 +1,96 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + from spack import * +import shutil + +# FIXME : lot of code is duplicated from packages/metis/package.py . Inheriting from there may reduce +# FIXME : the installation rules to just a few lines + class Parmetis(Package): - """ParMETIS is an MPI-based parallel library that implements a - variety of algorithms for partitioning unstructured graphs, - meshes, and for computing fill-reducing orderings of sparse - matrices.""" - homepage = "http://glaros.dtc.umn.edu/gkhome/metis/parmetis/overview" - url = "http://glaros.dtc.umn.edu/gkhome/fetch/sw/parmetis/parmetis-4.0.3.tar.gz" + """ + ParMETIS is an MPI-based parallel library that implements a variety of algorithms for partitioning unstructured + graphs, meshes, and for computing fill-reducing orderings of sparse matrices. + """ + homepage = 'http://glaros.dtc.umn.edu/gkhome/metis/parmetis/overview' + url = 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/parmetis/parmetis-4.0.3.tar.gz' version('4.0.3', 'f69c479586bf6bb7aff6a9bc0c739628') - depends_on('cmake @2.8:') # build dependency + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Builds the library in debug mode') + variant('gdb', default=False, description='Enables gdb support') + + variant('idx64', default=False, description='Use int64_t as default index type') + variant('double', default=False, description='Use double precision floating point types') + + depends_on('cmake @2.8:') # build dependency depends_on('mpi') - def install(self, spec, prefix): - cmake(".", - '-DGKLIB_PATH=%s/metis/GKlib' % pwd(), - '-DMETIS_PATH=%s/metis' % pwd(), - '-DSHARED=1', - '-DCMAKE_C_COMPILER=mpicc', - '-DCMAKE_CXX_COMPILER=mpicxx', - '-DSHARED=1', - *std_cmake_args) + # FIXME : this should conflict with metis as it builds its own version internally - make() - make("install") + depends_on('gdb', when='+gdb') + + def install(self, spec, prefix): + options = [] + options.extend(std_cmake_args) + + build_directory = join_path(self.stage.path, 'spack-build') + source_directory = self.stage.source_path + metis_source = join_path(source_directory, 'metis') + + # FIXME : Once a contract is defined, MPI compilers should be retrieved indirectly via spec['mpi'] in case + # FIXME : they use a non-standard name + options.extend(['-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=metis_source), + '-DMETIS_PATH:PATH={metis_source}'.format(metis_source=metis_source), + '-DCMAKE_C_COMPILER:STRING=mpicc', + '-DCMAKE_CXX_COMPILER:STRING=mpicxx']) + + if '+shared' in spec: + options.append('-DSHARED:BOOL=ON') + + if '+debug' in spec: + options.extend(['-DDEBUG:BOOL=ON', + '-DCMAKE_BUILD_TYPE:STRING=Debug']) + + if '+gdb' in spec: + options.append('-DGDB:BOOL=ON') + + metis_header = join_path(metis_source, 'include', 'metis.h') + + if '+idx64' in spec: + filter_file('IDXTYPEWIDTH 32', 'IDXTYPEWIDTH 64', metis_header) + + if '+double' in spec: + filter_file('REALTYPEWIDTH 32', 'REALTYPEWIDTH 64', metis_header) + + with working_dir(build_directory, create=True): + cmake(source_directory, *options) + make() + make("install") + # Parmetis build system doesn't allow for an external metis to be used, but doesn't copy the required + # metis header either + shutil.copy(metis_header, join_path(self.prefix, 'include')) From da8b08b5fe1791754ea49590d84c5eeda1e542f2 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 12 Jan 2016 14:55:29 +0100 Subject: [PATCH 290/910] parmetis : used spack install method instead of shutil.copy --- var/spack/packages/parmetis/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/parmetis/package.py b/var/spack/packages/parmetis/package.py index fa1b317eebf..613eabccff0 100644 --- a/var/spack/packages/parmetis/package.py +++ b/var/spack/packages/parmetis/package.py @@ -93,4 +93,4 @@ def install(self, spec, prefix): make("install") # Parmetis build system doesn't allow for an external metis to be used, but doesn't copy the required # metis header either - shutil.copy(metis_header, join_path(self.prefix, 'include')) + install(metis_header, self.prefix.include) From 1f78b9c7afe79eabc51cb4fdc9a1298dfcc35097 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 12 Jan 2016 16:59:47 +0100 Subject: [PATCH 291/910] parmetis : removed leftover import statement --- var/spack/packages/parmetis/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/var/spack/packages/parmetis/package.py b/var/spack/packages/parmetis/package.py index 613eabccff0..c897dec7e44 100644 --- a/var/spack/packages/parmetis/package.py +++ b/var/spack/packages/parmetis/package.py @@ -24,7 +24,6 @@ ############################################################################## from spack import * -import shutil # FIXME : lot of code is duplicated from packages/metis/package.py . Inheriting from there may reduce # FIXME : the installation rules to just a few lines From 9b3e0255f4e8c1ec02aae2a75a11383ad274507d Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 12 Jan 2016 11:55:42 -0600 Subject: [PATCH 292/910] Add HDF package --- var/spack/packages/hdf/package.py | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 var/spack/packages/hdf/package.py diff --git a/var/spack/packages/hdf/package.py b/var/spack/packages/hdf/package.py new file mode 100644 index 00000000000..7882665dbaf --- /dev/null +++ b/var/spack/packages/hdf/package.py @@ -0,0 +1,35 @@ +from spack import * + +class Hdf(Package): + """HDF4 (also known as HDF) is a library and multi-object + file format for storing and managing data between machines.""" + + homepage = "https://www.hdfgroup.org/products/hdf4/" + url = "https://www.hdfgroup.org/ftp/HDF/releases/HDF4.2.11/src/hdf-4.2.11.tar.gz" + list_url = "https://www.hdfgroup.org/ftp/HDF/releases/" + list_depth = 3 + + version('4.2.11', '063f9928f3a19cc21367b71c3b8bbf19') + + depends_on("jpeg") + depends_on("szip@2.1") + depends_on("zlib") + + + def url_for_version(self, version): + return "https://www.hdfgroup.org/ftp/HDF/releases/HDF" + str(version) + "/src/hdf-" + str(version) + ".tar.gz" + + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, + '--with-jpeg=%s' % spec['jpeg'].prefix, + '--with-szlib=%s' % spec['szip'].prefix, + '--with-zlib=%s' % spec['zlib'].prefix, + '--disable-netcdf', + '--enable-fortran', + '--disable-shared', + '--enable-static', + '--enable-production') + + make() + make("install") From 5520ce3c4a46476f5eec7a340471dd044932ac06 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 12 Jan 2016 12:19:03 -0600 Subject: [PATCH 293/910] Removed unnecessary URLs --- var/spack/packages/xz/package.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/xz/package.py b/var/spack/packages/xz/package.py index ba6c9733a7e..fdcac953458 100644 --- a/var/spack/packages/xz/package.py +++ b/var/spack/packages/xz/package.py @@ -8,11 +8,9 @@ class Xz(Package): homepage = "http://tukaani.org/xz/" url = "http://tukaani.org/xz/xz-5.2.0.tar.bz2" - version('5.2.0', '867cc8611760240ebf3440bd6e170bb9', - url = 'http://tukaani.org/xz/xz-5.2.0.tar.bz2') - version('5.2.2', 'f90c9a0c8b259aee2234c4e0d7fd70af', - url = 'http://tukaani.org/xz/xz-5.2.2.tar.bz2') - + version('5.2.0', '867cc8611760240ebf3440bd6e170bb9') + version('5.2.2', 'f90c9a0c8b259aee2234c4e0d7fd70af') + def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() From 6985d2c914159f7978c304530f7fb1eb38b2b78d Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 12 Jan 2016 13:00:05 -0600 Subject: [PATCH 294/910] Add url_for_version function to calculate url --- var/spack/packages/hwloc/package.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/hwloc/package.py b/var/spack/packages/hwloc/package.py index 7ebede76a3b..60b315119ba 100644 --- a/var/spack/packages/hwloc/package.py +++ b/var/spack/packages/hwloc/package.py @@ -14,15 +14,18 @@ class Hwloc(Package): efficiently.""" homepage = "http://www.open-mpi.org/projects/hwloc/" url = "http://www.open-mpi.org/software/hwloc/v1.9/downloads/hwloc-1.9.tar.gz" + list_url = "http://www.open-mpi.org/software/hwloc/" + list_depth = 3 - version('1.11.2', '486169cbe111cdea57be12638828ebbf', - url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.2.tar.bz2') - version('1.11.1', '002742efd3a8431f98d6315365a2b543', - url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.1.tar.bz2') - version('1.9', '1f9f9155682fe8946a97c08896109508') + version('1.11.2', '486169cbe111cdea57be12638828ebbf') + version('1.11.1', '002742efd3a8431f98d6315365a2b543') + version('1.9', '1f9f9155682fe8946a97c08896109508') depends_on('libpciaccess') + def url_for_version(self, version): + return "http://www.open-mpi.org/software/hwloc/v%s/downloads/hwloc-%s.tar.gz" % (version.up_to(2), version) + def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 180c673c782d10864dce6cd7df43465dd05579a1 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 12 Jan 2016 13:21:53 -0600 Subject: [PATCH 295/910] Add different URL for older source code --- var/spack/packages/mpc/package.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/mpc/package.py b/var/spack/packages/mpc/package.py index 50477a0ccba..65c0ba7ea13 100644 --- a/var/spack/packages/mpc/package.py +++ b/var/spack/packages/mpc/package.py @@ -34,8 +34,14 @@ class Mpc(Package): version('1.0.3', 'd6a1d5f8ddea3abd2cc3e98f58352d26') version('1.0.2', '68fadff3358fb3e7976c7a398a0af4c3') - depends_on("gmp") - depends_on("mpfr") + depends_on("gmp@4.3.2:") + depends_on("mpfr@2.4.2:") + + def url_for_version(self, version): + if version < Version("1.0.1"): + return "http://www.multiprecision.org/mpc/download/mpc-%s.tar.gz" % version + else: + return "ftp://ftp.gnu.org/gnu/mpc/mpc-%s.tar.gz" % version def install(self, spec, prefix): configure("--prefix=%s" % prefix) From a6f2cc9cbf82bae1b82680651ac6c1804524c77c Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 12 Jan 2016 13:28:48 -0600 Subject: [PATCH 296/910] Add required dependency version --- var/spack/packages/mpfr/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/mpfr/package.py b/var/spack/packages/mpfr/package.py index 0f2baac0041..2758b0da2ed 100644 --- a/var/spack/packages/mpfr/package.py +++ b/var/spack/packages/mpfr/package.py @@ -31,9 +31,9 @@ class Mpfr(Package): url = "http://www.mpfr.org/mpfr-current/mpfr-3.1.3.tar.bz2" version('3.1.3', '5fdfa3cfa5c86514ee4a241a1affa138') - # version('3.1.2', 'ee2c3ac63bf0c2359bf08fc3ee094c19') + version('3.1.2', 'ee2c3ac63bf0c2359bf08fc3ee094c19') - depends_on('gmp') + depends_on('gmp@4.1.0:') def install(self, spec, prefix): configure("--prefix=%s" % prefix) From ffaa4a8922e8bb41a45eecb3ab7ba72bf677317e Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 13 Jan 2016 10:05:19 +0100 Subject: [PATCH 297/910] gmsh : added package with basic compile options --- var/spack/packages/gmsh/package.py | 80 ++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 var/spack/packages/gmsh/package.py diff --git a/var/spack/packages/gmsh/package.py b/var/spack/packages/gmsh/package.py new file mode 100644 index 00000000000..45f2d4eadfc --- /dev/null +++ b/var/spack/packages/gmsh/package.py @@ -0,0 +1,80 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Gmsh(Package): + """ + Gmsh is a free 3D finite element grid generator with a build-in CAD engine and post-processor. Its design goal is + to provide a fast, light and user-friendly meshing tool with parametric input and advanced visualization + capabilities. Gmsh is built around four modules: geometry, mesh, solver and post-processing. The specification of + any input to these modules is done either interactively using the graphical user interface or in ASCII text files + using Gmsh's own scripting language. + """ + homepage = 'http://gmsh.info' + url = 'http://gmsh.info/src/gmsh-2.11.0-source.tgz' + + version('2.11.0', 'f15b6e7ac9ca649c9a74440e1259d0db') + + # FIXME : Misses dependencies on gmm, FLTK, PetsC, TetGen, HDF5 + + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Builds the library in debug mode') + variant('mpi', default=False, description='Builds MPI support for parser and solver') + variant('compression', default=True, description='Enables IO compression through zlib') + + depends_on('blas') + depends_on('lapack') + depends_on('gmp') + depends_on('mpi', when='+mpi') + depends_on('zlib', when='+compression') + + def install(self, spec, prefix): + + options = [] + options.extend(std_cmake_args) + + build_directory = join_path(self.stage.path, 'spack-build') + source_directory = self.stage.source_path + + if '+shared' in spec: + options.extend(['-DENABLE_BUILD_SHARED:BOOL=ON', + '-DENABLE_BUILD_DYNAMIC:BOOL=ON']) # Builds dynamic executable and installs shared library + else: + options.append('-DENABLE_BUILD_LIB:BOOL=ON') # Builds and installs static library + + if '+debug' in spec: + options.append('-DCMAKE_BUILD_TYPE:STRING=Debug') + + if '+mpi' in spec: + options.append('-DENABLE_MPI:BOOL=ON') + + if '+compression' in spec: + options.append('-DENABLE_COMPRESSED_IO:BOOL=ON') + + with working_dir(build_directory, create=True): + cmake(source_directory, *options) + make() + make('install') From 767f286a9fbdd1a44e2231ef7e21ecd64c1bfb30 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 13 Jan 2016 11:26:10 +0100 Subject: [PATCH 298/910] gmsh : added fltk support --- var/spack/packages/fltk/font.patch | 44 +++++++++++++++++++++++ var/spack/packages/fltk/package.py | 58 ++++++++++++++++++++++++++++++ var/spack/packages/gmsh/package.py | 6 ++-- 3 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 var/spack/packages/fltk/font.patch create mode 100644 var/spack/packages/fltk/package.py diff --git a/var/spack/packages/fltk/font.patch b/var/spack/packages/fltk/font.patch new file mode 100644 index 00000000000..7706a1b4eee --- /dev/null +++ b/var/spack/packages/fltk/font.patch @@ -0,0 +1,44 @@ +Index: FL/x.H +=================================================================== +--- a/FL/x.H (revision 10476) ++++ b/FL/x.H (working copy) +@@ -132,6 +132,7 @@ + XFontStruct *ptr; + }; + extern FL_EXPORT Fl_XFont_On_Demand fl_xfont; ++extern FL_EXPORT XFontStruct* fl_core_font(); + + // this object contains all X-specific stuff about a window: + // Warning: this object is highly subject to change! +Index: src/fl_font.cxx +=================================================================== +--- a/src/fl_font.cxx (revision 10476) ++++ b/src/fl_font.cxx (working copy) +@@ -55,6 +55,14 @@ + # include "fl_font_x.cxx" + #endif // WIN32 + ++#ifdef WIN32 ++#elif defined(__APPLE__) ++#else ++XFontStruct *fl_core_font() ++{ ++ return fl_xfont.value(); ++} ++#endif + + double fl_width(const char* c) { + if (c) return fl_width(c, (int) strlen(c)); +Index: src/gl_draw.cxx +=================================================================== +--- a/src/gl_draw.cxx (revision 10476) ++++ b/src/gl_draw.cxx (working copy) +@@ -84,7 +84,7 @@ + * then sorting through them at draw time (for normal X rendering) to find which one can + * render the current glyph... But for now, just use the first font in the list for GL... + */ +- XFontStruct *font = fl_xfont; ++ XFontStruct *font = fl_core_font(); + int base = font->min_char_or_byte2; + int count = font->max_char_or_byte2-base+1; + fl_fontsize->listbase = glGenLists(256); diff --git a/var/spack/packages/fltk/package.py b/var/spack/packages/fltk/package.py new file mode 100644 index 00000000000..0b462f83f8d --- /dev/null +++ b/var/spack/packages/fltk/package.py @@ -0,0 +1,58 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Fltk(Package): + """ + FLTK (pronounced "fulltick") is a cross-platform C++ GUI toolkit for UNIX/Linux (X11), Microsoft Windows, and + MacOS X. FLTK provides modern GUI functionality without the bloat and supports 3D graphics via OpenGL and its + built-in GLUT emulation. + + FLTK is designed to be small and modular enough to be statically linked, but works fine as a shared library. FLTK + also includes an excellent UI builder called FLUID that can be used to create applications in minutes. + """ + homepage = 'http://www.fltk.org/' + url = 'http://fltk.org/pub/fltk/1.3.3/fltk-1.3.3-source.tar.gz' + + version('1.3.3', '9ccdb0d19dc104b87179bd9fd10822e3') + + patch('font.patch', when='@1.3.3') + + variant('shared', default=True, description='Enables the build of shared libraries') + + def install(self, spec, prefix): + options = ['--prefix=%s' % prefix, + '--enable-localjpeg', + '--enable-localpng', + '--enable-localzlib'] + + if '+shared' in spec: + options.append('--enable-shared') + + # FLTK needs to be built in-source + configure(*options) + make() + make('install') diff --git a/var/spack/packages/gmsh/package.py b/var/spack/packages/gmsh/package.py index 45f2d4eadfc..4e298d2a09b 100644 --- a/var/spack/packages/gmsh/package.py +++ b/var/spack/packages/gmsh/package.py @@ -27,7 +27,7 @@ class Gmsh(Package): """ - Gmsh is a free 3D finite element grid generator with a build-in CAD engine and post-processor. Its design goal is + Gmsh is a free 3D finite element grid generator with a built-in CAD engine and post-processor. Its design goal is to provide a fast, light and user-friendly meshing tool with parametric input and advanced visualization capabilities. Gmsh is built around four modules: geometry, mesh, solver and post-processing. The specification of any input to these modules is done either interactively using the graphical user interface or in ASCII text files @@ -38,17 +38,19 @@ class Gmsh(Package): version('2.11.0', 'f15b6e7ac9ca649c9a74440e1259d0db') - # FIXME : Misses dependencies on gmm, FLTK, PetsC, TetGen, HDF5 + # FIXME : Misses dependencies on gmm, PetsC, TetGen, HDF5 variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds the library in debug mode') variant('mpi', default=False, description='Builds MPI support for parser and solver') + variant('fltk', default=False, description='Enables the build of the FLTK GUI') variant('compression', default=True, description='Enables IO compression through zlib') depends_on('blas') depends_on('lapack') depends_on('gmp') depends_on('mpi', when='+mpi') + depends_on('fltk', when='+fltk') # Assumes OpenGL with GLU is already provided by the system depends_on('zlib', when='+compression') def install(self, spec, prefix): From 78b57787faebf906a9617d1425e489ca598153f4 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 13 Jan 2016 12:22:59 +0100 Subject: [PATCH 299/910] gmsh : added hdf5 support --- var/spack/packages/gmsh/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/gmsh/package.py b/var/spack/packages/gmsh/package.py index 4e298d2a09b..9d759303cbb 100644 --- a/var/spack/packages/gmsh/package.py +++ b/var/spack/packages/gmsh/package.py @@ -38,12 +38,13 @@ class Gmsh(Package): version('2.11.0', 'f15b6e7ac9ca649c9a74440e1259d0db') - # FIXME : Misses dependencies on gmm, PetsC, TetGen, HDF5 + # FIXME : Misses dependencies on gmm, PetsC, TetGen variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds the library in debug mode') variant('mpi', default=False, description='Builds MPI support for parser and solver') variant('fltk', default=False, description='Enables the build of the FLTK GUI') + variant('hdf5', default=False, description='Enables HDF5 support') variant('compression', default=True, description='Enables IO compression through zlib') depends_on('blas') @@ -51,6 +52,7 @@ class Gmsh(Package): depends_on('gmp') depends_on('mpi', when='+mpi') depends_on('fltk', when='+fltk') # Assumes OpenGL with GLU is already provided by the system + depends_on('hdf5', when='+hdf5') depends_on('zlib', when='+compression') def install(self, spec, prefix): From aa1f96c73b0839f560f97b4814959c35be1ac056 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 13 Jan 2016 11:21:52 -0500 Subject: [PATCH 300/910] New package "cereal" --- var/spack/packages/cereal/Werror.patch | 33 +++++++++++++++++++++++++ var/spack/packages/cereal/package.py | 34 ++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 var/spack/packages/cereal/Werror.patch create mode 100644 var/spack/packages/cereal/package.py diff --git a/var/spack/packages/cereal/Werror.patch b/var/spack/packages/cereal/Werror.patch new file mode 100644 index 00000000000..d39eaaffdb8 --- /dev/null +++ b/var/spack/packages/cereal/Werror.patch @@ -0,0 +1,33 @@ +--- old/sandbox/CMakeLists.txt ++++ new/sandbox/CMakeLists.txt +@@ -4,9 +4,11 @@ + add_executable(sandbox_json sandbox_json.cpp) + add_executable(sandbox_rtti sandbox_rtti.cpp) + ++if(Boost_FOUND) + add_executable(sandbox_vs sandbox_vs.cpp) + target_link_libraries(sandbox_vs sandbox_vs_dll) + include_directories(sandbox_shared_lib) ++endif(Boost_FOUND) + + if(Boost_FOUND) + add_executable(performance performance.cpp) +--- old/include/cereal/types/common.hpp ++++ new/include/cereal/types/common.hpp +@@ -106,14 +106,16 @@ + t = reinterpret_cast::type const &>( value ); + } + ++#ifndef CEREAL_ENABLE_RAW_POINTER_SERIALIZATION + //! Serialization for raw pointers + /*! This exists only to throw a static_assert to let users know we don't support raw pointers. */ + template inline + void CEREAL_SERIALIZE_FUNCTION_NAME( Archive &, T * & ) + { + static_assert(cereal::traits::detail::delay_static_assert::value, + "Cereal does not support serializing raw pointers - please use a smart pointer"); + } ++#endif + + //! Serialization for C style arrays + template inline diff --git a/var/spack/packages/cereal/package.py b/var/spack/packages/cereal/package.py new file mode 100644 index 00000000000..a83927456f1 --- /dev/null +++ b/var/spack/packages/cereal/package.py @@ -0,0 +1,34 @@ +from spack import * +import shutil + +class Cereal(Package): + """cereal is a header-only C++11 serialization library. cereal takes arbitrary data types and reversibly turns them into different representations, such as compact binary encodings, XML, or JSON. cereal was designed to be fast, light-weight, and easy to extend - it has no external dependencies and can be easily bundled with other code or used standalone.""" + homepage = "http://uscilab.github.io/cereal/" + url = "https://github.com/USCiLab/cereal/archive/v1.1.2.tar.gz" + + version('1.1.2', '34d4ad174acbff005c36d4d10e48cbb9') + version('1.1.1', '0ceff308c38f37d5b5f6df3927451c27') + version('1.1.0', '9f2d5f72e935c54f4c6d23e954ce699f') + version('1.0.0', 'd1bacca70a95cec0ddbff68b0871296b') + version('0.9.1', '8872d4444ff274ce6cd1ed364d0fc0ad') + + patch("Werror.patch") + + depends_on("cmake @2.6.2:") + + def install(self, spec, prefix): + # Don't use -Werror + filter_file(r'-Werror', '', 'CMakeLists.txt') + + # configure + # Boost is only used for self-tests, which we are not running (yet?) + cmake('.', '-DCMAKE_DISABLE_FIND_PACKAGE_Boost=TRUE', *std_cmake_args) + + # Build + make() + + # Install + shutil.rmtree(join_path(prefix, 'doc'), ignore_errors=True) + shutil.rmtree(join_path(prefix, 'include'), ignore_errors=True) + shutil.copytree('doc', join_path(prefix, 'doc'), symlinks=True) + shutil.copytree('include', join_path(prefix, 'include'), symlinks=True) From 244ee673d67794806e8259fdc379c83950161a2a Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 13 Jan 2016 11:44:51 -0600 Subject: [PATCH 301/910] Add latest version --- var/spack/packages/mpich/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py index 7cfa0a3b619..00b7dfda751 100644 --- a/var/spack/packages/mpich/package.py +++ b/var/spack/packages/mpich/package.py @@ -33,11 +33,12 @@ class Mpich(Package): list_url = "http://www.mpich.org/static/downloads/" list_depth = 2 + version('3.2', 'f414cfa77099cd1fa1a5ae4e22db508a') version('3.1.4', '2ab544607986486562e076b83937bba2') version('3.1.3', '93cb17f91ac758cbf9174ecb03563778') version('3.1.2', '7fbf4b81dcb74b07ae85939d1ceee7f1') version('3.1.1', '40dc408b1e03cc36d80209baaa2d32b7') - version('3.1', '5643dd176499bfb7d25079aaff25f2ec') + version('3.1', '5643dd176499bfb7d25079aaff25f2ec') version('3.0.4', '9c5d5d4fe1e17dd12153f40bc5b6dbc0') provides('mpi@:3.0', when='@3:') From f9fbb57d3182bf0e9c39e4b7c0a958c6ba87a2fa Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 13 Jan 2016 12:32:04 -0600 Subject: [PATCH 302/910] Add NAG Fortran Compiler support --- lib/spack/env/cc | 4 ++-- lib/spack/spack/compilers/__init__.py | 2 +- lib/spack/spack/compilers/nag.py | 24 ++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 lib/spack/spack/compilers/nag.py diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 0966277a91e..aacba996b39 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -94,11 +94,11 @@ case "$command" in command="$SPACK_CXX" language="C++" ;; - f90|fc|f95|gfortran|ifort|pgf90|xlf90) + f90|fc|f95|gfortran|ifort|pgf90|xlf90|nagfor) command="$SPACK_FC" language="Fortran 90" ;; - f77|gfortran|ifort|pgf77|xlf) + f77|gfortran|ifort|pgf77|xlf|nagfor) command="$SPACK_F77" language="Fortran 77" ;; diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 66e608cf79f..a6198fc1ce2 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -45,7 +45,7 @@ _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] -_default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] +_default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc', 'nag'] def _auto_compiler_spec(function): def converter(cspec_like): diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py new file mode 100644 index 00000000000..2dfc97af739 --- /dev/null +++ b/lib/spack/spack/compilers/nag.py @@ -0,0 +1,24 @@ +class Nag(Compiler): + # Subclasses use possible names of C compiler + cc_names = [] + + # Subclasses use possible names of C++ compiler + cxx_names = [] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['nagfor'] + + # Subclasses use possible names of Fortran 90 compiler + fc_names = ['nagfor'] + + @classmethod + def default_version(self, comp): + """The '-V' option works for nag compilers. + Output looks like this:: + + NAG Fortran Compiler Release 6.0(Hibiya) Build 1037 + Product NPL6A60NA for x86-64 Linux + Copyright 1990-2015 The Numerical Algorithms Group Ltd., Oxford, U.K. + """ + return get_compiler_version( + comp, '-V', r'NAG Fortran Compiler Release ([0-9.]+)') From 05b30bf83e0bc8723472b6d609c692c3b83b486c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 14 Jan 2016 10:26:31 -0800 Subject: [PATCH 303/910] Make text wrapping off by default in tty, add a kwarg for it. --- lib/spack/llnl/util/tty/__init__.py | 39 ++++++++++++++++++---------- lib/spack/spack/config.py | 40 ++++++++++++++++++++++++----- lib/spack/spack/package.py | 2 +- 3 files changed, 60 insertions(+), 21 deletions(-) diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index 203f429a481..3ecd3a4ac2e 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -63,35 +63,46 @@ def msg(message, *args): def info(message, *args, **kwargs): format = kwargs.get('format', '*b') stream = kwargs.get('stream', sys.stdout) + wrap = kwargs.get('wrap', False) cprint("@%s{==>} %s" % (format, cescape(str(message))), stream=stream) for arg in args: - lines = textwrap.wrap( - str(arg), initial_indent=indent, subsequent_indent=indent) - for line in lines: - stream.write(line + '\n') + if wrap: + lines = textwrap.wrap( + str(arg), initial_indent=indent, subsequent_indent=indent) + for line in lines: + stream.write(line + '\n') + else: + stream.write(indent + str(arg) + '\n') -def verbose(message, *args): +def verbose(message, *args, **kwargs): if _verbose: - info(message, *args, format='c') + kwargs.setdefault('format', 'c') + info(message, *args, **kwargs) -def debug(message, *args): +def debug(message, *args, **kwargs): if _debug: - info(message, *args, format='g', stream=sys.stderr) + kwargs.setdefault('format', 'g') + kwargs.setdefault('stream', sys.stderr) + info(message, *args, **kwargs) -def error(message, *args): - info("Error: " + str(message), *args, format='*r', stream=sys.stderr) +def error(message, *args, **kwargs): + kwargs.setdefault('format', '*r') + kwargs.setdefault('stream', sys.stderr) + info("Error: " + str(message), *args, **kwargs) -def warn(message, *args): - info("Warning: " + str(message), *args, format='*Y', stream=sys.stderr) +def warn(message, *args, **kwargs): + kwargs.setdefault('format', '*Y') + kwargs.setdefault('stream', sys.stderr) + info("Warning: " + str(message), *args, **kwargs) -def die(message, *args): - error(message, *args) +def die(message, *args, **kwargs): + error(message, *args, **kwargs) sys.exit(1) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index c53dcbc4050..3ff83ae529b 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -135,8 +135,34 @@ import spack.util.spack_yaml as syaml -"""Dict from section names -> function to check section YAML format.""" -valid_sections = ['compilers', 'mirrors', 'repos'] +"""Dict from section names -> schema for that section.""" +section_schemas = { + 'compilers' : { + '$schema': 'http://json-schema.org/schema#', + 'title' : 'Spack compiler configuration file schema', + 'type' : 'object', + 'properties' : { + 'compilers' : { + 'type' : 'map', + }, + }, + }, + + 'mirrors' : { + '$schema': 'http://json-schema.org/schema#', + 'title' : 'Spack mirror configuration file schema', + 'type' : 'map', + 'properties' : { + 'mirrors' : { + + } + }, + }, + + 'repos' : { + '$schema': 'http://json-schema.org/schema#', + 'title' : 'Spack repository configuration file schema', + }} """OrderedDict of config scopes keyed by name. Later scopes will override earlier scopes. @@ -146,9 +172,9 @@ def validate_section(section): """Raise a ValueError if the section is not a valid section.""" - if section not in valid_sections: + if section not in section_schemas: raise ValueError("Invalid config section: '%s'. Options are %s." - % (section, valid_sections)) + % (section, section_schemas)) class ConfigScope(object): @@ -369,10 +395,12 @@ def update_config(section, update_data, scope=None): scope.write_section(section) -"""Print a configuration to stdout""" def print_section(section): + """Print a configuration to stdout.""" try: - yaml.dump(get_config(section), stream=sys.stdout, default_flow_style=False) + data = syaml.syaml_dict() + data[section] = get_config(section) + syaml.dump(data, stream=sys.stdout, default_flow_style=False) except (yaml.YAMLError, IOError) as e: raise ConfigError("Error reading configuration: %s" % section) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 5db83064b58..f926a14206d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -858,7 +858,7 @@ def cleanup(): tty.warn("Keeping install prefix in place despite error.", "Spack will think this package is installed." + "Manually remove this directory to fix:", - self.prefix) + self.prefix, wrap=True) def real_work(): From b567cb57e1c74dcb876b0ed935295a848ec45438 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 14 Jan 2016 10:40:34 -0800 Subject: [PATCH 304/910] Add jsonschema validation for config files. --- lib/spack/external/jsonschema/COPYING | 19 + lib/spack/external/jsonschema/README.rst | 104 +++ lib/spack/external/jsonschema/__init__.py | 26 + lib/spack/external/jsonschema/__main__.py | 2 + lib/spack/external/jsonschema/_format.py | 240 ++++++ lib/spack/external/jsonschema/_reflect.py | 155 ++++ lib/spack/external/jsonschema/_utils.py | 213 +++++ lib/spack/external/jsonschema/_validators.py | 358 ++++++++ lib/spack/external/jsonschema/cli.py | 72 ++ lib/spack/external/jsonschema/compat.py | 53 ++ lib/spack/external/jsonschema/exceptions.py | 264 ++++++ .../external/jsonschema/schemas/draft3.json | 201 +++++ .../external/jsonschema/schemas/draft4.json | 221 +++++ .../external/jsonschema/tests/__init__.py | 0 lib/spack/external/jsonschema/tests/compat.py | 15 + .../external/jsonschema/tests/test_cli.py | 110 +++ .../jsonschema/tests/test_exceptions.py | 382 +++++++++ .../external/jsonschema/tests/test_format.py | 63 ++ .../tests/test_jsonschema_test_suite.py | 290 +++++++ .../jsonschema/tests/test_validators.py | 786 ++++++++++++++++++ lib/spack/external/jsonschema/validators.py | 428 ++++++++++ lib/spack/spack/__init__.py | 5 +- lib/spack/spack/cmd/repo.py | 5 +- lib/spack/spack/config.py | 195 ++++- 24 files changed, 4166 insertions(+), 41 deletions(-) create mode 100644 lib/spack/external/jsonschema/COPYING create mode 100644 lib/spack/external/jsonschema/README.rst create mode 100644 lib/spack/external/jsonschema/__init__.py create mode 100644 lib/spack/external/jsonschema/__main__.py create mode 100644 lib/spack/external/jsonschema/_format.py create mode 100644 lib/spack/external/jsonschema/_reflect.py create mode 100644 lib/spack/external/jsonschema/_utils.py create mode 100644 lib/spack/external/jsonschema/_validators.py create mode 100644 lib/spack/external/jsonschema/cli.py create mode 100644 lib/spack/external/jsonschema/compat.py create mode 100644 lib/spack/external/jsonschema/exceptions.py create mode 100644 lib/spack/external/jsonschema/schemas/draft3.json create mode 100644 lib/spack/external/jsonschema/schemas/draft4.json create mode 100644 lib/spack/external/jsonschema/tests/__init__.py create mode 100644 lib/spack/external/jsonschema/tests/compat.py create mode 100644 lib/spack/external/jsonschema/tests/test_cli.py create mode 100644 lib/spack/external/jsonschema/tests/test_exceptions.py create mode 100644 lib/spack/external/jsonschema/tests/test_format.py create mode 100644 lib/spack/external/jsonschema/tests/test_jsonschema_test_suite.py create mode 100644 lib/spack/external/jsonschema/tests/test_validators.py create mode 100644 lib/spack/external/jsonschema/validators.py diff --git a/lib/spack/external/jsonschema/COPYING b/lib/spack/external/jsonschema/COPYING new file mode 100644 index 00000000000..af9cfbdb134 --- /dev/null +++ b/lib/spack/external/jsonschema/COPYING @@ -0,0 +1,19 @@ +Copyright (c) 2013 Julian Berman + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/lib/spack/external/jsonschema/README.rst b/lib/spack/external/jsonschema/README.rst new file mode 100644 index 00000000000..20c2fe62667 --- /dev/null +++ b/lib/spack/external/jsonschema/README.rst @@ -0,0 +1,104 @@ +========== +jsonschema +========== + +``jsonschema`` is an implementation of `JSON Schema `_ +for Python (supporting 2.6+ including Python 3). + +.. code-block:: python + + >>> from jsonschema import validate + + >>> # A sample schema, like what we'd get from json.load() + >>> schema = { + ... "type" : "object", + ... "properties" : { + ... "price" : {"type" : "number"}, + ... "name" : {"type" : "string"}, + ... }, + ... } + + >>> # If no exception is raised by validate(), the instance is valid. + >>> validate({"name" : "Eggs", "price" : 34.99}, schema) + + >>> validate( + ... {"name" : "Eggs", "price" : "Invalid"}, schema + ... ) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValidationError: 'Invalid' is not of type 'number' + + +Features +-------- + +* Full support for + `Draft 3 `_ + **and** `Draft 4 `_ + of the schema. + +* `Lazy validation `_ + that can iteratively report *all* validation errors. + +* Small and extensible + +* `Programmatic querying `_ + of which properties or items failed validation. + + +Release Notes +------------- + +* A simple CLI was added for validation +* Validation errors now keep full absolute paths and absolute schema paths in + their ``absolute_path`` and ``absolute_schema_path`` attributes. The ``path`` + and ``schema_path`` attributes are deprecated in favor of ``relative_path`` + and ``relative_schema_path``\ . + +*Note:* Support for Python 3.2 was dropped in this release, and installation +now uses setuptools. + + +Running the Test Suite +---------------------- + +``jsonschema`` uses the wonderful `Tox `_ for its +test suite. (It really is wonderful, if for some reason you haven't heard of +it, you really should use it for your projects). + +Assuming you have ``tox`` installed (perhaps via ``pip install tox`` or your +package manager), just run ``tox`` in the directory of your source checkout to +run ``jsonschema``'s test suite on all of the versions of Python ``jsonschema`` +supports. Note that you'll need to have all of those versions installed in +order to run the tests on each of them, otherwise ``tox`` will skip (and fail) +the tests on that version. + +Of course you're also free to just run the tests on a single version with your +favorite test runner. The tests live in the ``jsonschema.tests`` package. + + +Community +--------- + +There's a `mailing list `_ +for this implementation on Google Groups. + +Please join, and feel free to send questions there. + + +Contributing +------------ + +I'm Julian Berman. + +``jsonschema`` is on `GitHub `_. + +Get in touch, via GitHub or otherwise, if you've got something to contribute, +it'd be most welcome! + +You can also generally find me on Freenode (nick: ``tos9``) in various +channels, including ``#python``. + +If you feel overwhelmingly grateful, you can woo me with beer money on +`Gittip `_ or via Google Wallet with the email +in my GitHub profile. diff --git a/lib/spack/external/jsonschema/__init__.py b/lib/spack/external/jsonschema/__init__.py new file mode 100644 index 00000000000..6c099f1d8bf --- /dev/null +++ b/lib/spack/external/jsonschema/__init__.py @@ -0,0 +1,26 @@ +""" +An implementation of JSON Schema for Python + +The main functionality is provided by the validator classes for each of the +supported JSON Schema versions. + +Most commonly, :func:`validate` is the quickest way to simply validate a given +instance under a schema, and will create a validator for you. + +""" + +from jsonschema.exceptions import ( + ErrorTree, FormatError, RefResolutionError, SchemaError, ValidationError +) +from jsonschema._format import ( + FormatChecker, draft3_format_checker, draft4_format_checker, +) +from jsonschema.validators import ( + Draft3Validator, Draft4Validator, RefResolver, validate +) + + +__version__ = "2.4.0" + + +# flake8: noqa diff --git a/lib/spack/external/jsonschema/__main__.py b/lib/spack/external/jsonschema/__main__.py new file mode 100644 index 00000000000..82c29fd39e7 --- /dev/null +++ b/lib/spack/external/jsonschema/__main__.py @@ -0,0 +1,2 @@ +from jsonschema.cli import main +main() diff --git a/lib/spack/external/jsonschema/_format.py b/lib/spack/external/jsonschema/_format.py new file mode 100644 index 00000000000..bb52d183ad6 --- /dev/null +++ b/lib/spack/external/jsonschema/_format.py @@ -0,0 +1,240 @@ +import datetime +import re +import socket + +from jsonschema.compat import str_types +from jsonschema.exceptions import FormatError + + +class FormatChecker(object): + """ + A ``format`` property checker. + + JSON Schema does not mandate that the ``format`` property actually do any + validation. If validation is desired however, instances of this class can + be hooked into validators to enable format validation. + + :class:`FormatChecker` objects always return ``True`` when asked about + formats that they do not know how to validate. + + To check a custom format using a function that takes an instance and + returns a ``bool``, use the :meth:`FormatChecker.checks` or + :meth:`FormatChecker.cls_checks` decorators. + + :argument iterable formats: the known formats to validate. This argument + can be used to limit which formats will be used + during validation. + + """ + + checkers = {} + + def __init__(self, formats=None): + if formats is None: + self.checkers = self.checkers.copy() + else: + self.checkers = dict((k, self.checkers[k]) for k in formats) + + def checks(self, format, raises=()): + """ + Register a decorated function as validating a new format. + + :argument str format: the format that the decorated function will check + :argument Exception raises: the exception(s) raised by the decorated + function when an invalid instance is found. The exception object + will be accessible as the :attr:`ValidationError.cause` attribute + of the resulting validation error. + + """ + + def _checks(func): + self.checkers[format] = (func, raises) + return func + return _checks + + cls_checks = classmethod(checks) + + def check(self, instance, format): + """ + Check whether the instance conforms to the given format. + + :argument instance: the instance to check + :type: any primitive type (str, number, bool) + :argument str format: the format that instance should conform to + :raises: :exc:`FormatError` if instance does not conform to format + + """ + + if format not in self.checkers: + return + + func, raises = self.checkers[format] + result, cause = None, None + try: + result = func(instance) + except raises as e: + cause = e + if not result: + raise FormatError( + "%r is not a %r" % (instance, format), cause=cause, + ) + + def conforms(self, instance, format): + """ + Check whether the instance conforms to the given format. + + :argument instance: the instance to check + :type: any primitive type (str, number, bool) + :argument str format: the format that instance should conform to + :rtype: bool + + """ + + try: + self.check(instance, format) + except FormatError: + return False + else: + return True + + +_draft_checkers = {"draft3": [], "draft4": []} + + +def _checks_drafts(both=None, draft3=None, draft4=None, raises=()): + draft3 = draft3 or both + draft4 = draft4 or both + + def wrap(func): + if draft3: + _draft_checkers["draft3"].append(draft3) + func = FormatChecker.cls_checks(draft3, raises)(func) + if draft4: + _draft_checkers["draft4"].append(draft4) + func = FormatChecker.cls_checks(draft4, raises)(func) + return func + return wrap + + +@_checks_drafts("email") +def is_email(instance): + if not isinstance(instance, str_types): + return True + return "@" in instance + + +_ipv4_re = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") + +@_checks_drafts(draft3="ip-address", draft4="ipv4") +def is_ipv4(instance): + if not isinstance(instance, str_types): + return True + if not _ipv4_re.match(instance): + return False + return all(0 <= int(component) <= 255 for component in instance.split(".")) + + +if hasattr(socket, "inet_pton"): + @_checks_drafts("ipv6", raises=socket.error) + def is_ipv6(instance): + if not isinstance(instance, str_types): + return True + return socket.inet_pton(socket.AF_INET6, instance) + + +_host_name_re = re.compile(r"^[A-Za-z0-9][A-Za-z0-9\.\-]{1,255}$") + +@_checks_drafts(draft3="host-name", draft4="hostname") +def is_host_name(instance): + if not isinstance(instance, str_types): + return True + if not _host_name_re.match(instance): + return False + components = instance.split(".") + for component in components: + if len(component) > 63: + return False + return True + + +try: + import rfc3987 +except ImportError: + pass +else: + @_checks_drafts("uri", raises=ValueError) + def is_uri(instance): + if not isinstance(instance, str_types): + return True + return rfc3987.parse(instance, rule="URI") + + +try: + import strict_rfc3339 +except ImportError: + try: + import isodate + except ImportError: + pass + else: + @_checks_drafts("date-time", raises=(ValueError, isodate.ISO8601Error)) + def is_date(instance): + if not isinstance(instance, str_types): + return True + return isodate.parse_datetime(instance) +else: + @_checks_drafts("date-time") + def is_date(instance): + if not isinstance(instance, str_types): + return True + return strict_rfc3339.validate_rfc3339(instance) + + +@_checks_drafts("regex", raises=re.error) +def is_regex(instance): + if not isinstance(instance, str_types): + return True + return re.compile(instance) + + +@_checks_drafts(draft3="date", raises=ValueError) +def is_date(instance): + if not isinstance(instance, str_types): + return True + return datetime.datetime.strptime(instance, "%Y-%m-%d") + + +@_checks_drafts(draft3="time", raises=ValueError) +def is_time(instance): + if not isinstance(instance, str_types): + return True + return datetime.datetime.strptime(instance, "%H:%M:%S") + + +try: + import webcolors +except ImportError: + pass +else: + def is_css_color_code(instance): + return webcolors.normalize_hex(instance) + + + @_checks_drafts(draft3="color", raises=(ValueError, TypeError)) + def is_css21_color(instance): + if ( + not isinstance(instance, str_types) or + instance.lower() in webcolors.css21_names_to_hex + ): + return True + return is_css_color_code(instance) + + + def is_css3_color(instance): + if instance.lower() in webcolors.css3_names_to_hex: + return True + return is_css_color_code(instance) + + +draft3_format_checker = FormatChecker(_draft_checkers["draft3"]) +draft4_format_checker = FormatChecker(_draft_checkers["draft4"]) diff --git a/lib/spack/external/jsonschema/_reflect.py b/lib/spack/external/jsonschema/_reflect.py new file mode 100644 index 00000000000..d09e38fbdcf --- /dev/null +++ b/lib/spack/external/jsonschema/_reflect.py @@ -0,0 +1,155 @@ +# -*- test-case-name: twisted.test.test_reflect -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Standardized versions of various cool and/or strange things that you can do +with Python's reflection capabilities. +""" + +import sys + +from jsonschema.compat import PY3 + + +class _NoModuleFound(Exception): + """ + No module was found because none exists. + """ + + + +class InvalidName(ValueError): + """ + The given name is not a dot-separated list of Python objects. + """ + + + +class ModuleNotFound(InvalidName): + """ + The module associated with the given name doesn't exist and it can't be + imported. + """ + + + +class ObjectNotFound(InvalidName): + """ + The object associated with the given name doesn't exist and it can't be + imported. + """ + + + +if PY3: + def reraise(exception, traceback): + raise exception.with_traceback(traceback) +else: + exec("""def reraise(exception, traceback): + raise exception.__class__, exception, traceback""") + +reraise.__doc__ = """ +Re-raise an exception, with an optional traceback, in a way that is compatible +with both Python 2 and Python 3. + +Note that on Python 3, re-raised exceptions will be mutated, with their +C{__traceback__} attribute being set. + +@param exception: The exception instance. +@param traceback: The traceback to use, or C{None} indicating a new traceback. +""" + + +def _importAndCheckStack(importName): + """ + Import the given name as a module, then walk the stack to determine whether + the failure was the module not existing, or some code in the module (for + example a dependent import) failing. This can be helpful to determine + whether any actual application code was run. For example, to distiguish + administrative error (entering the wrong module name), from programmer + error (writing buggy code in a module that fails to import). + + @param importName: The name of the module to import. + @type importName: C{str} + @raise Exception: if something bad happens. This can be any type of + exception, since nobody knows what loading some arbitrary code might + do. + @raise _NoModuleFound: if no module was found. + """ + try: + return __import__(importName) + except ImportError: + excType, excValue, excTraceback = sys.exc_info() + while excTraceback: + execName = excTraceback.tb_frame.f_globals["__name__"] + # in Python 2 execName is None when an ImportError is encountered, + # where in Python 3 execName is equal to the importName. + if execName is None or execName == importName: + reraise(excValue, excTraceback) + excTraceback = excTraceback.tb_next + raise _NoModuleFound() + + + +def namedAny(name): + """ + Retrieve a Python object by its fully qualified name from the global Python + module namespace. The first part of the name, that describes a module, + will be discovered and imported. Each subsequent part of the name is + treated as the name of an attribute of the object specified by all of the + name which came before it. For example, the fully-qualified name of this + object is 'twisted.python.reflect.namedAny'. + + @type name: L{str} + @param name: The name of the object to return. + + @raise InvalidName: If the name is an empty string, starts or ends with + a '.', or is otherwise syntactically incorrect. + + @raise ModuleNotFound: If the name is syntactically correct but the + module it specifies cannot be imported because it does not appear to + exist. + + @raise ObjectNotFound: If the name is syntactically correct, includes at + least one '.', but the module it specifies cannot be imported because + it does not appear to exist. + + @raise AttributeError: If an attribute of an object along the way cannot be + accessed, or a module along the way is not found. + + @return: the Python object identified by 'name'. + """ + if not name: + raise InvalidName('Empty module name') + + names = name.split('.') + + # if the name starts or ends with a '.' or contains '..', the __import__ + # will raise an 'Empty module name' error. This will provide a better error + # message. + if '' in names: + raise InvalidName( + "name must be a string giving a '.'-separated list of Python " + "identifiers, not %r" % (name,)) + + topLevelPackage = None + moduleNames = names[:] + while not topLevelPackage: + if moduleNames: + trialname = '.'.join(moduleNames) + try: + topLevelPackage = _importAndCheckStack(trialname) + except _NoModuleFound: + moduleNames.pop() + else: + if len(names) == 1: + raise ModuleNotFound("No module named %r" % (name,)) + else: + raise ObjectNotFound('%r does not name an object' % (name,)) + + obj = topLevelPackage + for n in names[1:]: + obj = getattr(obj, n) + + return obj diff --git a/lib/spack/external/jsonschema/_utils.py b/lib/spack/external/jsonschema/_utils.py new file mode 100644 index 00000000000..2262f3305db --- /dev/null +++ b/lib/spack/external/jsonschema/_utils.py @@ -0,0 +1,213 @@ +import itertools +import json +import pkgutil +import re + +from jsonschema.compat import str_types, MutableMapping, urlsplit + + +class URIDict(MutableMapping): + """ + Dictionary which uses normalized URIs as keys. + + """ + + def normalize(self, uri): + return urlsplit(uri).geturl() + + def __init__(self, *args, **kwargs): + self.store = dict() + self.store.update(*args, **kwargs) + + def __getitem__(self, uri): + return self.store[self.normalize(uri)] + + def __setitem__(self, uri, value): + self.store[self.normalize(uri)] = value + + def __delitem__(self, uri): + del self.store[self.normalize(uri)] + + def __iter__(self): + return iter(self.store) + + def __len__(self): + return len(self.store) + + def __repr__(self): + return repr(self.store) + + +class Unset(object): + """ + An as-of-yet unset attribute or unprovided default parameter. + + """ + + def __repr__(self): + return "" + + +def load_schema(name): + """ + Load a schema from ./schemas/``name``.json and return it. + + """ + + data = pkgutil.get_data(__package__, "schemas/{0}.json".format(name)) + return json.loads(data.decode("utf-8")) + + +def indent(string, times=1): + """ + A dumb version of :func:`textwrap.indent` from Python 3.3. + + """ + + return "\n".join(" " * (4 * times) + line for line in string.splitlines()) + + +def format_as_index(indices): + """ + Construct a single string containing indexing operations for the indices. + + For example, [1, 2, "foo"] -> [1][2]["foo"] + + :type indices: sequence + + """ + + if not indices: + return "" + return "[%s]" % "][".join(repr(index) for index in indices) + + +def find_additional_properties(instance, schema): + """ + Return the set of additional properties for the given ``instance``. + + Weeds out properties that should have been validated by ``properties`` and + / or ``patternProperties``. + + Assumes ``instance`` is dict-like already. + + """ + + properties = schema.get("properties", {}) + patterns = "|".join(schema.get("patternProperties", {})) + for property in instance: + if property not in properties: + if patterns and re.search(patterns, property): + continue + yield property + + +def extras_msg(extras): + """ + Create an error message for extra items or properties. + + """ + + if len(extras) == 1: + verb = "was" + else: + verb = "were" + return ", ".join(repr(extra) for extra in extras), verb + + +def types_msg(instance, types): + """ + Create an error message for a failure to match the given types. + + If the ``instance`` is an object and contains a ``name`` property, it will + be considered to be a description of that object and used as its type. + + Otherwise the message is simply the reprs of the given ``types``. + + """ + + reprs = [] + for type in types: + try: + reprs.append(repr(type["name"])) + except Exception: + reprs.append(repr(type)) + return "%r is not of type %s" % (instance, ", ".join(reprs)) + + +def flatten(suitable_for_isinstance): + """ + isinstance() can accept a bunch of really annoying different types: + * a single type + * a tuple of types + * an arbitrary nested tree of tuples + + Return a flattened tuple of the given argument. + + """ + + types = set() + + if not isinstance(suitable_for_isinstance, tuple): + suitable_for_isinstance = (suitable_for_isinstance,) + for thing in suitable_for_isinstance: + if isinstance(thing, tuple): + types.update(flatten(thing)) + else: + types.add(thing) + return tuple(types) + + +def ensure_list(thing): + """ + Wrap ``thing`` in a list if it's a single str. + + Otherwise, return it unchanged. + + """ + + if isinstance(thing, str_types): + return [thing] + return thing + + +def unbool(element, true=object(), false=object()): + """ + A hack to make True and 1 and False and 0 unique for ``uniq``. + + """ + + if element is True: + return true + elif element is False: + return false + return element + + +def uniq(container): + """ + Check if all of a container's elements are unique. + + Successively tries first to rely that the elements are hashable, then + falls back on them being sortable, and finally falls back on brute + force. + + """ + + try: + return len(set(unbool(i) for i in container)) == len(container) + except TypeError: + try: + sort = sorted(unbool(i) for i in container) + sliced = itertools.islice(sort, 1, None) + for i, j in zip(sort, sliced): + if i == j: + return False + except (NotImplementedError, TypeError): + seen = [] + for e in container: + e = unbool(e) + if e in seen: + return False + seen.append(e) + return True diff --git a/lib/spack/external/jsonschema/_validators.py b/lib/spack/external/jsonschema/_validators.py new file mode 100644 index 00000000000..c6e801ccb23 --- /dev/null +++ b/lib/spack/external/jsonschema/_validators.py @@ -0,0 +1,358 @@ +import re + +from jsonschema import _utils +from jsonschema.exceptions import FormatError, ValidationError +from jsonschema.compat import iteritems + + +FLOAT_TOLERANCE = 10 ** -15 + + +def patternProperties(validator, patternProperties, instance, schema): + if not validator.is_type(instance, "object"): + return + + for pattern, subschema in iteritems(patternProperties): + for k, v in iteritems(instance): + if re.search(pattern, k): + for error in validator.descend( + v, subschema, path=k, schema_path=pattern, + ): + yield error + + +def additionalProperties(validator, aP, instance, schema): + if not validator.is_type(instance, "object"): + return + + extras = set(_utils.find_additional_properties(instance, schema)) + + if validator.is_type(aP, "object"): + for extra in extras: + for error in validator.descend(instance[extra], aP, path=extra): + yield error + elif not aP and extras: + error = "Additional properties are not allowed (%s %s unexpected)" + yield ValidationError(error % _utils.extras_msg(extras)) + + +def items(validator, items, instance, schema): + if not validator.is_type(instance, "array"): + return + + if validator.is_type(items, "object"): + for index, item in enumerate(instance): + for error in validator.descend(item, items, path=index): + yield error + else: + for (index, item), subschema in zip(enumerate(instance), items): + for error in validator.descend( + item, subschema, path=index, schema_path=index, + ): + yield error + + +def additionalItems(validator, aI, instance, schema): + if ( + not validator.is_type(instance, "array") or + validator.is_type(schema.get("items", {}), "object") + ): + return + + len_items = len(schema.get("items", [])) + if validator.is_type(aI, "object"): + for index, item in enumerate(instance[len_items:], start=len_items): + for error in validator.descend(item, aI, path=index): + yield error + elif not aI and len(instance) > len(schema.get("items", [])): + error = "Additional items are not allowed (%s %s unexpected)" + yield ValidationError( + error % + _utils.extras_msg(instance[len(schema.get("items", [])):]) + ) + + +def minimum(validator, minimum, instance, schema): + if not validator.is_type(instance, "number"): + return + + if schema.get("exclusiveMinimum", False): + failed = float(instance) <= minimum + cmp = "less than or equal to" + else: + failed = float(instance) < minimum + cmp = "less than" + + if failed: + yield ValidationError( + "%r is %s the minimum of %r" % (instance, cmp, minimum) + ) + + +def maximum(validator, maximum, instance, schema): + if not validator.is_type(instance, "number"): + return + + if schema.get("exclusiveMaximum", False): + failed = instance >= maximum + cmp = "greater than or equal to" + else: + failed = instance > maximum + cmp = "greater than" + + if failed: + yield ValidationError( + "%r is %s the maximum of %r" % (instance, cmp, maximum) + ) + + +def multipleOf(validator, dB, instance, schema): + if not validator.is_type(instance, "number"): + return + + if isinstance(dB, float): + mod = instance % dB + failed = (mod > FLOAT_TOLERANCE) and (dB - mod) > FLOAT_TOLERANCE + else: + failed = instance % dB + + if failed: + yield ValidationError("%r is not a multiple of %r" % (instance, dB)) + + +def minItems(validator, mI, instance, schema): + if validator.is_type(instance, "array") and len(instance) < mI: + yield ValidationError("%r is too short" % (instance,)) + + +def maxItems(validator, mI, instance, schema): + if validator.is_type(instance, "array") and len(instance) > mI: + yield ValidationError("%r is too long" % (instance,)) + + +def uniqueItems(validator, uI, instance, schema): + if ( + uI and + validator.is_type(instance, "array") and + not _utils.uniq(instance) + ): + yield ValidationError("%r has non-unique elements" % instance) + + +def pattern(validator, patrn, instance, schema): + if ( + validator.is_type(instance, "string") and + not re.search(patrn, instance) + ): + yield ValidationError("%r does not match %r" % (instance, patrn)) + + +def format(validator, format, instance, schema): + if validator.format_checker is not None: + try: + validator.format_checker.check(instance, format) + except FormatError as error: + yield ValidationError(error.message, cause=error.cause) + + +def minLength(validator, mL, instance, schema): + if validator.is_type(instance, "string") and len(instance) < mL: + yield ValidationError("%r is too short" % (instance,)) + + +def maxLength(validator, mL, instance, schema): + if validator.is_type(instance, "string") and len(instance) > mL: + yield ValidationError("%r is too long" % (instance,)) + + +def dependencies(validator, dependencies, instance, schema): + if not validator.is_type(instance, "object"): + return + + for property, dependency in iteritems(dependencies): + if property not in instance: + continue + + if validator.is_type(dependency, "object"): + for error in validator.descend( + instance, dependency, schema_path=property, + ): + yield error + else: + dependencies = _utils.ensure_list(dependency) + for dependency in dependencies: + if dependency not in instance: + yield ValidationError( + "%r is a dependency of %r" % (dependency, property) + ) + + +def enum(validator, enums, instance, schema): + if instance not in enums: + yield ValidationError("%r is not one of %r" % (instance, enums)) + + +def ref(validator, ref, instance, schema): + with validator.resolver.resolving(ref) as resolved: + for error in validator.descend(instance, resolved): + yield error + + +def type_draft3(validator, types, instance, schema): + types = _utils.ensure_list(types) + + all_errors = [] + for index, type in enumerate(types): + if type == "any": + return + if validator.is_type(type, "object"): + errors = list(validator.descend(instance, type, schema_path=index)) + if not errors: + return + all_errors.extend(errors) + else: + if validator.is_type(instance, type): + return + else: + yield ValidationError( + _utils.types_msg(instance, types), context=all_errors, + ) + + +def properties_draft3(validator, properties, instance, schema): + if not validator.is_type(instance, "object"): + return + + for property, subschema in iteritems(properties): + if property in instance: + for error in validator.descend( + instance[property], + subschema, + path=property, + schema_path=property, + ): + yield error + elif subschema.get("required", False): + error = ValidationError("%r is a required property" % property) + error._set( + validator="required", + validator_value=subschema["required"], + instance=instance, + schema=schema, + ) + error.path.appendleft(property) + error.schema_path.extend([property, "required"]) + yield error + + +def disallow_draft3(validator, disallow, instance, schema): + for disallowed in _utils.ensure_list(disallow): + if validator.is_valid(instance, {"type" : [disallowed]}): + yield ValidationError( + "%r is disallowed for %r" % (disallowed, instance) + ) + + +def extends_draft3(validator, extends, instance, schema): + if validator.is_type(extends, "object"): + for error in validator.descend(instance, extends): + yield error + return + for index, subschema in enumerate(extends): + for error in validator.descend(instance, subschema, schema_path=index): + yield error + + +def type_draft4(validator, types, instance, schema): + types = _utils.ensure_list(types) + + if not any(validator.is_type(instance, type) for type in types): + yield ValidationError(_utils.types_msg(instance, types)) + + +def properties_draft4(validator, properties, instance, schema): + if not validator.is_type(instance, "object"): + return + + for property, subschema in iteritems(properties): + if property in instance: + for error in validator.descend( + instance[property], + subschema, + path=property, + schema_path=property, + ): + yield error + + +def required_draft4(validator, required, instance, schema): + if not validator.is_type(instance, "object"): + return + for property in required: + if property not in instance: + yield ValidationError("%r is a required property" % property) + + +def minProperties_draft4(validator, mP, instance, schema): + if validator.is_type(instance, "object") and len(instance) < mP: + yield ValidationError( + "%r does not have enough properties" % (instance,) + ) + + +def maxProperties_draft4(validator, mP, instance, schema): + if not validator.is_type(instance, "object"): + return + if validator.is_type(instance, "object") and len(instance) > mP: + yield ValidationError("%r has too many properties" % (instance,)) + + +def allOf_draft4(validator, allOf, instance, schema): + for index, subschema in enumerate(allOf): + for error in validator.descend(instance, subschema, schema_path=index): + yield error + + +def oneOf_draft4(validator, oneOf, instance, schema): + subschemas = enumerate(oneOf) + all_errors = [] + for index, subschema in subschemas: + errs = list(validator.descend(instance, subschema, schema_path=index)) + if not errs: + first_valid = subschema + break + all_errors.extend(errs) + else: + yield ValidationError( + "%r is not valid under any of the given schemas" % (instance,), + context=all_errors, + ) + + more_valid = [s for i, s in subschemas if validator.is_valid(instance, s)] + if more_valid: + more_valid.append(first_valid) + reprs = ", ".join(repr(schema) for schema in more_valid) + yield ValidationError( + "%r is valid under each of %s" % (instance, reprs) + ) + + +def anyOf_draft4(validator, anyOf, instance, schema): + all_errors = [] + for index, subschema in enumerate(anyOf): + errs = list(validator.descend(instance, subschema, schema_path=index)) + if not errs: + break + all_errors.extend(errs) + else: + yield ValidationError( + "%r is not valid under any of the given schemas" % (instance,), + context=all_errors, + ) + + +def not_draft4(validator, not_schema, instance, schema): + if validator.is_valid(instance, not_schema): + yield ValidationError( + "%r is not allowed for %r" % (not_schema, instance) + ) diff --git a/lib/spack/external/jsonschema/cli.py b/lib/spack/external/jsonschema/cli.py new file mode 100644 index 00000000000..0126564f463 --- /dev/null +++ b/lib/spack/external/jsonschema/cli.py @@ -0,0 +1,72 @@ +from __future__ import absolute_import +import argparse +import json +import sys + +from jsonschema._reflect import namedAny +from jsonschema.validators import validator_for + + +def _namedAnyWithDefault(name): + if "." not in name: + name = "jsonschema." + name + return namedAny(name) + + +def _json_file(path): + with open(path) as file: + return json.load(file) + + +parser = argparse.ArgumentParser( + description="JSON Schema Validation CLI", +) +parser.add_argument( + "-i", "--instance", + action="append", + dest="instances", + type=_json_file, + help="a path to a JSON instance to validate " + "(may be specified multiple times)", +) +parser.add_argument( + "-F", "--error-format", + default="{error.instance}: {error.message}\n", + help="the format to use for each error output message, specified in " + "a form suitable for passing to str.format, which will be called " + "with 'error' for each error", +) +parser.add_argument( + "-V", "--validator", + type=_namedAnyWithDefault, + help="the fully qualified object name of a validator to use, or, for " + "validators that are registered with jsonschema, simply the name " + "of the class.", +) +parser.add_argument( + "schema", + help="the JSON Schema to validate with", + type=_json_file, +) + + +def parse_args(args): + arguments = vars(parser.parse_args(args=args or ["--help"])) + if arguments["validator"] is None: + arguments["validator"] = validator_for(arguments["schema"]) + return arguments + + +def main(args=sys.argv[1:]): + sys.exit(run(arguments=parse_args(args=args))) + + +def run(arguments, stdout=sys.stdout, stderr=sys.stderr): + error_format = arguments["error_format"] + validator = arguments["validator"](schema=arguments["schema"]) + errored = False + for instance in arguments["instances"] or (): + for error in validator.iter_errors(instance): + stderr.write(error_format.format(error=error)) + errored = True + return errored diff --git a/lib/spack/external/jsonschema/compat.py b/lib/spack/external/jsonschema/compat.py new file mode 100644 index 00000000000..6ca49ab6be9 --- /dev/null +++ b/lib/spack/external/jsonschema/compat.py @@ -0,0 +1,53 @@ +from __future__ import unicode_literals +import sys +import operator + +try: + from collections import MutableMapping, Sequence # noqa +except ImportError: + from collections.abc import MutableMapping, Sequence # noqa + +PY3 = sys.version_info[0] >= 3 + +if PY3: + zip = zip + from io import StringIO + from urllib.parse import ( + unquote, urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit + ) + from urllib.request import urlopen + str_types = str, + int_types = int, + iteritems = operator.methodcaller("items") +else: + from itertools import izip as zip # noqa + from StringIO import StringIO + from urlparse import ( + urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit # noqa + ) + from urllib import unquote # noqa + from urllib2 import urlopen # noqa + str_types = basestring + int_types = int, long + iteritems = operator.methodcaller("iteritems") + + +# On python < 3.3 fragments are not handled properly with unknown schemes +def urlsplit(url): + scheme, netloc, path, query, fragment = _urlsplit(url) + if "#" in path: + path, fragment = path.split("#", 1) + return SplitResult(scheme, netloc, path, query, fragment) + + +def urldefrag(url): + if "#" in url: + s, n, p, q, frag = urlsplit(url) + defrag = urlunsplit((s, n, p, q, '')) + else: + defrag = url + frag = '' + return defrag, frag + + +# flake8: noqa diff --git a/lib/spack/external/jsonschema/exceptions.py b/lib/spack/external/jsonschema/exceptions.py new file mode 100644 index 00000000000..478e59c531f --- /dev/null +++ b/lib/spack/external/jsonschema/exceptions.py @@ -0,0 +1,264 @@ +from collections import defaultdict, deque +import itertools +import pprint +import textwrap + +from jsonschema import _utils +from jsonschema.compat import PY3, iteritems + + +WEAK_MATCHES = frozenset(["anyOf", "oneOf"]) +STRONG_MATCHES = frozenset() + +_unset = _utils.Unset() + + +class _Error(Exception): + def __init__( + self, + message, + validator=_unset, + path=(), + cause=None, + context=(), + validator_value=_unset, + instance=_unset, + schema=_unset, + schema_path=(), + parent=None, + ): + self.message = message + self.path = self.relative_path = deque(path) + self.schema_path = self.relative_schema_path = deque(schema_path) + self.context = list(context) + self.cause = self.__cause__ = cause + self.validator = validator + self.validator_value = validator_value + self.instance = instance + self.schema = schema + self.parent = parent + + for error in context: + error.parent = self + + def __repr__(self): + return "<%s: %r>" % (self.__class__.__name__, self.message) + + def __str__(self): + return unicode(self).encode("utf-8") + + def __unicode__(self): + essential_for_verbose = ( + self.validator, self.validator_value, self.instance, self.schema, + ) + if any(m is _unset for m in essential_for_verbose): + return self.message + + pschema = pprint.pformat(self.schema, width=72) + pinstance = pprint.pformat(self.instance, width=72) + return self.message + textwrap.dedent(""" + + Failed validating %r in schema%s: + %s + + On instance%s: + %s + """.rstrip() + ) % ( + self.validator, + _utils.format_as_index(list(self.relative_schema_path)[:-1]), + _utils.indent(pschema), + _utils.format_as_index(self.relative_path), + _utils.indent(pinstance), + ) + + if PY3: + __str__ = __unicode__ + + @classmethod + def create_from(cls, other): + return cls(**other._contents()) + + @property + def absolute_path(self): + parent = self.parent + if parent is None: + return self.relative_path + + path = deque(self.relative_path) + path.extendleft(parent.absolute_path) + return path + + @property + def absolute_schema_path(self): + parent = self.parent + if parent is None: + return self.relative_schema_path + + path = deque(self.relative_schema_path) + path.extendleft(parent.absolute_schema_path) + return path + + def _set(self, **kwargs): + for k, v in iteritems(kwargs): + if getattr(self, k) is _unset: + setattr(self, k, v) + + def _contents(self): + attrs = ( + "message", "cause", "context", "validator", "validator_value", + "path", "schema_path", "instance", "schema", "parent", + ) + return dict((attr, getattr(self, attr)) for attr in attrs) + + +class ValidationError(_Error): + pass + + +class SchemaError(_Error): + pass + + +class RefResolutionError(Exception): + pass + + +class UnknownType(Exception): + def __init__(self, type, instance, schema): + self.type = type + self.instance = instance + self.schema = schema + + def __str__(self): + return unicode(self).encode("utf-8") + + def __unicode__(self): + pschema = pprint.pformat(self.schema, width=72) + pinstance = pprint.pformat(self.instance, width=72) + return textwrap.dedent(""" + Unknown type %r for validator with schema: + %s + + While checking instance: + %s + """.rstrip() + ) % (self.type, _utils.indent(pschema), _utils.indent(pinstance)) + + if PY3: + __str__ = __unicode__ + + + +class FormatError(Exception): + def __init__(self, message, cause=None): + super(FormatError, self).__init__(message, cause) + self.message = message + self.cause = self.__cause__ = cause + + def __str__(self): + return self.message.encode("utf-8") + + def __unicode__(self): + return self.message + + if PY3: + __str__ = __unicode__ + + +class ErrorTree(object): + """ + ErrorTrees make it easier to check which validations failed. + + """ + + _instance = _unset + + def __init__(self, errors=()): + self.errors = {} + self._contents = defaultdict(self.__class__) + + for error in errors: + container = self + for element in error.path: + container = container[element] + container.errors[error.validator] = error + + self._instance = error.instance + + def __contains__(self, index): + """ + Check whether ``instance[index]`` has any errors. + + """ + + return index in self._contents + + def __getitem__(self, index): + """ + Retrieve the child tree one level down at the given ``index``. + + If the index is not in the instance that this tree corresponds to and + is not known by this tree, whatever error would be raised by + ``instance.__getitem__`` will be propagated (usually this is some + subclass of :class:`LookupError`. + + """ + + if self._instance is not _unset and index not in self: + self._instance[index] + return self._contents[index] + + def __setitem__(self, index, value): + self._contents[index] = value + + def __iter__(self): + """ + Iterate (non-recursively) over the indices in the instance with errors. + + """ + + return iter(self._contents) + + def __len__(self): + """ + Same as :attr:`total_errors`. + + """ + + return self.total_errors + + def __repr__(self): + return "<%s (%s total errors)>" % (self.__class__.__name__, len(self)) + + @property + def total_errors(self): + """ + The total number of errors in the entire tree, including children. + + """ + + child_errors = sum(len(tree) for _, tree in iteritems(self._contents)) + return len(self.errors) + child_errors + + +def by_relevance(weak=WEAK_MATCHES, strong=STRONG_MATCHES): + def relevance(error): + validator = error.validator + return -len(error.path), validator not in weak, validator in strong + return relevance + + +relevance = by_relevance() + + +def best_match(errors, key=relevance): + errors = iter(errors) + best = next(errors, None) + if best is None: + return + best = max(itertools.chain([best], errors), key=key) + + while best.context: + best = min(best.context, key=key) + return best diff --git a/lib/spack/external/jsonschema/schemas/draft3.json b/lib/spack/external/jsonschema/schemas/draft3.json new file mode 100644 index 00000000000..5bcefe30d5c --- /dev/null +++ b/lib/spack/external/jsonschema/schemas/draft3.json @@ -0,0 +1,201 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema#", + "dependencies": { + "exclusiveMaximum": "maximum", + "exclusiveMinimum": "minimum" + }, + "id": "http://json-schema.org/draft-03/schema#", + "properties": { + "$ref": { + "format": "uri", + "type": "string" + }, + "$schema": { + "format": "uri", + "type": "string" + }, + "additionalItems": { + "default": {}, + "type": [ + { + "$ref": "#" + }, + "boolean" + ] + }, + "additionalProperties": { + "default": {}, + "type": [ + { + "$ref": "#" + }, + "boolean" + ] + }, + "default": { + "type": "any" + }, + "dependencies": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": [ + "string", + "array", + { + "$ref": "#" + } + ] + }, + "default": {}, + "type": [ + "string", + "array", + "object" + ] + }, + "description": { + "type": "string" + }, + "disallow": { + "items": { + "type": [ + "string", + { + "$ref": "#" + } + ] + }, + "type": [ + "string", + "array" + ], + "uniqueItems": true + }, + "divisibleBy": { + "default": 1, + "exclusiveMinimum": true, + "minimum": 0, + "type": "number" + }, + "enum": { + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "exclusiveMaximum": { + "default": false, + "type": "boolean" + }, + "exclusiveMinimum": { + "default": false, + "type": "boolean" + }, + "extends": { + "default": {}, + "items": { + "$ref": "#" + }, + "type": [ + { + "$ref": "#" + }, + "array" + ] + }, + "format": { + "type": "string" + }, + "id": { + "format": "uri", + "type": "string" + }, + "items": { + "default": {}, + "items": { + "$ref": "#" + }, + "type": [ + { + "$ref": "#" + }, + "array" + ] + }, + "maxDecimal": { + "minimum": 0, + "type": "number" + }, + "maxItems": { + "minimum": 0, + "type": "integer" + }, + "maxLength": { + "type": "integer" + }, + "maximum": { + "type": "number" + }, + "minItems": { + "default": 0, + "minimum": 0, + "type": "integer" + }, + "minLength": { + "default": 0, + "minimum": 0, + "type": "integer" + }, + "minimum": { + "type": "number" + }, + "pattern": { + "format": "regex", + "type": "string" + }, + "patternProperties": { + "additionalProperties": { + "$ref": "#" + }, + "default": {}, + "type": "object" + }, + "properties": { + "additionalProperties": { + "$ref": "#", + "type": "object" + }, + "default": {}, + "type": "object" + }, + "required": { + "default": false, + "type": "boolean" + }, + "title": { + "type": "string" + }, + "type": { + "default": "any", + "items": { + "type": [ + "string", + { + "$ref": "#" + } + ] + }, + "type": [ + "string", + "array" + ], + "uniqueItems": true + }, + "uniqueItems": { + "default": false, + "type": "boolean" + } + }, + "type": "object" +} diff --git a/lib/spack/external/jsonschema/schemas/draft4.json b/lib/spack/external/jsonschema/schemas/draft4.json new file mode 100644 index 00000000000..fead5cefab7 --- /dev/null +++ b/lib/spack/external/jsonschema/schemas/draft4.json @@ -0,0 +1,221 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "default": {}, + "definitions": { + "positiveInteger": { + "minimum": 0, + "type": "integer" + }, + "positiveIntegerDefault0": { + "allOf": [ + { + "$ref": "#/definitions/positiveInteger" + }, + { + "default": 0 + } + ] + }, + "schemaArray": { + "items": { + "$ref": "#" + }, + "minItems": 1, + "type": "array" + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "items": { + "type": "string" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + } + }, + "dependencies": { + "exclusiveMaximum": [ + "maximum" + ], + "exclusiveMinimum": [ + "minimum" + ] + }, + "description": "Core schema meta-schema", + "id": "http://json-schema.org/draft-04/schema#", + "properties": { + "$schema": { + "format": "uri", + "type": "string" + }, + "additionalItems": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#" + } + ], + "default": {} + }, + "additionalProperties": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#" + } + ], + "default": {} + }, + "allOf": { + "$ref": "#/definitions/schemaArray" + }, + "anyOf": { + "$ref": "#/definitions/schemaArray" + }, + "default": {}, + "definitions": { + "additionalProperties": { + "$ref": "#" + }, + "default": {}, + "type": "object" + }, + "dependencies": { + "additionalProperties": { + "anyOf": [ + { + "$ref": "#" + }, + { + "$ref": "#/definitions/stringArray" + } + ] + }, + "type": "object" + }, + "description": { + "type": "string" + }, + "enum": { + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "exclusiveMaximum": { + "default": false, + "type": "boolean" + }, + "exclusiveMinimum": { + "default": false, + "type": "boolean" + }, + "id": { + "format": "uri", + "type": "string" + }, + "items": { + "anyOf": [ + { + "$ref": "#" + }, + { + "$ref": "#/definitions/schemaArray" + } + ], + "default": {} + }, + "maxItems": { + "$ref": "#/definitions/positiveInteger" + }, + "maxLength": { + "$ref": "#/definitions/positiveInteger" + }, + "maxProperties": { + "$ref": "#/definitions/positiveInteger" + }, + "maximum": { + "type": "number" + }, + "minItems": { + "$ref": "#/definitions/positiveIntegerDefault0" + }, + "minLength": { + "$ref": "#/definitions/positiveIntegerDefault0" + }, + "minProperties": { + "$ref": "#/definitions/positiveIntegerDefault0" + }, + "minimum": { + "type": "number" + }, + "multipleOf": { + "exclusiveMinimum": true, + "minimum": 0, + "type": "number" + }, + "not": { + "$ref": "#" + }, + "oneOf": { + "$ref": "#/definitions/schemaArray" + }, + "pattern": { + "format": "regex", + "type": "string" + }, + "patternProperties": { + "additionalProperties": { + "$ref": "#" + }, + "default": {}, + "type": "object" + }, + "properties": { + "additionalProperties": { + "$ref": "#" + }, + "default": {}, + "type": "object" + }, + "required": { + "$ref": "#/definitions/stringArray" + }, + "title": { + "type": "string" + }, + "type": { + "anyOf": [ + { + "$ref": "#/definitions/simpleTypes" + }, + { + "items": { + "$ref": "#/definitions/simpleTypes" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + } + ] + }, + "uniqueItems": { + "default": false, + "type": "boolean" + } + }, + "type": "object" +} diff --git a/lib/spack/external/jsonschema/tests/__init__.py b/lib/spack/external/jsonschema/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/spack/external/jsonschema/tests/compat.py b/lib/spack/external/jsonschema/tests/compat.py new file mode 100644 index 00000000000..b37483f5ddf --- /dev/null +++ b/lib/spack/external/jsonschema/tests/compat.py @@ -0,0 +1,15 @@ +import sys + + +if sys.version_info[:2] < (2, 7): # pragma: no cover + import unittest2 as unittest +else: + import unittest + +try: + from unittest import mock +except ImportError: + import mock + + +# flake8: noqa diff --git a/lib/spack/external/jsonschema/tests/test_cli.py b/lib/spack/external/jsonschema/tests/test_cli.py new file mode 100644 index 00000000000..f625ca989d1 --- /dev/null +++ b/lib/spack/external/jsonschema/tests/test_cli.py @@ -0,0 +1,110 @@ +from jsonschema import Draft4Validator, ValidationError, cli +from jsonschema.compat import StringIO +from jsonschema.tests.compat import mock, unittest + + +def fake_validator(*errors): + errors = list(reversed(errors)) + + class FakeValidator(object): + def __init__(self, *args, **kwargs): + pass + + def iter_errors(self, instance): + if errors: + return errors.pop() + return [] + return FakeValidator + + +class TestParser(unittest.TestCase): + FakeValidator = fake_validator() + + def setUp(self): + mock_open = mock.mock_open() + patch_open = mock.patch.object(cli, "open", mock_open, create=True) + patch_open.start() + self.addCleanup(patch_open.stop) + + mock_json_load = mock.Mock() + mock_json_load.return_value = {} + patch_json_load = mock.patch("json.load") + patch_json_load.start() + self.addCleanup(patch_json_load.stop) + + def test_find_validator_by_fully_qualified_object_name(self): + arguments = cli.parse_args( + [ + "--validator", + "jsonschema.tests.test_cli.TestParser.FakeValidator", + "--instance", "foo.json", + "schema.json", + ] + ) + self.assertIs(arguments["validator"], self.FakeValidator) + + def test_find_validator_in_jsonschema(self): + arguments = cli.parse_args( + [ + "--validator", "Draft4Validator", + "--instance", "foo.json", + "schema.json", + ] + ) + self.assertIs(arguments["validator"], Draft4Validator) + + +class TestCLI(unittest.TestCase): + def test_successful_validation(self): + stdout, stderr = StringIO(), StringIO() + exit_code = cli.run( + { + "validator": fake_validator(), + "schema": {}, + "instances": [1], + "error_format": "{error.message}", + }, + stdout=stdout, + stderr=stderr, + ) + self.assertFalse(stdout.getvalue()) + self.assertFalse(stderr.getvalue()) + self.assertEqual(exit_code, 0) + + def test_unsuccessful_validation(self): + error = ValidationError("I am an error!", instance=1) + stdout, stderr = StringIO(), StringIO() + exit_code = cli.run( + { + "validator": fake_validator([error]), + "schema": {}, + "instances": [1], + "error_format": "{error.instance} - {error.message}", + }, + stdout=stdout, + stderr=stderr, + ) + self.assertFalse(stdout.getvalue()) + self.assertEqual(stderr.getvalue(), "1 - I am an error!") + self.assertEqual(exit_code, 1) + + def test_unsuccessful_validation_multiple_instances(self): + first_errors = [ + ValidationError("9", instance=1), + ValidationError("8", instance=1), + ] + second_errors = [ValidationError("7", instance=2)] + stdout, stderr = StringIO(), StringIO() + exit_code = cli.run( + { + "validator": fake_validator(first_errors, second_errors), + "schema": {}, + "instances": [1, 2], + "error_format": "{error.instance} - {error.message}\t", + }, + stdout=stdout, + stderr=stderr, + ) + self.assertFalse(stdout.getvalue()) + self.assertEqual(stderr.getvalue(), "1 - 9\t1 - 8\t2 - 7\t") + self.assertEqual(exit_code, 1) diff --git a/lib/spack/external/jsonschema/tests/test_exceptions.py b/lib/spack/external/jsonschema/tests/test_exceptions.py new file mode 100644 index 00000000000..9e5793c6283 --- /dev/null +++ b/lib/spack/external/jsonschema/tests/test_exceptions.py @@ -0,0 +1,382 @@ +import textwrap + +from jsonschema import Draft4Validator, exceptions +from jsonschema.compat import PY3 +from jsonschema.tests.compat import mock, unittest + + +class TestBestMatch(unittest.TestCase): + def best_match(self, errors): + errors = list(errors) + best = exceptions.best_match(errors) + reversed_best = exceptions.best_match(reversed(errors)) + self.assertEqual( + best, + reversed_best, + msg="Didn't return a consistent best match!\n" + "Got: {0}\n\nThen: {1}".format(best, reversed_best), + ) + return best + + def test_shallower_errors_are_better_matches(self): + validator = Draft4Validator( + { + "properties" : { + "foo" : { + "minProperties" : 2, + "properties" : {"bar" : {"type" : "object"}}, + } + } + } + ) + best = self.best_match(validator.iter_errors({"foo" : {"bar" : []}})) + self.assertEqual(best.validator, "minProperties") + + def test_oneOf_and_anyOf_are_weak_matches(self): + """ + A property you *must* match is probably better than one you have to + match a part of. + + """ + + validator = Draft4Validator( + { + "minProperties" : 2, + "anyOf" : [{"type" : "string"}, {"type" : "number"}], + "oneOf" : [{"type" : "string"}, {"type" : "number"}], + } + ) + best = self.best_match(validator.iter_errors({})) + self.assertEqual(best.validator, "minProperties") + + def test_if_the_most_relevant_error_is_anyOf_it_is_traversed(self): + """ + If the most relevant error is an anyOf, then we traverse its context + and select the otherwise *least* relevant error, since in this case + that means the most specific, deep, error inside the instance. + + I.e. since only one of the schemas must match, we look for the most + relevant one. + + """ + + validator = Draft4Validator( + { + "properties" : { + "foo" : { + "anyOf" : [ + {"type" : "string"}, + {"properties" : {"bar" : {"type" : "array"}}}, + ], + }, + }, + }, + ) + best = self.best_match(validator.iter_errors({"foo" : {"bar" : 12}})) + self.assertEqual(best.validator_value, "array") + + def test_if_the_most_relevant_error_is_oneOf_it_is_traversed(self): + """ + If the most relevant error is an oneOf, then we traverse its context + and select the otherwise *least* relevant error, since in this case + that means the most specific, deep, error inside the instance. + + I.e. since only one of the schemas must match, we look for the most + relevant one. + + """ + + validator = Draft4Validator( + { + "properties" : { + "foo" : { + "oneOf" : [ + {"type" : "string"}, + {"properties" : {"bar" : {"type" : "array"}}}, + ], + }, + }, + }, + ) + best = self.best_match(validator.iter_errors({"foo" : {"bar" : 12}})) + self.assertEqual(best.validator_value, "array") + + def test_if_the_most_relevant_error_is_allOf_it_is_traversed(self): + """ + Now, if the error is allOf, we traverse but select the *most* relevant + error from the context, because all schemas here must match anyways. + + """ + + validator = Draft4Validator( + { + "properties" : { + "foo" : { + "allOf" : [ + {"type" : "string"}, + {"properties" : {"bar" : {"type" : "array"}}}, + ], + }, + }, + }, + ) + best = self.best_match(validator.iter_errors({"foo" : {"bar" : 12}})) + self.assertEqual(best.validator_value, "string") + + def test_nested_context_for_oneOf(self): + validator = Draft4Validator( + { + "properties" : { + "foo" : { + "oneOf" : [ + {"type" : "string"}, + { + "oneOf" : [ + {"type" : "string"}, + { + "properties" : { + "bar" : {"type" : "array"} + }, + }, + ], + }, + ], + }, + }, + }, + ) + best = self.best_match(validator.iter_errors({"foo" : {"bar" : 12}})) + self.assertEqual(best.validator_value, "array") + + def test_one_error(self): + validator = Draft4Validator({"minProperties" : 2}) + error, = validator.iter_errors({}) + self.assertEqual( + exceptions.best_match(validator.iter_errors({})).validator, + "minProperties", + ) + + def test_no_errors(self): + validator = Draft4Validator({}) + self.assertIsNone(exceptions.best_match(validator.iter_errors({}))) + + +class TestByRelevance(unittest.TestCase): + def test_short_paths_are_better_matches(self): + shallow = exceptions.ValidationError("Oh no!", path=["baz"]) + deep = exceptions.ValidationError("Oh yes!", path=["foo", "bar"]) + match = max([shallow, deep], key=exceptions.relevance) + self.assertIs(match, shallow) + + match = max([deep, shallow], key=exceptions.relevance) + self.assertIs(match, shallow) + + def test_global_errors_are_even_better_matches(self): + shallow = exceptions.ValidationError("Oh no!", path=[]) + deep = exceptions.ValidationError("Oh yes!", path=["foo"]) + + errors = sorted([shallow, deep], key=exceptions.relevance) + self.assertEqual( + [list(error.path) for error in errors], + [["foo"], []], + ) + + errors = sorted([deep, shallow], key=exceptions.relevance) + self.assertEqual( + [list(error.path) for error in errors], + [["foo"], []], + ) + + def test_weak_validators_are_lower_priority(self): + weak = exceptions.ValidationError("Oh no!", path=[], validator="a") + normal = exceptions.ValidationError("Oh yes!", path=[], validator="b") + + best_match = exceptions.by_relevance(weak="a") + + match = max([weak, normal], key=best_match) + self.assertIs(match, normal) + + match = max([normal, weak], key=best_match) + self.assertIs(match, normal) + + def test_strong_validators_are_higher_priority(self): + weak = exceptions.ValidationError("Oh no!", path=[], validator="a") + normal = exceptions.ValidationError("Oh yes!", path=[], validator="b") + strong = exceptions.ValidationError("Oh fine!", path=[], validator="c") + + best_match = exceptions.by_relevance(weak="a", strong="c") + + match = max([weak, normal, strong], key=best_match) + self.assertIs(match, strong) + + match = max([strong, normal, weak], key=best_match) + self.assertIs(match, strong) + + +class TestErrorTree(unittest.TestCase): + def test_it_knows_how_many_total_errors_it_contains(self): + errors = [mock.MagicMock() for _ in range(8)] + tree = exceptions.ErrorTree(errors) + self.assertEqual(tree.total_errors, 8) + + def test_it_contains_an_item_if_the_item_had_an_error(self): + errors = [exceptions.ValidationError("a message", path=["bar"])] + tree = exceptions.ErrorTree(errors) + self.assertIn("bar", tree) + + def test_it_does_not_contain_an_item_if_the_item_had_no_error(self): + errors = [exceptions.ValidationError("a message", path=["bar"])] + tree = exceptions.ErrorTree(errors) + self.assertNotIn("foo", tree) + + def test_validators_that_failed_appear_in_errors_dict(self): + error = exceptions.ValidationError("a message", validator="foo") + tree = exceptions.ErrorTree([error]) + self.assertEqual(tree.errors, {"foo" : error}) + + def test_it_creates_a_child_tree_for_each_nested_path(self): + errors = [ + exceptions.ValidationError("a bar message", path=["bar"]), + exceptions.ValidationError("a bar -> 0 message", path=["bar", 0]), + ] + tree = exceptions.ErrorTree(errors) + self.assertIn(0, tree["bar"]) + self.assertNotIn(1, tree["bar"]) + + def test_children_have_their_errors_dicts_built(self): + e1, e2 = ( + exceptions.ValidationError("1", validator="foo", path=["bar", 0]), + exceptions.ValidationError("2", validator="quux", path=["bar", 0]), + ) + tree = exceptions.ErrorTree([e1, e2]) + self.assertEqual(tree["bar"][0].errors, {"foo" : e1, "quux" : e2}) + + def test_it_does_not_contain_subtrees_that_are_not_in_the_instance(self): + error = exceptions.ValidationError("123", validator="foo", instance=[]) + tree = exceptions.ErrorTree([error]) + + with self.assertRaises(IndexError): + tree[0] + + def test_if_its_in_the_tree_anyhow_it_does_not_raise_an_error(self): + """ + If a validator is dumb (like :validator:`required` in draft 3) and + refers to a path that isn't in the instance, the tree still properly + returns a subtree for that path. + + """ + + error = exceptions.ValidationError( + "a message", validator="foo", instance={}, path=["foo"], + ) + tree = exceptions.ErrorTree([error]) + self.assertIsInstance(tree["foo"], exceptions.ErrorTree) + + +class TestErrorReprStr(unittest.TestCase): + def make_error(self, **kwargs): + defaults = dict( + message=u"hello", + validator=u"type", + validator_value=u"string", + instance=5, + schema={u"type": u"string"}, + ) + defaults.update(kwargs) + return exceptions.ValidationError(**defaults) + + def assertShows(self, expected, **kwargs): + if PY3: + expected = expected.replace("u'", "'") + expected = textwrap.dedent(expected).rstrip("\n") + + error = self.make_error(**kwargs) + message_line, _, rest = str(error).partition("\n") + self.assertEqual(message_line, error.message) + self.assertEqual(rest, expected) + + def test_repr(self): + self.assertEqual( + repr(exceptions.ValidationError(message="Hello!")), + "" % "Hello!", + ) + + def test_unset_error(self): + error = exceptions.ValidationError("message") + self.assertEqual(str(error), "message") + + kwargs = { + "validator": "type", + "validator_value": "string", + "instance": 5, + "schema": {"type": "string"} + } + # Just the message should show if any of the attributes are unset + for attr in kwargs: + k = dict(kwargs) + del k[attr] + error = exceptions.ValidationError("message", **k) + self.assertEqual(str(error), "message") + + def test_empty_paths(self): + self.assertShows( + """ + Failed validating u'type' in schema: + {u'type': u'string'} + + On instance: + 5 + """, + path=[], + schema_path=[], + ) + + def test_one_item_paths(self): + self.assertShows( + """ + Failed validating u'type' in schema: + {u'type': u'string'} + + On instance[0]: + 5 + """, + path=[0], + schema_path=["items"], + ) + + def test_multiple_item_paths(self): + self.assertShows( + """ + Failed validating u'type' in schema[u'items'][0]: + {u'type': u'string'} + + On instance[0][u'a']: + 5 + """, + path=[0, u"a"], + schema_path=[u"items", 0, 1], + ) + + def test_uses_pprint(self): + with mock.patch("pprint.pformat") as pformat: + str(self.make_error()) + self.assertEqual(pformat.call_count, 2) # schema + instance + + def test_str_works_with_instances_having_overriden_eq_operator(self): + """ + Check for https://github.com/Julian/jsonschema/issues/164 which + rendered exceptions unusable when a `ValidationError` involved + instances with an `__eq__` method that returned truthy values. + + """ + + instance = mock.MagicMock() + error = exceptions.ValidationError( + "a message", + validator="foo", + instance=instance, + validator_value="some", + schema="schema", + ) + str(error) + self.assertFalse(instance.__eq__.called) diff --git a/lib/spack/external/jsonschema/tests/test_format.py b/lib/spack/external/jsonschema/tests/test_format.py new file mode 100644 index 00000000000..8392ca1de33 --- /dev/null +++ b/lib/spack/external/jsonschema/tests/test_format.py @@ -0,0 +1,63 @@ +""" +Tests for the parts of jsonschema related to the :validator:`format` property. + +""" + +from jsonschema.tests.compat import mock, unittest + +from jsonschema import FormatError, ValidationError, FormatChecker +from jsonschema.validators import Draft4Validator + + +class TestFormatChecker(unittest.TestCase): + def setUp(self): + self.fn = mock.Mock() + + def test_it_can_validate_no_formats(self): + checker = FormatChecker(formats=()) + self.assertFalse(checker.checkers) + + def test_it_raises_a_key_error_for_unknown_formats(self): + with self.assertRaises(KeyError): + FormatChecker(formats=["o noes"]) + + def test_it_can_register_cls_checkers(self): + with mock.patch.dict(FormatChecker.checkers, clear=True): + FormatChecker.cls_checks("new")(self.fn) + self.assertEqual(FormatChecker.checkers, {"new" : (self.fn, ())}) + + def test_it_can_register_checkers(self): + checker = FormatChecker() + checker.checks("new")(self.fn) + self.assertEqual( + checker.checkers, + dict(FormatChecker.checkers, new=(self.fn, ())) + ) + + def test_it_catches_registered_errors(self): + checker = FormatChecker() + cause = self.fn.side_effect = ValueError() + + checker.checks("foo", raises=ValueError)(self.fn) + + with self.assertRaises(FormatError) as cm: + checker.check("bar", "foo") + + self.assertIs(cm.exception.cause, cause) + self.assertIs(cm.exception.__cause__, cause) + + # Unregistered errors should not be caught + self.fn.side_effect = AttributeError + with self.assertRaises(AttributeError): + checker.check("bar", "foo") + + def test_format_error_causes_become_validation_error_causes(self): + checker = FormatChecker() + checker.checks("foo", raises=ValueError)(self.fn) + cause = self.fn.side_effect = ValueError() + validator = Draft4Validator({"format" : "foo"}, format_checker=checker) + + with self.assertRaises(ValidationError) as cm: + validator.validate("bar") + + self.assertIs(cm.exception.__cause__, cause) diff --git a/lib/spack/external/jsonschema/tests/test_jsonschema_test_suite.py b/lib/spack/external/jsonschema/tests/test_jsonschema_test_suite.py new file mode 100644 index 00000000000..75c6857bc03 --- /dev/null +++ b/lib/spack/external/jsonschema/tests/test_jsonschema_test_suite.py @@ -0,0 +1,290 @@ +""" +Test runner for the JSON Schema official test suite + +Tests comprehensive correctness of each draft's validator. + +See https://github.com/json-schema/JSON-Schema-Test-Suite for details. + +""" + +from contextlib import closing +from decimal import Decimal +import glob +import json +import io +import itertools +import os +import re +import subprocess +import sys + +try: + from sys import pypy_version_info +except ImportError: + pypy_version_info = None + +from jsonschema import ( + FormatError, SchemaError, ValidationError, Draft3Validator, + Draft4Validator, FormatChecker, draft3_format_checker, + draft4_format_checker, validate, +) +from jsonschema.compat import PY3 +from jsonschema.tests.compat import mock, unittest +import jsonschema + + +REPO_ROOT = os.path.join(os.path.dirname(jsonschema.__file__), os.path.pardir) +SUITE = os.getenv("JSON_SCHEMA_TEST_SUITE", os.path.join(REPO_ROOT, "json")) + +if not os.path.isdir(SUITE): + raise ValueError( + "Can't find the JSON-Schema-Test-Suite directory. Set the " + "'JSON_SCHEMA_TEST_SUITE' environment variable or run the tests from " + "alongside a checkout of the suite." + ) + +TESTS_DIR = os.path.join(SUITE, "tests") +JSONSCHEMA_SUITE = os.path.join(SUITE, "bin", "jsonschema_suite") + +remotes_stdout = subprocess.Popen( + ["python", JSONSCHEMA_SUITE, "remotes"], stdout=subprocess.PIPE, +).stdout + +with closing(remotes_stdout): + if PY3: + remotes_stdout = io.TextIOWrapper(remotes_stdout) + REMOTES = json.load(remotes_stdout) + + +def make_case(schema, data, valid, name): + if valid: + def test_case(self): + kwargs = getattr(self, "validator_kwargs", {}) + validate(data, schema, cls=self.validator_class, **kwargs) + else: + def test_case(self): + kwargs = getattr(self, "validator_kwargs", {}) + with self.assertRaises(ValidationError): + validate(data, schema, cls=self.validator_class, **kwargs) + + if not PY3: + name = name.encode("utf-8") + test_case.__name__ = name + + return test_case + + +def maybe_skip(skip, test_case, case, test): + if skip is not None: + reason = skip(case, test) + if reason is not None: + test_case = unittest.skip(reason)(test_case) + return test_case + + +def load_json_cases(tests_glob, ignore_glob="", basedir=TESTS_DIR, skip=None): + if ignore_glob: + ignore_glob = os.path.join(basedir, ignore_glob) + + def add_test_methods(test_class): + ignored = set(glob.iglob(ignore_glob)) + + for filename in glob.iglob(os.path.join(basedir, tests_glob)): + if filename in ignored: + continue + + validating, _ = os.path.splitext(os.path.basename(filename)) + id = itertools.count(1) + + with open(filename) as test_file: + for case in json.load(test_file): + for test in case["tests"]: + name = "test_%s_%s_%s" % ( + validating, + next(id), + re.sub(r"[\W ]+", "_", test["description"]), + ) + assert not hasattr(test_class, name), name + + test_case = make_case( + data=test["data"], + schema=case["schema"], + valid=test["valid"], + name=name, + ) + test_case = maybe_skip(skip, test_case, case, test) + setattr(test_class, name, test_case) + + return test_class + return add_test_methods + + +class TypesMixin(object): + @unittest.skipIf(PY3, "In Python 3 json.load always produces unicode") + def test_string_a_bytestring_is_a_string(self): + self.validator_class({"type" : "string"}).validate(b"foo") + + +class DecimalMixin(object): + def test_it_can_validate_with_decimals(self): + schema = {"type" : "number"} + validator = self.validator_class( + schema, types={"number" : (int, float, Decimal)} + ) + + for valid in [1, 1.1, Decimal(1) / Decimal(8)]: + validator.validate(valid) + + for invalid in ["foo", {}, [], True, None]: + with self.assertRaises(ValidationError): + validator.validate(invalid) + + +def missing_format(checker): + def missing_format(case, test): + format = case["schema"].get("format") + if format not in checker.checkers: + return "Format checker {0!r} not found.".format(format) + elif ( + format == "date-time" and + pypy_version_info is not None and + pypy_version_info[:2] <= (1, 9) + ): + # datetime.datetime is overzealous about typechecking in <=1.9 + return "datetime.datetime is broken on this version of PyPy." + return missing_format + + +class FormatMixin(object): + def test_it_returns_true_for_formats_it_does_not_know_about(self): + validator = self.validator_class( + {"format" : "carrot"}, format_checker=FormatChecker(), + ) + validator.validate("bugs") + + def test_it_does_not_validate_formats_by_default(self): + validator = self.validator_class({}) + self.assertIsNone(validator.format_checker) + + def test_it_validates_formats_if_a_checker_is_provided(self): + checker = mock.Mock(spec=FormatChecker) + validator = self.validator_class( + {"format" : "foo"}, format_checker=checker, + ) + + validator.validate("bar") + + checker.check.assert_called_once_with("bar", "foo") + + cause = ValueError() + checker.check.side_effect = FormatError('aoeu', cause=cause) + + with self.assertRaises(ValidationError) as cm: + validator.validate("bar") + # Make sure original cause is attached + self.assertIs(cm.exception.cause, cause) + + def test_it_validates_formats_of_any_type(self): + checker = mock.Mock(spec=FormatChecker) + validator = self.validator_class( + {"format" : "foo"}, format_checker=checker, + ) + + validator.validate([1, 2, 3]) + + checker.check.assert_called_once_with([1, 2, 3], "foo") + + cause = ValueError() + checker.check.side_effect = FormatError('aoeu', cause=cause) + + with self.assertRaises(ValidationError) as cm: + validator.validate([1, 2, 3]) + # Make sure original cause is attached + self.assertIs(cm.exception.cause, cause) + + +if sys.maxunicode == 2 ** 16 - 1: # This is a narrow build. + def narrow_unicode_build(case, test): + if "supplementary Unicode" in test["description"]: + return "Not running surrogate Unicode case, this Python is narrow." +else: + def narrow_unicode_build(case, test): # This isn't, skip nothing. + return + + +@load_json_cases( + "draft3/*.json", + skip=narrow_unicode_build, + ignore_glob="draft3/refRemote.json", +) +@load_json_cases( + "draft3/optional/format.json", skip=missing_format(draft3_format_checker) +) +@load_json_cases("draft3/optional/bignum.json") +@load_json_cases("draft3/optional/zeroTerminatedFloats.json") +class TestDraft3(unittest.TestCase, TypesMixin, DecimalMixin, FormatMixin): + validator_class = Draft3Validator + validator_kwargs = {"format_checker" : draft3_format_checker} + + def test_any_type_is_valid_for_type_any(self): + validator = self.validator_class({"type" : "any"}) + validator.validate(mock.Mock()) + + # TODO: we're in need of more meta schema tests + def test_invalid_properties(self): + with self.assertRaises(SchemaError): + validate({}, {"properties": {"test": True}}, + cls=self.validator_class) + + def test_minItems_invalid_string(self): + with self.assertRaises(SchemaError): + # needs to be an integer + validate([1], {"minItems" : "1"}, cls=self.validator_class) + + +@load_json_cases( + "draft4/*.json", + skip=narrow_unicode_build, + ignore_glob="draft4/refRemote.json", +) +@load_json_cases( + "draft4/optional/format.json", skip=missing_format(draft4_format_checker) +) +@load_json_cases("draft4/optional/bignum.json") +@load_json_cases("draft4/optional/zeroTerminatedFloats.json") +class TestDraft4(unittest.TestCase, TypesMixin, DecimalMixin, FormatMixin): + validator_class = Draft4Validator + validator_kwargs = {"format_checker" : draft4_format_checker} + + # TODO: we're in need of more meta schema tests + def test_invalid_properties(self): + with self.assertRaises(SchemaError): + validate({}, {"properties": {"test": True}}, + cls=self.validator_class) + + def test_minItems_invalid_string(self): + with self.assertRaises(SchemaError): + # needs to be an integer + validate([1], {"minItems" : "1"}, cls=self.validator_class) + + +class RemoteRefResolutionMixin(object): + def setUp(self): + patch = mock.patch("jsonschema.validators.requests") + requests = patch.start() + requests.get.side_effect = self.resolve + self.addCleanup(patch.stop) + + def resolve(self, reference): + _, _, reference = reference.partition("http://localhost:1234/") + return mock.Mock(**{"json.return_value" : REMOTES.get(reference)}) + + +@load_json_cases("draft3/refRemote.json") +class Draft3RemoteResolution(RemoteRefResolutionMixin, unittest.TestCase): + validator_class = Draft3Validator + + +@load_json_cases("draft4/refRemote.json") +class Draft4RemoteResolution(RemoteRefResolutionMixin, unittest.TestCase): + validator_class = Draft4Validator diff --git a/lib/spack/external/jsonschema/tests/test_validators.py b/lib/spack/external/jsonschema/tests/test_validators.py new file mode 100644 index 00000000000..f8692388ea4 --- /dev/null +++ b/lib/spack/external/jsonschema/tests/test_validators.py @@ -0,0 +1,786 @@ +from collections import deque +from contextlib import contextmanager +import json + +from jsonschema import FormatChecker, ValidationError +from jsonschema.tests.compat import mock, unittest +from jsonschema.validators import ( + RefResolutionError, UnknownType, Draft3Validator, + Draft4Validator, RefResolver, create, extend, validator_for, validate, +) + + +class TestCreateAndExtend(unittest.TestCase): + def setUp(self): + self.meta_schema = {u"properties" : {u"smelly" : {}}} + self.smelly = mock.MagicMock() + self.validators = {u"smelly" : self.smelly} + self.types = {u"dict" : dict} + self.Validator = create( + meta_schema=self.meta_schema, + validators=self.validators, + default_types=self.types, + ) + + self.validator_value = 12 + self.schema = {u"smelly" : self.validator_value} + self.validator = self.Validator(self.schema) + + def test_attrs(self): + self.assertEqual(self.Validator.VALIDATORS, self.validators) + self.assertEqual(self.Validator.META_SCHEMA, self.meta_schema) + self.assertEqual(self.Validator.DEFAULT_TYPES, self.types) + + def test_init(self): + self.assertEqual(self.validator.schema, self.schema) + + def test_iter_errors(self): + instance = "hello" + + self.smelly.return_value = [] + self.assertEqual(list(self.validator.iter_errors(instance)), []) + + error = mock.Mock() + self.smelly.return_value = [error] + self.assertEqual(list(self.validator.iter_errors(instance)), [error]) + + self.smelly.assert_called_with( + self.validator, self.validator_value, instance, self.schema, + ) + + def test_if_a_version_is_provided_it_is_registered(self): + with mock.patch("jsonschema.validators.validates") as validates: + validates.side_effect = lambda version : lambda cls : cls + Validator = create(meta_schema={u"id" : ""}, version="my version") + validates.assert_called_once_with("my version") + self.assertEqual(Validator.__name__, "MyVersionValidator") + + def test_if_a_version_is_not_provided_it_is_not_registered(self): + with mock.patch("jsonschema.validators.validates") as validates: + create(meta_schema={u"id" : "id"}) + self.assertFalse(validates.called) + + def test_extend(self): + validators = dict(self.Validator.VALIDATORS) + new = mock.Mock() + + Extended = extend(self.Validator, validators={u"a new one" : new}) + + validators.update([(u"a new one", new)]) + self.assertEqual(Extended.VALIDATORS, validators) + self.assertNotIn(u"a new one", self.Validator.VALIDATORS) + + self.assertEqual(Extended.META_SCHEMA, self.Validator.META_SCHEMA) + self.assertEqual(Extended.DEFAULT_TYPES, self.Validator.DEFAULT_TYPES) + + +class TestIterErrors(unittest.TestCase): + def setUp(self): + self.validator = Draft3Validator({}) + + def test_iter_errors(self): + instance = [1, 2] + schema = { + u"disallow" : u"array", + u"enum" : [["a", "b", "c"], ["d", "e", "f"]], + u"minItems" : 3 + } + + got = (e.message for e in self.validator.iter_errors(instance, schema)) + expected = [ + "%r is disallowed for [1, 2]" % (schema["disallow"],), + "[1, 2] is too short", + "[1, 2] is not one of %r" % (schema["enum"],), + ] + self.assertEqual(sorted(got), sorted(expected)) + + def test_iter_errors_multiple_failures_one_validator(self): + instance = {"foo" : 2, "bar" : [1], "baz" : 15, "quux" : "spam"} + schema = { + u"properties" : { + "foo" : {u"type" : "string"}, + "bar" : {u"minItems" : 2}, + "baz" : {u"maximum" : 10, u"enum" : [2, 4, 6, 8]}, + } + } + + errors = list(self.validator.iter_errors(instance, schema)) + self.assertEqual(len(errors), 4) + + +class TestValidationErrorMessages(unittest.TestCase): + def message_for(self, instance, schema, *args, **kwargs): + kwargs.setdefault("cls", Draft3Validator) + with self.assertRaises(ValidationError) as e: + validate(instance, schema, *args, **kwargs) + return e.exception.message + + def test_single_type_failure(self): + message = self.message_for(instance=1, schema={u"type" : u"string"}) + self.assertEqual(message, "1 is not of type %r" % u"string") + + def test_single_type_list_failure(self): + message = self.message_for(instance=1, schema={u"type" : [u"string"]}) + self.assertEqual(message, "1 is not of type %r" % u"string") + + def test_multiple_type_failure(self): + types = u"string", u"object" + message = self.message_for(instance=1, schema={u"type" : list(types)}) + self.assertEqual(message, "1 is not of type %r, %r" % types) + + def test_object_without_title_type_failure(self): + type = {u"type" : [{u"minimum" : 3}]} + message = self.message_for(instance=1, schema={u"type" : [type]}) + self.assertEqual(message, "1 is not of type %r" % (type,)) + + def test_object_with_name_type_failure(self): + name = "Foo" + schema = {u"type" : [{u"name" : name, u"minimum" : 3}]} + message = self.message_for(instance=1, schema=schema) + self.assertEqual(message, "1 is not of type %r" % (name,)) + + def test_minimum(self): + message = self.message_for(instance=1, schema={"minimum" : 2}) + self.assertEqual(message, "1 is less than the minimum of 2") + + def test_maximum(self): + message = self.message_for(instance=1, schema={"maximum" : 0}) + self.assertEqual(message, "1 is greater than the maximum of 0") + + def test_dependencies_failure_has_single_element_not_list(self): + depend, on = "bar", "foo" + schema = {u"dependencies" : {depend : on}} + message = self.message_for({"bar" : 2}, schema) + self.assertEqual(message, "%r is a dependency of %r" % (on, depend)) + + def test_additionalItems_single_failure(self): + message = self.message_for( + [2], {u"items" : [], u"additionalItems" : False}, + ) + self.assertIn("(2 was unexpected)", message) + + def test_additionalItems_multiple_failures(self): + message = self.message_for( + [1, 2, 3], {u"items" : [], u"additionalItems" : False} + ) + self.assertIn("(1, 2, 3 were unexpected)", message) + + def test_additionalProperties_single_failure(self): + additional = "foo" + schema = {u"additionalProperties" : False} + message = self.message_for({additional : 2}, schema) + self.assertIn("(%r was unexpected)" % (additional,), message) + + def test_additionalProperties_multiple_failures(self): + schema = {u"additionalProperties" : False} + message = self.message_for(dict.fromkeys(["foo", "bar"]), schema) + + self.assertIn(repr("foo"), message) + self.assertIn(repr("bar"), message) + self.assertIn("were unexpected)", message) + + def test_invalid_format_default_message(self): + checker = FormatChecker(formats=()) + check_fn = mock.Mock(return_value=False) + checker.checks(u"thing")(check_fn) + + schema = {u"format" : u"thing"} + message = self.message_for("bla", schema, format_checker=checker) + + self.assertIn(repr("bla"), message) + self.assertIn(repr("thing"), message) + self.assertIn("is not a", message) + + +class TestValidationErrorDetails(unittest.TestCase): + # TODO: These really need unit tests for each individual validator, rather + # than just these higher level tests. + def test_anyOf(self): + instance = 5 + schema = { + "anyOf": [ + {"minimum": 20}, + {"type": "string"} + ] + } + + validator = Draft4Validator(schema) + errors = list(validator.iter_errors(instance)) + self.assertEqual(len(errors), 1) + e = errors[0] + + self.assertEqual(e.validator, "anyOf") + self.assertEqual(e.validator_value, schema["anyOf"]) + self.assertEqual(e.instance, instance) + self.assertEqual(e.schema, schema) + self.assertIsNone(e.parent) + + self.assertEqual(e.path, deque([])) + self.assertEqual(e.relative_path, deque([])) + self.assertEqual(e.absolute_path, deque([])) + + self.assertEqual(e.schema_path, deque(["anyOf"])) + self.assertEqual(e.relative_schema_path, deque(["anyOf"])) + self.assertEqual(e.absolute_schema_path, deque(["anyOf"])) + + self.assertEqual(len(e.context), 2) + + e1, e2 = sorted_errors(e.context) + + self.assertEqual(e1.validator, "minimum") + self.assertEqual(e1.validator_value, schema["anyOf"][0]["minimum"]) + self.assertEqual(e1.instance, instance) + self.assertEqual(e1.schema, schema["anyOf"][0]) + self.assertIs(e1.parent, e) + + self.assertEqual(e1.path, deque([])) + self.assertEqual(e1.absolute_path, deque([])) + self.assertEqual(e1.relative_path, deque([])) + + self.assertEqual(e1.schema_path, deque([0, "minimum"])) + self.assertEqual(e1.relative_schema_path, deque([0, "minimum"])) + self.assertEqual( + e1.absolute_schema_path, deque(["anyOf", 0, "minimum"]), + ) + + self.assertFalse(e1.context) + + self.assertEqual(e2.validator, "type") + self.assertEqual(e2.validator_value, schema["anyOf"][1]["type"]) + self.assertEqual(e2.instance, instance) + self.assertEqual(e2.schema, schema["anyOf"][1]) + self.assertIs(e2.parent, e) + + self.assertEqual(e2.path, deque([])) + self.assertEqual(e2.relative_path, deque([])) + self.assertEqual(e2.absolute_path, deque([])) + + self.assertEqual(e2.schema_path, deque([1, "type"])) + self.assertEqual(e2.relative_schema_path, deque([1, "type"])) + self.assertEqual(e2.absolute_schema_path, deque(["anyOf", 1, "type"])) + + self.assertEqual(len(e2.context), 0) + + def test_type(self): + instance = {"foo": 1} + schema = { + "type": [ + {"type": "integer"}, + { + "type": "object", + "properties": { + "foo": {"enum": [2]} + } + } + ] + } + + validator = Draft3Validator(schema) + errors = list(validator.iter_errors(instance)) + self.assertEqual(len(errors), 1) + e = errors[0] + + self.assertEqual(e.validator, "type") + self.assertEqual(e.validator_value, schema["type"]) + self.assertEqual(e.instance, instance) + self.assertEqual(e.schema, schema) + self.assertIsNone(e.parent) + + self.assertEqual(e.path, deque([])) + self.assertEqual(e.relative_path, deque([])) + self.assertEqual(e.absolute_path, deque([])) + + self.assertEqual(e.schema_path, deque(["type"])) + self.assertEqual(e.relative_schema_path, deque(["type"])) + self.assertEqual(e.absolute_schema_path, deque(["type"])) + + self.assertEqual(len(e.context), 2) + + e1, e2 = sorted_errors(e.context) + + self.assertEqual(e1.validator, "type") + self.assertEqual(e1.validator_value, schema["type"][0]["type"]) + self.assertEqual(e1.instance, instance) + self.assertEqual(e1.schema, schema["type"][0]) + self.assertIs(e1.parent, e) + + self.assertEqual(e1.path, deque([])) + self.assertEqual(e1.relative_path, deque([])) + self.assertEqual(e1.absolute_path, deque([])) + + self.assertEqual(e1.schema_path, deque([0, "type"])) + self.assertEqual(e1.relative_schema_path, deque([0, "type"])) + self.assertEqual(e1.absolute_schema_path, deque(["type", 0, "type"])) + + self.assertFalse(e1.context) + + self.assertEqual(e2.validator, "enum") + self.assertEqual(e2.validator_value, [2]) + self.assertEqual(e2.instance, 1) + self.assertEqual(e2.schema, {u"enum" : [2]}) + self.assertIs(e2.parent, e) + + self.assertEqual(e2.path, deque(["foo"])) + self.assertEqual(e2.relative_path, deque(["foo"])) + self.assertEqual(e2.absolute_path, deque(["foo"])) + + self.assertEqual( + e2.schema_path, deque([1, "properties", "foo", "enum"]), + ) + self.assertEqual( + e2.relative_schema_path, deque([1, "properties", "foo", "enum"]), + ) + self.assertEqual( + e2.absolute_schema_path, + deque(["type", 1, "properties", "foo", "enum"]), + ) + + self.assertFalse(e2.context) + + def test_single_nesting(self): + instance = {"foo" : 2, "bar" : [1], "baz" : 15, "quux" : "spam"} + schema = { + "properties" : { + "foo" : {"type" : "string"}, + "bar" : {"minItems" : 2}, + "baz" : {"maximum" : 10, "enum" : [2, 4, 6, 8]}, + } + } + + validator = Draft3Validator(schema) + errors = validator.iter_errors(instance) + e1, e2, e3, e4 = sorted_errors(errors) + + self.assertEqual(e1.path, deque(["bar"])) + self.assertEqual(e2.path, deque(["baz"])) + self.assertEqual(e3.path, deque(["baz"])) + self.assertEqual(e4.path, deque(["foo"])) + + self.assertEqual(e1.relative_path, deque(["bar"])) + self.assertEqual(e2.relative_path, deque(["baz"])) + self.assertEqual(e3.relative_path, deque(["baz"])) + self.assertEqual(e4.relative_path, deque(["foo"])) + + self.assertEqual(e1.absolute_path, deque(["bar"])) + self.assertEqual(e2.absolute_path, deque(["baz"])) + self.assertEqual(e3.absolute_path, deque(["baz"])) + self.assertEqual(e4.absolute_path, deque(["foo"])) + + self.assertEqual(e1.validator, "minItems") + self.assertEqual(e2.validator, "enum") + self.assertEqual(e3.validator, "maximum") + self.assertEqual(e4.validator, "type") + + def test_multiple_nesting(self): + instance = [1, {"foo" : 2, "bar" : {"baz" : [1]}}, "quux"] + schema = { + "type" : "string", + "items" : { + "type" : ["string", "object"], + "properties" : { + "foo" : {"enum" : [1, 3]}, + "bar" : { + "type" : "array", + "properties" : { + "bar" : {"required" : True}, + "baz" : {"minItems" : 2}, + } + } + } + } + } + + validator = Draft3Validator(schema) + errors = validator.iter_errors(instance) + e1, e2, e3, e4, e5, e6 = sorted_errors(errors) + + self.assertEqual(e1.path, deque([])) + self.assertEqual(e2.path, deque([0])) + self.assertEqual(e3.path, deque([1, "bar"])) + self.assertEqual(e4.path, deque([1, "bar", "bar"])) + self.assertEqual(e5.path, deque([1, "bar", "baz"])) + self.assertEqual(e6.path, deque([1, "foo"])) + + self.assertEqual(e1.schema_path, deque(["type"])) + self.assertEqual(e2.schema_path, deque(["items", "type"])) + self.assertEqual( + list(e3.schema_path), ["items", "properties", "bar", "type"], + ) + self.assertEqual( + list(e4.schema_path), + ["items", "properties", "bar", "properties", "bar", "required"], + ) + self.assertEqual( + list(e5.schema_path), + ["items", "properties", "bar", "properties", "baz", "minItems"] + ) + self.assertEqual( + list(e6.schema_path), ["items", "properties", "foo", "enum"], + ) + + self.assertEqual(e1.validator, "type") + self.assertEqual(e2.validator, "type") + self.assertEqual(e3.validator, "type") + self.assertEqual(e4.validator, "required") + self.assertEqual(e5.validator, "minItems") + self.assertEqual(e6.validator, "enum") + + def test_additionalProperties(self): + instance = {"bar": "bar", "foo": 2} + schema = { + "additionalProperties" : {"type": "integer", "minimum": 5} + } + + validator = Draft3Validator(schema) + errors = validator.iter_errors(instance) + e1, e2 = sorted_errors(errors) + + self.assertEqual(e1.path, deque(["bar"])) + self.assertEqual(e2.path, deque(["foo"])) + + self.assertEqual(e1.validator, "type") + self.assertEqual(e2.validator, "minimum") + + def test_patternProperties(self): + instance = {"bar": 1, "foo": 2} + schema = { + "patternProperties" : { + "bar": {"type": "string"}, + "foo": {"minimum": 5} + } + } + + validator = Draft3Validator(schema) + errors = validator.iter_errors(instance) + e1, e2 = sorted_errors(errors) + + self.assertEqual(e1.path, deque(["bar"])) + self.assertEqual(e2.path, deque(["foo"])) + + self.assertEqual(e1.validator, "type") + self.assertEqual(e2.validator, "minimum") + + def test_additionalItems(self): + instance = ["foo", 1] + schema = { + "items": [], + "additionalItems" : {"type": "integer", "minimum": 5} + } + + validator = Draft3Validator(schema) + errors = validator.iter_errors(instance) + e1, e2 = sorted_errors(errors) + + self.assertEqual(e1.path, deque([0])) + self.assertEqual(e2.path, deque([1])) + + self.assertEqual(e1.validator, "type") + self.assertEqual(e2.validator, "minimum") + + def test_additionalItems_with_items(self): + instance = ["foo", "bar", 1] + schema = { + "items": [{}], + "additionalItems" : {"type": "integer", "minimum": 5} + } + + validator = Draft3Validator(schema) + errors = validator.iter_errors(instance) + e1, e2 = sorted_errors(errors) + + self.assertEqual(e1.path, deque([1])) + self.assertEqual(e2.path, deque([2])) + + self.assertEqual(e1.validator, "type") + self.assertEqual(e2.validator, "minimum") + + +class ValidatorTestMixin(object): + def setUp(self): + self.instance = mock.Mock() + self.schema = {} + self.resolver = mock.Mock() + self.validator = self.validator_class(self.schema) + + def test_valid_instances_are_valid(self): + errors = iter([]) + + with mock.patch.object( + self.validator, "iter_errors", return_value=errors, + ): + self.assertTrue( + self.validator.is_valid(self.instance, self.schema) + ) + + def test_invalid_instances_are_not_valid(self): + errors = iter([mock.Mock()]) + + with mock.patch.object( + self.validator, "iter_errors", return_value=errors, + ): + self.assertFalse( + self.validator.is_valid(self.instance, self.schema) + ) + + def test_non_existent_properties_are_ignored(self): + instance, my_property, my_value = mock.Mock(), mock.Mock(), mock.Mock() + validate(instance=instance, schema={my_property : my_value}) + + def test_it_creates_a_ref_resolver_if_not_provided(self): + self.assertIsInstance(self.validator.resolver, RefResolver) + + def test_it_delegates_to_a_ref_resolver(self): + resolver = RefResolver("", {}) + schema = {"$ref" : mock.Mock()} + + @contextmanager + def resolving(): + yield {"type": "integer"} + + with mock.patch.object(resolver, "resolving") as resolve: + resolve.return_value = resolving() + with self.assertRaises(ValidationError): + self.validator_class(schema, resolver=resolver).validate(None) + + resolve.assert_called_once_with(schema["$ref"]) + + def test_is_type_is_true_for_valid_type(self): + self.assertTrue(self.validator.is_type("foo", "string")) + + def test_is_type_is_false_for_invalid_type(self): + self.assertFalse(self.validator.is_type("foo", "array")) + + def test_is_type_evades_bool_inheriting_from_int(self): + self.assertFalse(self.validator.is_type(True, "integer")) + self.assertFalse(self.validator.is_type(True, "number")) + + def test_is_type_raises_exception_for_unknown_type(self): + with self.assertRaises(UnknownType): + self.validator.is_type("foo", object()) + + +class TestDraft3Validator(ValidatorTestMixin, unittest.TestCase): + validator_class = Draft3Validator + + def test_is_type_is_true_for_any_type(self): + self.assertTrue(self.validator.is_valid(mock.Mock(), {"type": "any"})) + + def test_is_type_does_not_evade_bool_if_it_is_being_tested(self): + self.assertTrue(self.validator.is_type(True, "boolean")) + self.assertTrue(self.validator.is_valid(True, {"type": "any"})) + + def test_non_string_custom_types(self): + schema = {'type': [None]} + cls = self.validator_class(schema, types={None: type(None)}) + cls.validate(None, schema) + + +class TestDraft4Validator(ValidatorTestMixin, unittest.TestCase): + validator_class = Draft4Validator + + +class TestBuiltinFormats(unittest.TestCase): + """ + The built-in (specification-defined) formats do not raise type errors. + + If an instance or value is not a string, it should be ignored. + + """ + + +for format in FormatChecker.checkers: + def test(self, format=format): + v = Draft4Validator({"format": format}, format_checker=FormatChecker()) + v.validate(123) + + name = "test_{0}_ignores_non_strings".format(format) + test.__name__ = name + setattr(TestBuiltinFormats, name, test) + del test # Ugh py.test. Stop discovering top level tests. + + +class TestValidatorFor(unittest.TestCase): + def test_draft_3(self): + schema = {"$schema" : "http://json-schema.org/draft-03/schema"} + self.assertIs(validator_for(schema), Draft3Validator) + + schema = {"$schema" : "http://json-schema.org/draft-03/schema#"} + self.assertIs(validator_for(schema), Draft3Validator) + + def test_draft_4(self): + schema = {"$schema" : "http://json-schema.org/draft-04/schema"} + self.assertIs(validator_for(schema), Draft4Validator) + + schema = {"$schema" : "http://json-schema.org/draft-04/schema#"} + self.assertIs(validator_for(schema), Draft4Validator) + + def test_custom_validator(self): + Validator = create(meta_schema={"id" : "meta schema id"}, version="12") + schema = {"$schema" : "meta schema id"} + self.assertIs(validator_for(schema), Validator) + + def test_validator_for_jsonschema_default(self): + self.assertIs(validator_for({}), Draft4Validator) + + def test_validator_for_custom_default(self): + self.assertIs(validator_for({}, default=None), None) + + +class TestValidate(unittest.TestCase): + def test_draft3_validator_is_chosen(self): + schema = {"$schema" : "http://json-schema.org/draft-03/schema#"} + with mock.patch.object(Draft3Validator, "check_schema") as chk_schema: + validate({}, schema) + chk_schema.assert_called_once_with(schema) + # Make sure it works without the empty fragment + schema = {"$schema" : "http://json-schema.org/draft-03/schema"} + with mock.patch.object(Draft3Validator, "check_schema") as chk_schema: + validate({}, schema) + chk_schema.assert_called_once_with(schema) + + def test_draft4_validator_is_chosen(self): + schema = {"$schema" : "http://json-schema.org/draft-04/schema#"} + with mock.patch.object(Draft4Validator, "check_schema") as chk_schema: + validate({}, schema) + chk_schema.assert_called_once_with(schema) + + def test_draft4_validator_is_the_default(self): + with mock.patch.object(Draft4Validator, "check_schema") as chk_schema: + validate({}, {}) + chk_schema.assert_called_once_with({}) + + +class TestRefResolver(unittest.TestCase): + + base_uri = "" + stored_uri = "foo://stored" + stored_schema = {"stored" : "schema"} + + def setUp(self): + self.referrer = {} + self.store = {self.stored_uri : self.stored_schema} + self.resolver = RefResolver(self.base_uri, self.referrer, self.store) + + def test_it_does_not_retrieve_schema_urls_from_the_network(self): + ref = Draft3Validator.META_SCHEMA["id"] + with mock.patch.object(self.resolver, "resolve_remote") as remote: + with self.resolver.resolving(ref) as resolved: + self.assertEqual(resolved, Draft3Validator.META_SCHEMA) + self.assertFalse(remote.called) + + def test_it_resolves_local_refs(self): + ref = "#/properties/foo" + self.referrer["properties"] = {"foo" : object()} + with self.resolver.resolving(ref) as resolved: + self.assertEqual(resolved, self.referrer["properties"]["foo"]) + + def test_it_resolves_local_refs_with_id(self): + schema = {"id": "foo://bar/schema#", "a": {"foo": "bar"}} + resolver = RefResolver.from_schema(schema) + with resolver.resolving("#/a") as resolved: + self.assertEqual(resolved, schema["a"]) + with resolver.resolving("foo://bar/schema#/a") as resolved: + self.assertEqual(resolved, schema["a"]) + + def test_it_retrieves_stored_refs(self): + with self.resolver.resolving(self.stored_uri) as resolved: + self.assertIs(resolved, self.stored_schema) + + self.resolver.store["cached_ref"] = {"foo" : 12} + with self.resolver.resolving("cached_ref#/foo") as resolved: + self.assertEqual(resolved, 12) + + def test_it_retrieves_unstored_refs_via_requests(self): + ref = "http://bar#baz" + schema = {"baz" : 12} + + with mock.patch("jsonschema.validators.requests") as requests: + requests.get.return_value.json.return_value = schema + with self.resolver.resolving(ref) as resolved: + self.assertEqual(resolved, 12) + requests.get.assert_called_once_with("http://bar") + + def test_it_retrieves_unstored_refs_via_urlopen(self): + ref = "http://bar#baz" + schema = {"baz" : 12} + + with mock.patch("jsonschema.validators.requests", None): + with mock.patch("jsonschema.validators.urlopen") as urlopen: + urlopen.return_value.read.return_value = ( + json.dumps(schema).encode("utf8")) + with self.resolver.resolving(ref) as resolved: + self.assertEqual(resolved, 12) + urlopen.assert_called_once_with("http://bar") + + def test_it_can_construct_a_base_uri_from_a_schema(self): + schema = {"id" : "foo"} + resolver = RefResolver.from_schema(schema) + self.assertEqual(resolver.base_uri, "foo") + with resolver.resolving("") as resolved: + self.assertEqual(resolved, schema) + with resolver.resolving("#") as resolved: + self.assertEqual(resolved, schema) + with resolver.resolving("foo") as resolved: + self.assertEqual(resolved, schema) + with resolver.resolving("foo#") as resolved: + self.assertEqual(resolved, schema) + + def test_it_can_construct_a_base_uri_from_a_schema_without_id(self): + schema = {} + resolver = RefResolver.from_schema(schema) + self.assertEqual(resolver.base_uri, "") + with resolver.resolving("") as resolved: + self.assertEqual(resolved, schema) + with resolver.resolving("#") as resolved: + self.assertEqual(resolved, schema) + + def test_custom_uri_scheme_handlers(self): + schema = {"foo": "bar"} + ref = "foo://bar" + foo_handler = mock.Mock(return_value=schema) + resolver = RefResolver("", {}, handlers={"foo": foo_handler}) + with resolver.resolving(ref) as resolved: + self.assertEqual(resolved, schema) + foo_handler.assert_called_once_with(ref) + + def test_cache_remote_on(self): + ref = "foo://bar" + foo_handler = mock.Mock() + resolver = RefResolver( + "", {}, cache_remote=True, handlers={"foo" : foo_handler}, + ) + with resolver.resolving(ref): + pass + with resolver.resolving(ref): + pass + foo_handler.assert_called_once_with(ref) + + def test_cache_remote_off(self): + ref = "foo://bar" + foo_handler = mock.Mock() + resolver = RefResolver( + "", {}, cache_remote=False, handlers={"foo" : foo_handler}, + ) + with resolver.resolving(ref): + pass + with resolver.resolving(ref): + pass + self.assertEqual(foo_handler.call_count, 2) + + def test_if_you_give_it_junk_you_get_a_resolution_error(self): + ref = "foo://bar" + foo_handler = mock.Mock(side_effect=ValueError("Oh no! What's this?")) + resolver = RefResolver("", {}, handlers={"foo" : foo_handler}) + with self.assertRaises(RefResolutionError) as err: + with resolver.resolving(ref): + pass + self.assertEqual(str(err.exception), "Oh no! What's this?") + + +def sorted_errors(errors): + def key(error): + return ( + [str(e) for e in error.path], + [str(e) for e in error.schema_path] + ) + return sorted(errors, key=key) diff --git a/lib/spack/external/jsonschema/validators.py b/lib/spack/external/jsonschema/validators.py new file mode 100644 index 00000000000..3e326844f4c --- /dev/null +++ b/lib/spack/external/jsonschema/validators.py @@ -0,0 +1,428 @@ +from __future__ import division + +import contextlib +import json +import numbers + +try: + import requests +except ImportError: + requests = None + +from jsonschema import _utils, _validators +from jsonschema.compat import ( + Sequence, urljoin, urlsplit, urldefrag, unquote, urlopen, + str_types, int_types, iteritems, +) +from jsonschema.exceptions import ErrorTree # Backwards compatibility # noqa +from jsonschema.exceptions import RefResolutionError, SchemaError, UnknownType + + +_unset = _utils.Unset() + +validators = {} +meta_schemas = _utils.URIDict() + + +def validates(version): + """ + Register the decorated validator for a ``version`` of the specification. + + Registered validators and their meta schemas will be considered when + parsing ``$schema`` properties' URIs. + + :argument str version: an identifier to use as the version's name + :returns: a class decorator to decorate the validator with the version + + """ + + def _validates(cls): + validators[version] = cls + if u"id" in cls.META_SCHEMA: + meta_schemas[cls.META_SCHEMA[u"id"]] = cls + return cls + return _validates + + +def create(meta_schema, validators=(), version=None, default_types=None): # noqa + if default_types is None: + default_types = { + u"array" : list, u"boolean" : bool, u"integer" : int_types, + u"null" : type(None), u"number" : numbers.Number, u"object" : dict, + u"string" : str_types, + } + + class Validator(object): + VALIDATORS = dict(validators) + META_SCHEMA = dict(meta_schema) + DEFAULT_TYPES = dict(default_types) + + def __init__( + self, schema, types=(), resolver=None, format_checker=None, + ): + self._types = dict(self.DEFAULT_TYPES) + self._types.update(types) + + if resolver is None: + resolver = RefResolver.from_schema(schema) + + self.resolver = resolver + self.format_checker = format_checker + self.schema = schema + + @classmethod + def check_schema(cls, schema): + for error in cls(cls.META_SCHEMA).iter_errors(schema): + raise SchemaError.create_from(error) + + def iter_errors(self, instance, _schema=None): + if _schema is None: + _schema = self.schema + + with self.resolver.in_scope(_schema.get(u"id", u"")): + ref = _schema.get(u"$ref") + if ref is not None: + validators = [(u"$ref", ref)] + else: + validators = iteritems(_schema) + + for k, v in validators: + validator = self.VALIDATORS.get(k) + if validator is None: + continue + + errors = validator(self, v, instance, _schema) or () + for error in errors: + # set details if not already set by the called fn + error._set( + validator=k, + validator_value=v, + instance=instance, + schema=_schema, + ) + if k != u"$ref": + error.schema_path.appendleft(k) + yield error + + def descend(self, instance, schema, path=None, schema_path=None): + for error in self.iter_errors(instance, schema): + if path is not None: + error.path.appendleft(path) + if schema_path is not None: + error.schema_path.appendleft(schema_path) + yield error + + def validate(self, *args, **kwargs): + for error in self.iter_errors(*args, **kwargs): + raise error + + def is_type(self, instance, type): + if type not in self._types: + raise UnknownType(type, instance, self.schema) + pytypes = self._types[type] + + # bool inherits from int, so ensure bools aren't reported as ints + if isinstance(instance, bool): + pytypes = _utils.flatten(pytypes) + is_number = any( + issubclass(pytype, numbers.Number) for pytype in pytypes + ) + if is_number and bool not in pytypes: + return False + return isinstance(instance, pytypes) + + def is_valid(self, instance, _schema=None): + error = next(self.iter_errors(instance, _schema), None) + return error is None + + if version is not None: + Validator = validates(version)(Validator) + Validator.__name__ = version.title().replace(" ", "") + "Validator" + + return Validator + + +def extend(validator, validators, version=None): + all_validators = dict(validator.VALIDATORS) + all_validators.update(validators) + return create( + meta_schema=validator.META_SCHEMA, + validators=all_validators, + version=version, + default_types=validator.DEFAULT_TYPES, + ) + + +Draft3Validator = create( + meta_schema=_utils.load_schema("draft3"), + validators={ + u"$ref" : _validators.ref, + u"additionalItems" : _validators.additionalItems, + u"additionalProperties" : _validators.additionalProperties, + u"dependencies" : _validators.dependencies, + u"disallow" : _validators.disallow_draft3, + u"divisibleBy" : _validators.multipleOf, + u"enum" : _validators.enum, + u"extends" : _validators.extends_draft3, + u"format" : _validators.format, + u"items" : _validators.items, + u"maxItems" : _validators.maxItems, + u"maxLength" : _validators.maxLength, + u"maximum" : _validators.maximum, + u"minItems" : _validators.minItems, + u"minLength" : _validators.minLength, + u"minimum" : _validators.minimum, + u"multipleOf" : _validators.multipleOf, + u"pattern" : _validators.pattern, + u"patternProperties" : _validators.patternProperties, + u"properties" : _validators.properties_draft3, + u"type" : _validators.type_draft3, + u"uniqueItems" : _validators.uniqueItems, + }, + version="draft3", +) + +Draft4Validator = create( + meta_schema=_utils.load_schema("draft4"), + validators={ + u"$ref" : _validators.ref, + u"additionalItems" : _validators.additionalItems, + u"additionalProperties" : _validators.additionalProperties, + u"allOf" : _validators.allOf_draft4, + u"anyOf" : _validators.anyOf_draft4, + u"dependencies" : _validators.dependencies, + u"enum" : _validators.enum, + u"format" : _validators.format, + u"items" : _validators.items, + u"maxItems" : _validators.maxItems, + u"maxLength" : _validators.maxLength, + u"maxProperties" : _validators.maxProperties_draft4, + u"maximum" : _validators.maximum, + u"minItems" : _validators.minItems, + u"minLength" : _validators.minLength, + u"minProperties" : _validators.minProperties_draft4, + u"minimum" : _validators.minimum, + u"multipleOf" : _validators.multipleOf, + u"not" : _validators.not_draft4, + u"oneOf" : _validators.oneOf_draft4, + u"pattern" : _validators.pattern, + u"patternProperties" : _validators.patternProperties, + u"properties" : _validators.properties_draft4, + u"required" : _validators.required_draft4, + u"type" : _validators.type_draft4, + u"uniqueItems" : _validators.uniqueItems, + }, + version="draft4", +) + + +class RefResolver(object): + """ + Resolve JSON References. + + :argument str base_uri: URI of the referring document + :argument referrer: the actual referring document + :argument dict store: a mapping from URIs to documents to cache + :argument bool cache_remote: whether remote refs should be cached after + first resolution + :argument dict handlers: a mapping from URI schemes to functions that + should be used to retrieve them + + """ + + def __init__( + self, base_uri, referrer, store=(), cache_remote=True, handlers=(), + ): + self.base_uri = base_uri + self.resolution_scope = base_uri + # This attribute is not used, it is for backwards compatibility + self.referrer = referrer + self.cache_remote = cache_remote + self.handlers = dict(handlers) + + self.store = _utils.URIDict( + (id, validator.META_SCHEMA) + for id, validator in iteritems(meta_schemas) + ) + self.store.update(store) + self.store[base_uri] = referrer + + @classmethod + def from_schema(cls, schema, *args, **kwargs): + """ + Construct a resolver from a JSON schema object. + + :argument schema schema: the referring schema + :rtype: :class:`RefResolver` + + """ + + return cls(schema.get(u"id", u""), schema, *args, **kwargs) + + @contextlib.contextmanager + def in_scope(self, scope): + old_scope = self.resolution_scope + self.resolution_scope = urljoin(old_scope, scope) + try: + yield + finally: + self.resolution_scope = old_scope + + @contextlib.contextmanager + def resolving(self, ref): + """ + Context manager which resolves a JSON ``ref`` and enters the + resolution scope of this ref. + + :argument str ref: reference to resolve + + """ + + full_uri = urljoin(self.resolution_scope, ref) + uri, fragment = urldefrag(full_uri) + if not uri: + uri = self.base_uri + + if uri in self.store: + document = self.store[uri] + else: + try: + document = self.resolve_remote(uri) + except Exception as exc: + raise RefResolutionError(exc) + + old_base_uri, self.base_uri = self.base_uri, uri + try: + with self.in_scope(uri): + yield self.resolve_fragment(document, fragment) + finally: + self.base_uri = old_base_uri + + def resolve_fragment(self, document, fragment): + """ + Resolve a ``fragment`` within the referenced ``document``. + + :argument document: the referrant document + :argument str fragment: a URI fragment to resolve within it + + """ + + fragment = fragment.lstrip(u"/") + parts = unquote(fragment).split(u"/") if fragment else [] + + for part in parts: + part = part.replace(u"~1", u"/").replace(u"~0", u"~") + + if isinstance(document, Sequence): + # Array indexes should be turned into integers + try: + part = int(part) + except ValueError: + pass + try: + document = document[part] + except (TypeError, LookupError): + raise RefResolutionError( + "Unresolvable JSON pointer: %r" % fragment + ) + + return document + + def resolve_remote(self, uri): + """ + Resolve a remote ``uri``. + + Does not check the store first, but stores the retrieved document in + the store if :attr:`RefResolver.cache_remote` is True. + + .. note:: + + If the requests_ library is present, ``jsonschema`` will use it to + request the remote ``uri``, so that the correct encoding is + detected and used. + + If it isn't, or if the scheme of the ``uri`` is not ``http`` or + ``https``, UTF-8 is assumed. + + :argument str uri: the URI to resolve + :returns: the retrieved document + + .. _requests: http://pypi.python.org/pypi/requests/ + + """ + + scheme = urlsplit(uri).scheme + + if scheme in self.handlers: + result = self.handlers[scheme](uri) + elif ( + scheme in [u"http", u"https"] and + requests and + getattr(requests.Response, "json", None) is not None + ): + # Requests has support for detecting the correct encoding of + # json over http + if callable(requests.Response.json): + result = requests.get(uri).json() + else: + result = requests.get(uri).json + else: + # Otherwise, pass off to urllib and assume utf-8 + result = json.loads(urlopen(uri).read().decode("utf-8")) + + if self.cache_remote: + self.store[uri] = result + return result + + +def validator_for(schema, default=_unset): + if default is _unset: + default = Draft4Validator + return meta_schemas.get(schema.get(u"$schema", u""), default) + + +def validate(instance, schema, cls=None, *args, **kwargs): + """ + Validate an instance under the given schema. + + >>> validate([2, 3, 4], {"maxItems" : 2}) + Traceback (most recent call last): + ... + ValidationError: [2, 3, 4] is too long + + :func:`validate` will first verify that the provided schema is itself + valid, since not doing so can lead to less obvious error messages and fail + in less obvious or consistent ways. If you know you have a valid schema + already or don't care, you might prefer using the + :meth:`~IValidator.validate` method directly on a specific validator + (e.g. :meth:`Draft4Validator.validate`). + + + :argument instance: the instance to validate + :argument schema: the schema to validate with + :argument cls: an :class:`IValidator` class that will be used to validate + the instance. + + If the ``cls`` argument is not provided, two things will happen in + accordance with the specification. First, if the schema has a + :validator:`$schema` property containing a known meta-schema [#]_ then the + proper validator will be used. The specification recommends that all + schemas contain :validator:`$schema` properties for this reason. If no + :validator:`$schema` property is found, the default validator class is + :class:`Draft4Validator`. + + Any other provided positional and keyword arguments will be passed on when + instantiating the ``cls``. + + :raises: + :exc:`ValidationError` if the instance is invalid + + :exc:`SchemaError` if the schema itself is invalid + + .. rubric:: Footnotes + .. [#] known by a validator registered with :func:`validates` + """ + if cls is None: + cls = validator_for(schema) + cls.check_schema(schema) + cls(schema, *args, **kwargs).validate(instance) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6c4a15aaabb..ab78ecef308 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -59,9 +59,8 @@ try: repo = spack.repository.RepoPath() sys.meta_path.append(repo) -except spack.repository.RepoError, e: - tty.error('while initializing Spack RepoPath:') - tty.die(e.message) +except spack.error.SpackError, e: + tty.die('while initializing Spack RepoPath:', e.message) # # Set up the installed packages database diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index ebe42d0138e..a792f04cfde 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -118,9 +118,8 @@ def repo_create(args): shutil.rmtree(root, ignore_errors=True) tty.msg("Created repo with namespace '%s'." % namespace) - tty.msg("To register it with Spack, add a line like this to ~/.spack/repos.yaml:", - 'repos:', - ' - ' + full_path) + tty.msg("To register it with spack, run this command:", + 'spack repo add %s' % full_path) def repo_add(args): diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3ff83ae529b..e57aa75f8e2 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -118,9 +118,11 @@ """ import os +import re import sys import copy - +import jsonschema +from jsonschema import Draft4Validator, validators import yaml from yaml.error import MarkedYAMLError from ordereddict_backport import OrderedDict @@ -137,32 +139,62 @@ """Dict from section names -> schema for that section.""" section_schemas = { - 'compilers' : { + 'compilers': { '$schema': 'http://json-schema.org/schema#', - 'title' : 'Spack compiler configuration file schema', - 'type' : 'object', - 'properties' : { - 'compilers' : { - 'type' : 'map', - }, - }, - }, + 'title': 'Spack compiler configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + 'compilers:?': { # optional colon for overriding site config. + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*': { # architecture + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*@\w[\w-]*': { # compiler spec + 'type': 'object', + 'additionalProperties': False, + 'required': ['cc', 'cxx', 'f77', 'fc'], + 'properties': { + 'cc': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cxx': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'f77': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'fc': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + },},},},},},},}, - 'mirrors' : { + 'mirrors': { '$schema': 'http://json-schema.org/schema#', - 'title' : 'Spack mirror configuration file schema', - 'type' : 'map', - 'properties' : { - 'mirrors' : { + 'title': 'Spack mirror configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + r'mirrors:?': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*': { + 'type': 'string'},},},},}, - } - }, - }, - - 'repos' : { + 'repos': { '$schema': 'http://json-schema.org/schema#', - 'title' : 'Spack repository configuration file schema', - }} + 'title': 'Spack repository configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + r'repos:?': { + 'type': 'array', + 'default': [], + 'items': { + 'type': 'string'},},},}, +} """OrderedDict of config scopes keyed by name. Later scopes will override earlier scopes. @@ -170,13 +202,64 @@ config_scopes = OrderedDict() -def validate_section(section): +def validate_section_name(section): """Raise a ValueError if the section is not a valid section.""" if section not in section_schemas: raise ValueError("Invalid config section: '%s'. Options are %s." % (section, section_schemas)) +def extend_with_default(validator_class): + """Add support for the 'default' attribute for properties and patternProperties. + + jsonschema does not handle this out of the box -- it only + validates. This allows us to set default values for configs + where certain fields are `None` b/c they're deleted or + commented out. + + """ + validate_properties = validator_class.VALIDATORS["properties"] + validate_pattern_properties = validator_class.VALIDATORS["patternProperties"] + + def set_defaults(validator, properties, instance, schema): + for property, subschema in properties.iteritems(): + if "default" in subschema: + instance.setdefault(property, subschema["default"]) + for err in validate_properties(validator, properties, instance, schema): + yield err + + def set_pp_defaults(validator, properties, instance, schema): + for property, subschema in properties.iteritems(): + if "default" in subschema: + if isinstance(instance, dict): + for key, val in instance.iteritems(): + if re.match(property, key) and val is None: + instance[key] = subschema["default"] + + for err in validate_pattern_properties(validator, properties, instance, schema): + yield err + + return validators.extend(validator_class, { + "properties" : set_defaults, + "patternProperties" : set_pp_defaults + }) + + +DefaultSettingValidator = extend_with_default(Draft4Validator) + +def validate_section(data, schema): + """Validate data read in from a Spack YAML file. + + This leverages the line information (start_mark, end_mark) stored + on Spack YAML structures. + + """ + try: + DefaultSettingValidator(schema).validate(data) + except jsonschema.ValidationError as e: + raise ConfigFormatError(e, data) + + class ConfigScope(object): """This class represents a configuration scope. @@ -195,18 +278,16 @@ def __init__(self, name, path): config_scopes[name] = self def get_section_filename(self, section): - validate_section(section) + validate_section_name(section) return os.path.join(self.path, "%s.yaml" % section) def get_section(self, section): if not section in self.sections: - path = self.get_section_filename(section) - data = _read_config_file(path) - if data is None: - self.sections[section] = {} - else: - self.sections[section] = data + path = self.get_section_filename(section) + schema = section_schemas[section] + data = _read_config_file(path, schema) + self.sections[section] = data return self.sections[section] @@ -255,7 +336,7 @@ def validate_scope(scope): % (scope, config_scopes.keys())) -def _read_config_file(filename): +def _read_config_file(filename, schema): """Read a YAML configuration file.""" # Ignore nonexisting files. if not os.path.exists(filename): @@ -269,8 +350,13 @@ def _read_config_file(filename): raise ConfigFileError("Config file is not readable: %s." % filename) try: + tty.debug("Reading config file %s" % filename) with open(filename) as f: - return syaml.load(f) + data = syaml.load(f) + + validate_section(data, schema) + + return data except MarkedYAMLError, e: raise ConfigFileError( @@ -337,7 +423,7 @@ def get_config(section, scope=None): Strips off the top-level section name from the YAML dict. """ - validate_section(section) + validate_section_name(section) merged_section = syaml.syaml_dict() if scope is None: @@ -387,7 +473,7 @@ def update_config(section, update_data, scope=None): # read in the config to ensure we've got current data get_config(section) - validate_section(section) # validate section name + validate_section_name(section) # validate section name scope = validate_scope(scope) # get ConfigScope object from string. # read only the requested section's data. @@ -407,4 +493,43 @@ def print_section(section): class ConfigError(SpackError): pass class ConfigFileError(ConfigError): pass -class ConfigFormatError(ConfigError): pass + +def get_path(path, data): + if path: + return get_path(path[1:], data[path[0]]) + else: + return data + +class ConfigFormatError(ConfigError): + """Raised when a configuration format does not match its schema.""" + def __init__(self, validation_error, data): + # Try to get line number from erroneous instance and its parent + instance_mark = getattr(validation_error.instance, '_start_mark', None) + parent_mark = getattr(validation_error.parent, '_start_mark', None) + path = getattr(validation_error, 'path', None) + + # Try really hard to get the parent (which sometimes is not + # set) This digs it out of the validated structure if it's not + # on the validation_error. + if not parent_mark: + parent_path = list(path)[:-1] + parent = get_path(parent_path, data) + if path[-1] in parent: + if isinstance(parent, dict): + keylist = parent.keys() + elif isinstance(parent, list): + keylist = parent + idx = keylist.index(path[-1]) + parent_mark = getattr(keylist[idx], '_start_mark', None) + + if instance_mark: + location = '%s:%d' % (instance_mark.name, instance_mark.line + 1) + elif parent_mark: + location = '%s:%d' % (parent_mark.name, parent_mark.line + 1) + elif path: + location = 'At ' + ':'.join(path) + else: + location = '' + + message = '%s: %s' % (location, validation_error.message) + super(ConfigError, self).__init__(message) From d869d8e86fcabfd04121806c8c1554a00cc5339c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 14 Jan 2016 12:00:33 -0800 Subject: [PATCH 305/910] add sanity check on config write as well as validation on read. --- lib/spack/spack/config.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index e57aa75f8e2..193b3114349 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -297,7 +297,10 @@ def write_section(self, section): try: mkdirp(self.path) with open(filename, 'w') as f: + validate_section(data, section_schemas[section]) syaml.dump(data, stream=f, default_flow_style=False) + except jsonschema.ValidationError as e: + raise ConfigSanityError(e, data) except (yaml.YAMLError, IOError) as e: raise ConfigFileError("Error writing to config file: '%s'" % str(e)) @@ -533,3 +536,6 @@ def __init__(self, validation_error, data): message = '%s: %s' % (location, validation_error.message) super(ConfigError, self).__init__(message) + +class ConfigSanityError(ConfigFormatError): + """Same as ConfigFormatError, raised when config is written by Spack.""" From aa00999fcacf6ba19987388c3998cdd4d1c6e65c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 14 Jan 2016 12:00:56 -0800 Subject: [PATCH 306/910] update compiler config test. --- lib/spack/spack/test/config.py | 70 ++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 5f99dcb9032..d8be5a855b8 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -33,39 +33,43 @@ # Some sample compiler config data a_comps = { - "gcc@4.7.3" : { - "cc" : "/gcc473", - "cxx" : "/g++473", - "f77" : None, - "f90" : None }, - "gcc@4.5.0" : { - "cc" : "/gcc450", - "cxx" : "/g++450", - "f77" : "/gfortran", - "f90" : "/gfortran" }, - "clang@3.3" : { - "cc" : "", - "cxx" : "", - "f77" : "", - "f90" : "" } + "all": { + "gcc@4.7.3" : { + "cc" : "/gcc473", + "cxx": "/g++473", + "f77": None, + "fc" : None }, + "gcc@4.5.0" : { + "cc" : "/gcc450", + "cxx": "/g++450", + "f77": "/gfortran", + "fc" : "/gfortran" }, + "clang@3.3" : { + "cc" : "", + "cxx": "", + "f77": "", + "fc" : "" } + } } b_comps = { - "icc@10.0" : { - "cc" : "/icc100", - "cxx" : "/icc100", - "f77" : None, - "f90" : None }, - "icc@11.1" : { - "cc" : "/icc111", - "cxx" : "/icp111", - "f77" : "/ifort", - "f90" : "/ifort" }, - "clang@3.3" : { - "cc" : "/clang", - "cxx" : "/clang++", - "f77" : None, - "f90" : None} + "all": { + "icc@10.0" : { + "cc" : "/icc100", + "cxx": "/icc100", + "f77": None, + "fc" : None }, + "icc@11.1" : { + "cc" : "/icc111", + "cxx": "/icp111", + "f77": "/ifort", + "fc" : "/ifort" }, + "clang@3.3" : { + "cc" : "/clang", + "cxx": "/clang++", + "f77": None, + "fc" : None} + } } class ConfigTest(MockPackagesTest): @@ -85,11 +89,11 @@ def tearDown(self): def check_config(self, comps, *compiler_names): """Check that named compilers in comps match Spack's config.""" config = spack.config.get_config('compilers') - compiler_list = ['cc', 'cxx', 'f77', 'f90'] + compiler_list = ['cc', 'cxx', 'f77', 'fc'] for key in compiler_names: for c in compiler_list: - expected = comps[key][c] - actual = config[key][c] + expected = comps['all'][key][c] + actual = config['all'][key][c] self.assertEqual(expected, actual) From aa3897a8809be7fd729a59fcc3308ecf480a4226 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 14 Jan 2016 17:40:11 -0500 Subject: [PATCH 307/910] Install ncurses properly Currently, ncurses's include files are installed into two separate subdirectories, "install/ncurses" and "install/ncursesw". The second level of subdirectories ("ncurses" and "ncursesw") are non-standard. I checked several systems to confirm this, and ncurses examples on the web also simply contain "#include " instead of "#include ", which would be necessary to use the currently installed ncurses packages. For example, this also breaks llvm, which uses ncurses, but does not expect the second level of subdirectories. I am now using the option "--enable-overwrite", which installs the header files directly in to ".../include". I also enable "widec" support all the time. These options are e.g. similar to the ones used by MacPorts, and I confirm that they make the llvm package build (which didn't build before). --- var/spack/packages/ncurses/package.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/var/spack/packages/ncurses/package.py b/var/spack/packages/ncurses/package.py index 31f53b6c43e..8dc808caacc 100644 --- a/var/spack/packages/ncurses/package.py +++ b/var/spack/packages/ncurses/package.py @@ -17,19 +17,14 @@ class Ncurses(Package): patch('patch_gcc_5.txt', when='%gcc@5.0:') def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-shared", - "--enable-widec", - "--disable-pc-files", - "--without-ada") + opts = [ + "--prefix=%s" % prefix, + "--with-shared", + "--with-cxx-shared", + "--enable-widec", + "--enable-overwrite", + "--disable-lib-suffixes", + "--without-ada"] + configure(*opts) make() make("install") - - configure("--prefix=%s" % prefix, - "--with-shared", - "--disable-widec", - "--disable-pc-files", - "--without-ada") - make() - make("install") - From 3e703cc2812ab9120baabeab3914ee9fd9aefc8c Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 14 Jan 2016 17:52:28 -0500 Subject: [PATCH 308/910] LLVM depends on cmake, and does not depend on isl Add cmake requirement. Remove gmp and isl requirements. Using an external isl leads to a build failure for me on a fairly standard Fedora Linux workstation. The Spack package file says that isl is required for polly, however, the polly documentation states that as of LLVM 3.7, polly includes isl, and has no external dependencies any more. --- var/spack/packages/llvm/package.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index d4be2b95414..a2b2c6ecccb 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -48,6 +48,9 @@ class Llvm(Package): variant('gold', default=True, description="Add support for LTO with the gold linker plugin") + # Build dependency + depends_on('cmake @2.8.12.2:') + # Universal dependency depends_on('python@2.7:') @@ -60,8 +63,8 @@ class Llvm(Package): depends_on('binutils+gold', when='+gold') # polly plugin - depends_on('gmp', when='+polly') - depends_on('isl', when='+polly') + depends_on('gmp', when='@:3.6.999 +polly') + depends_on('isl', when='@:3.6.999 +polly') base_url = 'http://llvm.org/releases/%%(version)s/%(pkg)s-%%(version)s.src.tar.xz' llvm_url = base_url % { 'pkg' : 'llvm'} @@ -172,7 +175,7 @@ def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag cmake_args = [ arg for arg in std_cmake_args if 'BUILD_TYPE' not in arg ] - build_type = 'RelWithDebInfo' if '+debug' in spec else 'Release' + build_type = 'RelWithDebInfo' if '+debug' in spec else 'Release' cmake_args.extend([ '..', '-DCMAKE_BUILD_TYPE=' + build_type, From 0a993cd9d7791c057fca91710434001857cf2818 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 14 Jan 2016 20:44:51 -0500 Subject: [PATCH 309/910] New package GNU tar --- var/spack/packages/tar/package.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 var/spack/packages/tar/package.py diff --git a/var/spack/packages/tar/package.py b/var/spack/packages/tar/package.py new file mode 100644 index 00000000000..539174017c9 --- /dev/null +++ b/var/spack/packages/tar/package.py @@ -0,0 +1,13 @@ +from spack import * + +class Tar(Package): + """GNU Tar provides the ability to create tar archives, as well as various other kinds of manipulation.""" + homepage = "https://www.gnu.org/software/tar/" + url = "http://ftp.gnu.org/gnu/tar/tar-1.28.tar.gz" + + version('1.28', '6ea3dbea1f2b0409b234048e021a9fd7') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make('install') From 7d0256c9fb5c3b8824cdeb0455d2a3e3b3f1eeb4 Mon Sep 17 00:00:00 2001 From: Nicola Varini Date: Fri, 15 Jan 2016 10:27:38 +0100 Subject: [PATCH 310/910] Quantum-ESPRESSO package --- var/spack/packages/espresso/package.py | 61 ++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 var/spack/packages/espresso/package.py diff --git a/var/spack/packages/espresso/package.py b/var/spack/packages/espresso/package.py new file mode 100644 index 00000000000..f9df74c0c08 --- /dev/null +++ b/var/spack/packages/espresso/package.py @@ -0,0 +1,61 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install espresso +# +# You can always get back here to change things with: +# +# spack edit espresso +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class Espresso(Package): + """FIXME: put a proper description of your package here.""" + # FIXME: add a proper url for your package's homepage here. + homepage = "http://quantum-espresso.org" + url = "http://www.qe-forge.org/gf/download/frsrelease/199/855/espresso-5.2.1.tar.gz" + + version('5.2.1', 'da3ec5302e4343804e65de60f6004c2d') + variant('mpi', default=True, description='Build Quantum-ESPRESSO with mpi support') + variant('openmp', default=False, description='Build Quantum-ESPRESSO with mpi openmp') + variant('scalapack', default=False, description='Build Quantum-ESPRESSO with mpi openmp') + + + # FIXME: Add dependencies if this package requires them. + # depends_on("foo") + depends_on('mpi', when='+mpi') + + +# def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. +# configure('--prefix=%s' % prefix) + + # FIXME: Add logic to build and install here +# make() +# make("install") + + def install(self, spec, prefix): + # TAU isn't happy with directories that have '@' in the path. Sigh. + + # TAU configure, despite the name , seems to be a manually written script (nothing related to autotools). + # As such it has a few #peculiarities# that make this build quite hackish. + options = ["-prefix=%s" % prefix, + "--enable-parallel"] + + if '+openmp' in spec: + options.append('--enable-openmp') + + if '+scalapack' in spec: + options.append('--with-scalapack=yes') + + configure(*options) + make("all") + make("install") + From 7fc308ed26d7abfcb71af8165523b421a1f315c0 Mon Sep 17 00:00:00 2001 From: Nicola Varini Date: Fri, 15 Jan 2016 17:04:25 +0100 Subject: [PATCH 311/910] Update to version 5.3 --- var/spack/packages/espresso/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/espresso/package.py b/var/spack/packages/espresso/package.py index f9df74c0c08..ce5dcc2accc 100644 --- a/var/spack/packages/espresso/package.py +++ b/var/spack/packages/espresso/package.py @@ -20,9 +20,9 @@ class Espresso(Package): """FIXME: put a proper description of your package here.""" # FIXME: add a proper url for your package's homepage here. homepage = "http://quantum-espresso.org" - url = "http://www.qe-forge.org/gf/download/frsrelease/199/855/espresso-5.2.1.tar.gz" + url = "http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz" - version('5.2.1', 'da3ec5302e4343804e65de60f6004c2d') + version('5.3.0', '6848fcfaeb118587d6be36bd10b7f2c3') variant('mpi', default=True, description='Build Quantum-ESPRESSO with mpi support') variant('openmp', default=False, description='Build Quantum-ESPRESSO with mpi openmp') variant('scalapack', default=False, description='Build Quantum-ESPRESSO with mpi openmp') From f6680a0051a586844386f0df57db8d6b9b11c3d3 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 15 Jan 2016 16:02:35 -0500 Subject: [PATCH 312/910] qhull: apply patch to work with libc++'s standard library Only seen on OS X with clang right now. --- var/spack/packages/qhull/package.py | 3 ++ var/spack/packages/qhull/qhull-iterator.patch | 45 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 var/spack/packages/qhull/qhull-iterator.patch diff --git a/var/spack/packages/qhull/package.py b/var/spack/packages/qhull/package.py index 9da4078a702..f6712ced38c 100644 --- a/var/spack/packages/qhull/package.py +++ b/var/spack/packages/qhull/package.py @@ -20,6 +20,9 @@ class Qhull(Package): version('1.0', 'd0f978c0d8dfb2e919caefa56ea2953c', url="http://www.qhull.org/download/qhull-2012.1-src.tgz") + # https://github.com/qhull/qhull/pull/5 + patch('qhull-iterator.patch') + def install(self, spec, prefix): with working_dir('spack-build', create=True): cmake('..', *std_cmake_args) diff --git a/var/spack/packages/qhull/qhull-iterator.patch b/var/spack/packages/qhull/qhull-iterator.patch new file mode 100644 index 00000000000..88e931d84f2 --- /dev/null +++ b/var/spack/packages/qhull/qhull-iterator.patch @@ -0,0 +1,45 @@ +From 93f4b306c54bb5be7724dcc19c6e747b62ac76dd Mon Sep 17 00:00:00 2001 +From: Ben Boeckel +Date: Thu, 28 May 2015 11:12:25 -0400 +Subject: [PATCH] iterator: use the header + +Standard libraries are doing funky things with inline namespaces which +make these declarations impossible to get right. Just include the +header. +--- + src/libqhullcpp/QhullIterator.h | 3 +-- + src/libqhullcpp/QhullLinkedList.h | 5 +---- + 2 files changed, 2 insertions(+), 6 deletions(-) + +diff --git a/src/libqhullcpp/QhullIterator.h b/src/libqhullcpp/QhullIterator.h +index 9dde894..49f3a3b 100644 +--- a/src/libqhullcpp/QhullIterator.h ++++ b/src/libqhullcpp/QhullIterator.h +@@ -14,10 +14,9 @@ extern "C" { + } + + #include ++#include + #include + #include +-//! Avoid dependence on +-namespace std { struct bidirectional_iterator_tag; struct random_access_iterator_tag; } + + namespace orgQhull { + +diff --git a/src/libqhullcpp/QhullLinkedList.h b/src/libqhullcpp/QhullLinkedList.h +index d828ac6..00b9008 100644 +--- a/src/libqhullcpp/QhullLinkedList.h ++++ b/src/libqhullcpp/QhullLinkedList.h +@@ -9,10 +9,7 @@ + #ifndef QHULLLINKEDLIST_H + #define QHULLLINKEDLIST_H + +-namespace std { +- struct bidirectional_iterator_tag; +- struct random_access_iterator_tag; +-}//std ++#include + + #include "QhullError.h" + extern "C" { From c3bf1a77415e4e71c726637299fe840cb1156d35 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 15 Jan 2016 16:03:51 -0500 Subject: [PATCH 313/910] py-numpy: make blas/lapack optional dependencies OS X is having trouble building these libraries, so make them optional dependencies. --- var/spack/packages/py-numpy/package.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/var/spack/packages/py-numpy/package.py b/var/spack/packages/py-numpy/package.py index 4c085fba6e5..03548111863 100644 --- a/var/spack/packages/py-numpy/package.py +++ b/var/spack/packages/py-numpy/package.py @@ -7,15 +7,18 @@ class PyNumpy(Package): version('1.9.1', '78842b73560ec378142665e712ae4ad9') version('1.9.2', 'a1ed53432dbcd256398898d35bc8e645') - + + variant('blas', default=True) + extends('python') depends_on('py-nose') - depends_on('netlib-blas+fpic') - depends_on('netlib-lapack+shared') + depends_on('netlib-blas+fpic', when='+blas') + depends_on('netlib-lapack+shared', when='+blas') def install(self, spec, prefix): - with open('site.cfg', 'w') as f: - f.write('[DEFAULT]\n') - f.write('libraries=lapack,blas\n') - f.write('library_dirs=%s/lib:%s/lib\n' % (spec['blas'].prefix, spec['lapack'].prefix)) + if '+blas' in spec: + with open('site.cfg', 'w') as f: + f.write('[DEFAULT]\n') + f.write('libraries=lapack,blas\n') + f.write('library_dirs=%s/lib:%s/lib\n' % (spec['blas'].prefix, spec['lapack'].prefix)) python('setup.py', 'install', '--prefix=%s' % prefix) From 4f60948bffd33deda0f5009e655fac1308729522 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 15 Jan 2016 16:04:49 -0500 Subject: [PATCH 314/910] py-matplotlib: depend on freetype Probably a missed dependency, but appeared on OS X. --- var/spack/packages/py-matplotlib/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py index 4776c581ee6..2167735fb89 100644 --- a/var/spack/packages/py-matplotlib/package.py +++ b/var/spack/packages/py-matplotlib/package.py @@ -26,6 +26,7 @@ class PyMatplotlib(Package): depends_on('py-pbr') depends_on('py-funcsigs') + depends_on('freetype') depends_on('qt', when='+gui') depends_on('bzip2') depends_on('tcl', when='+gui') From eac795abc3637f79363a4a2648642a4a50055d19 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 15 Jan 2016 15:09:36 -0600 Subject: [PATCH 315/910] Add verbs and psm variants --- var/spack/packages/openmpi/package.py | 46 +++++++++++++++++++-------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/var/spack/packages/openmpi/package.py b/var/spack/packages/openmpi/package.py index be2202fbbd7..463719f9dbe 100644 --- a/var/spack/packages/openmpi/package.py +++ b/var/spack/packages/openmpi/package.py @@ -13,29 +13,33 @@ class Openmpi(Package): """ homepage = "http://www.open-mpi.org" + url = "http://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-1.10.1.tar.bz2" + list_url = "http://www.open-mpi.org/software/ompi/" + list_depth = 3 - version('1.10.1', 'f0fcd77ed345b7eafb431968124ba16e', - url = "http://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-1.10.1.tar.bz2") - version('1.10.0', '280cf952de68369cebaca886c5ce0304', - url = "http://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-1.10.0.tar.bz2") - version('1.8.8', '0dab8e602372da1425e9242ae37faf8c', - url = 'http://www.open-mpi.org/software/ompi/v1.8/downloads/openmpi-1.8.8.tar.bz2') - version('1.6.5', '03aed2a4aa4d0b27196962a2a65fc475', - url = "http://www.open-mpi.org/software/ompi/v1.6/downloads/openmpi-1.6.5.tar.bz2") + version('1.10.1', 'f0fcd77ed345b7eafb431968124ba16e') + version('1.10.0', '280cf952de68369cebaca886c5ce0304') + version('1.8.8', '0dab8e602372da1425e9242ae37faf8c') + version('1.6.5', '03aed2a4aa4d0b27196962a2a65fc475') patch('ad_lustre_rwcontig_open_source.patch', when="@1.6.5") patch('llnl-platforms.patch', when="@1.6.5") patch('configure.patch', when="@1.10.0:") - provides('mpi@:2.2', when='@1.6.5') # Open MPI 1.6.5 supports MPI-2.2 - provides('mpi@:3.0', when='@1.8.8') # Open MPI 1.8.8 supports MPI-3.0 - provides('mpi@:3.0', when='@1.10.0') # Open MPI 1.10.0 supports MPI-3.0 - provides('mpi@:3.0', when='@1.10.1') # Open MPI 1.10.1 supports MPI-3.0 + variant('psm', default=False, description='Build support for the PSM library.') + variant('verbs', default=False, description='Build support for OpenFabrics verbs.') + + provides('mpi@:2.2', when='@1.6.5') + provides('mpi@:3.0', when='@1.7.5:') depends_on('hwloc') + def url_for_version(self, version): + return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) + + def setup_dependent_environment(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" os.environ['OMPI_CC'] = 'cc' @@ -45,9 +49,23 @@ def setup_dependent_environment(self, module, spec, dep_spec): def install(self, spec, prefix): - config_args = ["--prefix=%s" % prefix] + config_args = ["--prefix=%s" % prefix, + "--with-hwloc=%s" % spec['hwloc'].prefix, + "--with-tm", # necessary for Torque support + "--enable-shared", + "--enable-static"] - config_args.append("--with-hwloc=%s" % spec['hwloc'].prefix) + # Variants + if '+psm' in spec: + config_args.append("--with-psm") + + if '+verbs' in spec: + # Up through version 1.6, this option was previously named --with-openib + if spec.satisfies('@:1.6'): + config_args.append("--with-openib") + # In version 1.7, it was renamed to be --with-verbs + elif spec.satisfies('@1.7:'): + config_args.append("--with-verbs") # TODO: use variants for this, e.g. +lanl, +llnl, etc. # use this for LANL builds, but for LLNL builds, we need: From d22cf1aed1d033610481cb451909c93916848ccc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 15 Jan 2016 18:07:41 -0800 Subject: [PATCH 316/910] 1. raise an exception if the multithreaded and singlethreaded options are both disabled 2. invoke the b2 installation once for each enabled threading option (apparently the install fails if a single call has both options enabled for mpi) --- var/spack/packages/boost/package.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index 5fb6c9dc049..1992d4d39a5 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -138,18 +138,20 @@ def determine_b2_options(self, spec, options): if '+shared' in spec: linkTypes.append('shared') - #TODO: at least one of these two options must be active threadingOpts = [] if '+multithreaded' in spec: threadingOpts.append('multi') if '+singlethreaded' in spec: threadingOpts.append('single') + if not threadingOpts: + raise RuntimeError("At least one of {singlethreaded, multithreaded} must be enabled") options.extend([ 'toolset=%s' % self.determine_toolset(spec), 'link=%s' % ','.join(linkTypes), - 'threading=%s' % ','.join(threadingOpts), '--layout=tagged']) + + return threadingOpts def install(self, spec, prefix): withLibs = list() @@ -181,6 +183,10 @@ def install(self, spec, prefix): b2 = Executable(b2name) b2_options = ['-j', '%s' % make_jobs] - self.determine_b2_options(spec, b2_options) + threadingOpts = self.determine_b2_options(spec, b2_options) - b2('install', *b2_options) + # In theory it could be done on one call but it fails on + # Boost.MPI if the threading options are not separated. + for threadingOpt in threadingOpts: + b2('install', 'threading=%s' % threadingOpt, *b2_options) + From 36a428b8fc2cfaaab7fb52034a3272c67eb47aea Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 17 Jan 2016 10:49:13 -0500 Subject: [PATCH 317/910] Make binutils build on OS X Binutils defines several global variables multiple times. Apparently this works fine under Linux, but it leads to a linker error on Darwin. Rename these global variables. Note that binutils on OS X is still not really useful, as important tools (e.g. ld) are not supported. --- var/spack/packages/binutils/cr16.patch | 26 ++++++++++++++++++++++++++ var/spack/packages/binutils/package.py | 2 ++ 2 files changed, 28 insertions(+) create mode 100644 var/spack/packages/binutils/cr16.patch diff --git a/var/spack/packages/binutils/cr16.patch b/var/spack/packages/binutils/cr16.patch new file mode 100644 index 00000000000..2727c70b238 --- /dev/null +++ b/var/spack/packages/binutils/cr16.patch @@ -0,0 +1,26 @@ +--- old/opcodes/cr16-dis.c 2014-10-14 03:32:04.000000000 -0400 ++++ new/opcodes/cr16-dis.c 2016-01-14 21:54:26.000000000 -0500 +@@ -78,7 +78,7 @@ + REG_ARG_TYPE; + + /* Current opcode table entry we're disassembling. */ +-const inst *instruction; ++extern const inst *instruction; + /* Current instruction we're disassembling. */ + ins cr16_currInsn; + /* The current instruction is read into 3 consecutive words. */ +@@ -86,12 +86,12 @@ + /* Contains all words in appropriate order. */ + ULONGLONG cr16_allWords; + /* Holds the current processed argument number. */ +-int processing_argument_number; ++extern int processing_argument_number; + /* Nonzero means a IMM4 instruction. */ + int imm4flag; + /* Nonzero means the instruction's original size is + incremented (escape sequence is used). */ +-int size_changed; ++extern int size_changed; + + + /* Print the constant expression length. */ diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py index 1de796bee0b..de04221e33a 100644 --- a/var/spack/packages/binutils/package.py +++ b/var/spack/packages/binutils/package.py @@ -14,6 +14,8 @@ class Binutils(Package): variant('gold', default=True, description="build the gold linker") patch('binutilskrell-2.24.patch', when='@2.24+krellpatch') + patch('cr16.patch') + variant('libiberty', default=False, description='Also install libiberty.') def install(self, spec, prefix): From 981cefe8d3f4f73461789713726a9658498a5f14 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 17 Jan 2016 10:54:12 -0500 Subject: [PATCH 318/910] Make Boost build with MPI on OS X Boost does not build on OS X with either gold or binutils. The gold linker does not exist on Darwin, and binutils on Darwin provides an assembler that doesn't work for Boost. - Introduce a variant that specifies whether to build with binutils, defaulting to true for backward compatibility - Auto-detect whether we build on Darwin; in this case, set the gold and binutils variant defaults to false - Clean up configure flags for as and ld --- var/spack/packages/boost/package.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index 3427b74ad68..894af9edd1b 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -1,5 +1,8 @@ from spack import * +import os +import sys + class Boost(Package): """Boost provides free peer-reviewed portable C++ source libraries, emphasizing libraries that work well with the C++ @@ -127,6 +130,16 @@ def determine_b2_options(self, spec, options): '--layout=tagged']) def install(self, spec, prefix): + # On Darwin, Boost expects the Darwin libtool. However, one of the + # dependencies may have pulled in Spack's GNU libtool, and these two are + # not compatible. We thus create a symlink to Darwin's libtool and add + # it at the beginning of PATH. + if sys.platform == 'darwin': + newdir = os.path.abspath('darwin-libtool') + mkdirp(newdir) + force_symlink('/usr/bin/libtool', join_path(newdir, 'libtool')) + env['PATH'] = newdir + ':' + env['PATH'] + # to make Boost find the user-config.jam env['BOOST_BUILD_PATH'] = './' From e585a5d8a3222a1203c4b4135746b8b5f9ab5e54 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 17 Jan 2016 11:02:30 -0500 Subject: [PATCH 319/910] Update Julia to 0.4.3 --- var/spack/packages/julia/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/julia/package.py b/var/spack/packages/julia/package.py index d978842d9c9..6900af38e4f 100644 --- a/var/spack/packages/julia/package.py +++ b/var/spack/packages/julia/package.py @@ -6,6 +6,7 @@ class Julia(Package): homepage = "http://julialang.org" url = "http://github.com/JuliaLang/julia/releases/download/v0.4.2/julia-0.4.2.tar.gz" + version('0.4.3', '7b9f096798fca4bef262a64674bc2b52') version('0.4.2', 'ccfeb4f4090c8b31083f5e1ccb03eb06') patch('gc.patch') From 60656b6255886c492ee6c2ee8665b9f5b4b86e97 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 17 Jan 2016 11:03:40 -0500 Subject: [PATCH 320/910] Update libtool to 2.4.6 --- var/spack/packages/libtool/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/libtool/package.py b/var/spack/packages/libtool/package.py index a07daf97811..82a54953b23 100644 --- a/var/spack/packages/libtool/package.py +++ b/var/spack/packages/libtool/package.py @@ -5,6 +5,7 @@ class Libtool(Package): homepage = "https://www.gnu.org/software/libtool/" url = "http://ftpmirror.gnu.org/libtool/libtool-2.4.2.tar.gz" + version('2.4.6' , 'addf44b646ddb4e3919805aa88fa7c5e') version('2.4.2' , 'd2f3b7d4627e69e13514a40e72a24d50') def install(self, spec, prefix): From f4b87e29675ef51ab671bc2cbe4bdb4574b9bd77 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 17 Jan 2016 11:05:15 -0500 Subject: [PATCH 321/910] Mbedtls depends on cmake --- var/spack/packages/mbedtls/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/packages/mbedtls/package.py b/var/spack/packages/mbedtls/package.py index 77455229568..3da00cf417f 100644 --- a/var/spack/packages/mbedtls/package.py +++ b/var/spack/packages/mbedtls/package.py @@ -13,6 +13,8 @@ class Mbedtls(Package): version('2.1.3' , '7eb4cf1dfa68578a2c8dbd0b6fa752dd') version('1.3.16', '4144d7320c691f721aeb9e67a1bc38e0') + depends_on('cmake') + def install(self, spec, prefix): cmake('.', *std_cmake_args) From 81f1311a604c3c9474fbb0bc4911a5648e54a1ab Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 17 Jan 2016 14:57:08 -0800 Subject: [PATCH 322/910] Netcdf requires cmake >=2.8.12 --- var/spack/packages/netcdf/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/netcdf/package.py b/var/spack/packages/netcdf/package.py index e1e0d836c62..239644d894c 100644 --- a/var/spack/packages/netcdf/package.py +++ b/var/spack/packages/netcdf/package.py @@ -13,6 +13,7 @@ class Netcdf(Package): patch('netcdf-4.3.3-mpi.patch') # Dependencies: + depends_on("cmake @2.8.12:") # >HDF5 depends_on("hdf5") From 358b2ab4ba9d08bc47b59bf39811b6472acbc53a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 17 Jan 2016 18:12:08 -0800 Subject: [PATCH 323/910] Temporary fix: Clang is the default compiler on Mac OS X. --- lib/spack/spack/compilers/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 2a3d67c731f..facc9c338b2 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -27,6 +27,7 @@ """ import imp import os +import platform from llnl.util.lang import memoized, list_modules from llnl.util.filesystem import join_path @@ -47,7 +48,11 @@ _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] # TODO: customize order in config file -_default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] +if platform.system() == 'Darwin': + _default_order = ['clang', 'gcc', 'intel'] +else: + _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] + def _auto_compiler_spec(function): def converter(cspec_like, *args, **kwargs): From 5984bc2ad3b3d161ac8e8c37ec1dc3dfa9250241 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 17 Jan 2016 18:12:53 -0800 Subject: [PATCH 324/910] Add namespace option to find command. --- lib/spack/spack/cmd/find.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index d951c37fe01..714f1d514b3 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -40,6 +40,9 @@ def setup_parser(subparser): format_group = subparser.add_mutually_exclusive_group() + format_group.add_argument( + '-s', '--short', action='store_const', dest='mode', const='short', + help='Show only specs (default)') format_group.add_argument( '-p', '--paths', action='store_const', dest='mode', const='paths', help='Show paths to package install directories') @@ -48,21 +51,24 @@ def setup_parser(subparser): help='Show full dependency DAG of installed packages') subparser.add_argument( - '-l', '--long', action='store_true', dest='long', + '-l', '--long', action='store_true', help='Show dependency hashes as well as versions.') subparser.add_argument( - '-L', '--very-long', action='store_true', dest='very_long', + '-L', '--very-long', action='store_true', help='Show dependency hashes as well as versions.') subparser.add_argument( - '-u', '--unknown', action='store_true', dest='unknown', + '-u', '--unknown', action='store_true', help='Show only specs Spack does not have a package for.') subparser.add_argument( - '-m', '--missing', action='store_true', dest='missing', + '-m', '--missing', action='store_true', help='Show missing dependencies as well as installed specs.') subparser.add_argument( - '-M', '--only-missing', action='store_true', dest='only_missing', + '-M', '--only-missing', action='store_true', help='Show only missing dependencies.') + subparser.add_argument( + '-N', '--namespace', action='store_true', + help='Show fully qualified package names.') subparser.add_argument( 'query_specs', nargs=argparse.REMAINDER, @@ -76,6 +82,7 @@ def gray_hash(spec, length): def display_specs(specs, **kwargs): mode = kwargs.get('mode', 'short') hashes = kwargs.get('long', False) + namespace = kwargs.get('namespace', False) hlen = 7 if kwargs.get('very_long', False): @@ -97,7 +104,8 @@ def display_specs(specs, **kwargs): specs = index[(architecture,compiler)] specs.sort() - abbreviated = [s.format('$_$@$+', color=True) for s in specs] + nfmt = '.' if namespace else '_' + abbreviated = [s.format('$%s$@$+' % nfmt, color=True) for s in specs] if mode == 'paths': # Print one spec per line along with prefix path width = max(len(s) for s in abbreviated) @@ -112,7 +120,7 @@ def display_specs(specs, **kwargs): elif mode == 'deps': for spec in specs: print spec.tree( - format='$_$@$+', + format='$%s$@$+' % nfmt, color=True, indent=4, prefix=(lambda s: gray_hash(s, hlen)) if hashes else None) @@ -122,7 +130,7 @@ def fmt(s): string = "" if hashes: string += gray_hash(s, hlen) + ' ' - string += s.format('$-_$@$+', color=True) + string += s.format('$-%s$@$+' % nfmt, color=True) return string colify(fmt(s) for s in specs) @@ -171,4 +179,5 @@ def find(parser, args): tty.msg("%d installed packages." % len(specs)) display_specs(specs, mode=args.mode, long=args.long, - very_long=args.very_long) + very_long=args.very_long, + namespace=args.namespace) From 97b492756acce93dbd5f1c305504f07df7582ba0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 17 Jan 2016 18:14:35 -0800 Subject: [PATCH 325/910] Fix create, diy, edit, and repo commands to use multiple repos. --- lib/spack/spack/cmd/create.py | 117 +++++++++++++++++++++++++-------- lib/spack/spack/cmd/diy.py | 2 +- lib/spack/spack/cmd/edit.py | 35 +++++++--- lib/spack/spack/cmd/repo.py | 57 ++++++++-------- lib/spack/spack/repository.py | 103 ++++++++++++++++++++++------- lib/spack/spack/util/naming.py | 25 ++++++- 6 files changed, 247 insertions(+), 92 deletions(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 1a60875de8a..7cea39cb55a 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -36,7 +36,9 @@ import spack.cmd.checksum import spack.url import spack.util.web +from spack.spec import Spec from spack.util.naming import * +from spack.repository import Repo, RepoError import spack.util.crypto as crypto from spack.util.executable import which @@ -85,21 +87,34 @@ def install(self, spec, prefix): """) +def make_version_calls(ver_hash_tuples): + """Adds a version() call to the package for each version found.""" + max_len = max(len(str(v)) for v, h in ver_hash_tuples) + format = " version(%%-%ds, '%%s')" % (max_len + 2) + return '\n'.join(format % ("'%s'" % v, h) for v, h in ver_hash_tuples) + + def setup_parser(subparser): subparser.add_argument('url', nargs='?', help="url of package archive") subparser.add_argument( - '--keep-stage', action='store_true', dest='keep_stage', + '--keep-stage', action='store_true', help="Don't clean up staging area when command completes.") subparser.add_argument( - '-n', '--name', dest='alternate_name', default=None, + '-n', '--name', dest='alternate_name', default=None, metavar='NAME', help="Override the autodetected name for the created package.") subparser.add_argument( - '-p', '--package-repo', dest='package_repo', default=None, - help="Create the package in the specified packagerepo.") + '-r', '--repo', default=None, + help="Path to a repository where the package should be created.") + subparser.add_argument( + '-N', '--namespace', + help="Specify a namespace for the package. Must be the namespace of " + "a repository registered with Spack.") subparser.add_argument( '-f', '--force', action='store_true', dest='force', help="Overwrite any existing package file with the same name.") + setup_parser.subparser = subparser + class ConfigureGuesser(object): def __call__(self, stage): @@ -137,16 +152,7 @@ def __call__(self, stage): self.build_system = build_system -def make_version_calls(ver_hash_tuples): - """Adds a version() call to the package for each version found.""" - max_len = max(len(str(v)) for v, h in ver_hash_tuples) - format = " version(%%-%ds, '%%s')" % (max_len + 2) - return '\n'.join(format % ("'%s'" % v, h) for v, h in ver_hash_tuples) - - -def create(parser, args): - url = args.url - +def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL version = spack.url.parse_version(url) if not version: @@ -163,21 +169,52 @@ def create(parser, args): tty.die("Couldn't guess a name for this package. Try running:", "", "spack create --name ") - package_repo = args.package_repo - - if not valid_module_name(name): + if not valid_fully_qualified_module_name(name): tty.die("Package name can only contain A-Z, a-z, 0-9, '_' and '-'") - tty.msg("This looks like a URL for %s version %s." % (name, version)) - tty.msg("Creating template for package %s" % name) + return name, version - # Create a directory for the new package. - pkg_path = spack.repo.filename_for_package_name(name, package_repo) - if os.path.exists(pkg_path) and not args.force: - tty.die("%s already exists." % pkg_path) + +def find_repository(spec, args): + # figure out namespace for spec + if spec.namespace and args.namespace and spec.namespace != args.namespace: + tty.die("Namespaces '%s' and '%s' do not match." % (spec.namespace, args.namespace)) + + if not spec.namespace and args.namespace: + spec.namespace = args.namespace + + # Figure out where the new package should live. + repo_path = args.repo + if repo_path is not None: + try: + repo = Repo(repo_path) + if spec.namespace and spec.namespace != repo.namespace: + tty.die("Can't create package with namespace %s in repo with namespace %s." + % (spec.namespace, repo.namespace)) + except RepoError as e: + tty.die(str(e)) else: - mkdirp(os.path.dirname(pkg_path)) + if spec.namespace: + repo = spack.repo.get_repo(spec.namespace, None) + if not repo: + tty.die("Unknown namespace: %s" % spec.namespace) + else: + repo = spack.repo.first_repo() + # Set the namespace on the spec if it's not there already + if not spec.namespace: + spec.namespace = repo.namespace + + return repo + + +def fetch_tarballs(url, name, args): + """Try to find versions of the supplied archive by scraping the web. + + Prompts the user to select how many to download if many are found. + + + """ versions = spack.util.web.find_versions_of_archive(url) rkeys = sorted(versions.keys(), reverse=True) versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys))) @@ -196,13 +233,35 @@ def create(parser, args): default=5, abort='q') if not archives_to_fetch: - tty.msg("Aborted.") - return + tty.die("Aborted.") + sorted_versions = sorted(versions.keys(), reverse=True) + sorted_urls = [versions[v] for v in sorted_versions] + return sorted_versions[:archives_to_fetch], sorted_urls[:archives_to_fetch] + + +def create(parser, args): + url = args.url + if not url: + setup_parser.subparser.print_help() + return + + # Figure out a name and repo for the package. + name, version = guess_name_and_version(url, args) + spec = Spec(name) + name = spec.name # factors out namespace, if any + repo = find_repository(spec, args) + + tty.msg("This looks like a URL for %s version %s." % (name, version)) + tty.msg("Creating template for package %s" % name) + + # Fetch tarballs (prompting user if necessary) + versions, urls = fetch_tarballs(url, name, args) + + # Try to guess what configure system is used. guesser = ConfigureGuesser() ver_hash_tuples = spack.cmd.checksum.get_checksums( - versions.keys()[:archives_to_fetch], - [versions[v] for v in versions.keys()[:archives_to_fetch]], + versions, urls, first_stage_function=guesser, keep_stage=args.keep_stage) @@ -214,7 +273,7 @@ def create(parser, args): name = 'py-%s' % name # Create a directory for the new package. - pkg_path = spack.repo.filename_for_package_name(name) + pkg_path = repo.filename_for_package_name(name) if os.path.exists(pkg_path) and not args.force: tty.die("%s already exists." % pkg_path) else: diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 1acbebbc15a..9df53312f80 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -69,7 +69,7 @@ def diy(self, args): sys.exit(1) else: tty.msg("Running 'spack edit -f %s'" % spec.name) - edit_package(spec.name, True) + edit_package(spec.name, spack.repo.first_repo(), None, True) return if not spec.version.concrete: diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index e0688dc96b0..a20e40df9bd 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -30,6 +30,8 @@ import spack import spack.cmd +from spack.spec import Spec +from spack.repository import Repo from spack.util.naming import mod_to_class description = "Open package files in $EDITOR" @@ -53,9 +55,16 @@ def install(self, spec, prefix): """) -def edit_package(name, force=False): - path = spack.repo.filename_for_package_name(name) +def edit_package(name, repo_path, namespace, force=False): + if repo_path: + repo = Repo(repo_path) + elif namespace: + repo = spack.repo.get_repo(namespace) + else: + repo = spack.repo + path = repo.filename_for_package_name(name) + spec = Spec(name) if os.path.exists(path): if not os.path.isfile(path): tty.die("Something's wrong. '%s' is not a file!" % path) @@ -63,13 +72,13 @@ def edit_package(name, force=False): tty.die("Insufficient permissions on '%s'!" % path) elif not force: tty.die("No package '%s'. Use spack create, or supply -f/--force " - "to edit a new file." % name) + "to edit a new file." % spec.name) else: mkdirp(os.path.dirname(path)) with open(path, "w") as pkg_file: pkg_file.write( package_template.substitute( - name=name, class_name=mod_to_class(name))) + name=spec.name, class_name=mod_to_class(spec.name))) spack.editor(path) @@ -79,17 +88,25 @@ def setup_parser(subparser): '-f', '--force', dest='force', action='store_true', help="Open a new file in $EDITOR even if package doesn't exist.") - filetypes = subparser.add_mutually_exclusive_group() - filetypes.add_argument( + excl_args = subparser.add_mutually_exclusive_group() + + # Various filetypes you can edit directly from the cmd line. + excl_args.add_argument( '-c', '--command', dest='path', action='store_const', const=spack.cmd.command_path, help="Edit the command with the supplied name.") - filetypes.add_argument( + excl_args.add_argument( '-t', '--test', dest='path', action='store_const', const=spack.test_path, help="Edit the test with the supplied name.") - filetypes.add_argument( + excl_args.add_argument( '-m', '--module', dest='path', action='store_const', const=spack.module_path, help="Edit the main spack module with the supplied name.") + # Options for editing packages + excl_args.add_argument( + '-r', '--repo', default=None, help="Path to repo to edit package in.") + excl_args.add_argument( + '-N', '--namespace', default=None, help="Namespace of package to edit.") + subparser.add_argument( 'name', nargs='?', default=None, help="name of package to edit") @@ -107,7 +124,7 @@ def edit(parser, args): spack.editor(path) elif name: - edit_package(name, args.force) + edit_package(name, args.repo, args.namespace, args.force) else: # By default open the directory where packages or commands live. spack.editor(path) diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index a792f04cfde..34c755fb67f 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -44,9 +44,10 @@ def setup_parser(subparser): # Create create_parser = sp.add_parser('create', help=repo_create.__doc__) create_parser.add_argument( - 'namespace', help="Namespace to identify packages in the repository.") + 'directory', help="Directory to create the repo in.") create_parser.add_argument( - 'directory', help="Directory to create the repo in. Defaults to same as namespace.", nargs='?') + 'namespace', help="Namespace to identify packages in the repository. " + "Defaults to the directory name.", nargs='?') # List list_parser = sp.add_parser('list', help=repo_list.__doc__) @@ -72,14 +73,15 @@ def setup_parser(subparser): def repo_create(args): - """Create a new package repo for a particular namespace.""" + """Create a new package repository.""" + root = canonicalize_path(args.directory) namespace = args.namespace - if not re.match(r'\w[\.\w-]*', namespace): - tty.die("Invalid namespace: '%s'" % namespace) - root = args.directory - if not root: - root = namespace + if not args.namespace: + namespace = os.path.basename(root) + + if not re.match(r'\w[\.\w-]*', namespace): + tty.die("'%s' is not a valid namespace." % namespace) existed = False if os.path.exists(root): @@ -123,27 +125,22 @@ def repo_create(args): def repo_add(args): - """Add a package source to the Spack configuration""" + """Add a package source to Spack's configuration.""" path = args.path - # check if the path is relative to the spack directory. - real_path = path - if path.startswith('$spack'): - real_path = spack.repository.substitute_spack_prefix(path) - elif not os.path.isabs(real_path): - real_path = os.path.abspath(real_path) - path = real_path + # real_path is absolute and handles substitution. + canon_path = canonicalize_path(path) # check if the path exists - if not os.path.exists(real_path): + if not os.path.exists(canon_path): tty.die("No such file or directory: '%s'." % path) # Make sure the path is a directory. - if not os.path.isdir(real_path): + if not os.path.isdir(canon_path): tty.die("Not a Spack repository: '%s'." % path) # Make sure it's actually a spack repository by constructing it. - repo = Repo(real_path) + repo = Repo(canon_path) # If that succeeds, finally add it to the configuration. repos = spack.config.get_config('repos', args.scope) @@ -152,30 +149,32 @@ def repo_add(args): if repo.root in repos or path in repos: tty.die("Repository is already registered with Spack: '%s'" % path) - repos.insert(0, path) + repos.insert(0, canon_path) spack.config.update_config('repos', repos, args.scope) tty.msg("Created repo with namespace '%s'." % repo.namespace) def repo_remove(args): - """Remove a repository from the Spack configuration.""" + """Remove a repository from Spack's configuration.""" repos = spack.config.get_config('repos', args.scope) path_or_namespace = args.path_or_namespace # If the argument is a path, remove that repository from config. - path = os.path.abspath(path_or_namespace) - if path in repos: - repos.remove(path) - spack.config.update_config('repos', repos, args.scope) - tty.msg("Removed repository '%s'." % path) - return + canon_path = canonicalize_path(path_or_namespace) + for repo_path in repos: + repo_canon_path = canonicalize_path(repo_path) + if canon_path == repo_canon_path: + repos.remove(repo_path) + spack.config.update_config('repos', repos, args.scope) + tty.msg("Removed repository '%s'." % repo_path) + return # If it is a namespace, remove corresponding repo for path in repos: try: repo = Repo(path) if repo.namespace == path_or_namespace: - repos.remove(repo.root) + repos.remove(path) spack.config.update_config('repos', repos, args.scope) tty.msg("Removed repository '%s' with namespace %s." % (repo.root, repo.namespace)) @@ -188,7 +187,7 @@ def repo_remove(args): def repo_list(args): - """List package sources and their mnemoics""" + """Show registered repositories and their namespaces.""" roots = spack.config.get_config('repos', args.scope) repos = [] for r in roots: diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 3367572ef50..31596cee7a5 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -54,6 +54,9 @@ packages_dir_name = 'packages' # Top-level repo directory containing pkgs. package_file_name = 'package.py' # Filename for packages in a repository. +# Guaranteed unused default value for some functions. +NOT_PROVIDED = object() + def _autospec(function): """Decorator that automatically converts the argument of a single-arg @@ -75,7 +78,15 @@ def _make_namespace_module(ns): def substitute_spack_prefix(path): """Replaces instances of $spack with Spack's prefix.""" - return path.replace('$spack', spack.prefix) + return re.sub(r'^\$spack', spack.prefix, path) + + +def canonicalize_path(path): + """Substitute $spack, expand user home, take abspath.""" + path = substitute_spack_prefix(path) + path = os.path.expanduser(path) + path = os.path.abspath(path) + return path class RepoPath(object): @@ -109,7 +120,10 @@ def __init__(self, *repo_dirs, **kwargs): repo = Repo(root, self.super_namespace) self.put_last(repo) except RepoError as e: - tty.warn("Failed to initialize repository at '%s'." % root, e.message) + tty.warn("Failed to initialize repository at '%s'." % root, + e.message, + "To remove the bad repository, run this command:", + " spack repo rm %s" % root) def swap(self, other): @@ -173,6 +187,31 @@ def remove(self, repo): self.repos.remove(repo) + def get_repo(self, namespace, default=NOT_PROVIDED): + """Get a repository by namespace. + Arguments + namespace + Look up this namespace in the RepoPath, and return + it if found. + + Optional Arguments + default + If default is provided, return it when the namespace + isn't found. If not, raise an UnknownNamespaceError. + """ + fullspace = '%s.%s' % (self.super_namespace, namespace) + if fullspace not in self.by_namespace: + if default == NOT_PROVIDED: + raise UnknownNamespaceError(namespace) + return default + return self.by_namespace[fullspace] + + + def first_repo(self): + """Get the first repo in precedence order.""" + return self.repos[0] if self.repos else None + + def all_package_names(self): """Return all unique package names in all repositories.""" return self._all_package_names @@ -229,7 +268,6 @@ def load_module(self, fullname): if fullname in sys.modules: return sys.modules[fullname] - # partition fullname into prefix and module name. namespace, dot, module_name = fullname.rpartition('.') @@ -242,11 +280,23 @@ def load_module(self, fullname): return module - def repo_for_pkg(self, pkg_name): + @_autospec + def repo_for_pkg(self, spec): + """Given a spec, get the repository for its package.""" + # If the spec already has a namespace, then return the + # corresponding repo if we know about it. + if spec.namespace: + fullspace = '%s.%s' % (self.super_namespace, spec.namespace) + if fullspace not in self.by_namespace: + raise UnknownNamespaceError(spec.namespace) + return self.by_namespace[fullspace] + + # If there's no namespace, search in the RepoPath. for repo in self.repos: - if pkg_name in repo: + if spec.name in repo: return repo - raise UnknownPackageError(pkg_name) + else: + raise UnknownPackageError(spec.name) @_autospec @@ -255,16 +305,7 @@ def get(self, spec, new=False): Raises UnknownPackageError if not found. """ - # if the spec has a fully qualified namespace, we grab it - # directly and ignore overlay precedence. - if spec.namespace: - fullspace = '%s.%s' % (self.super_namespace, spec.namespace) - if not fullspace in self.by_namespace: - raise UnknownPackageError( - "No configured repository contains package %s." % spec.fullname) - return self.by_namespace[fullspace].get(spec) - else: - return self.repo_for_pkg(spec.name).get(spec) + return self.repo_for_pkg(spec).get(spec) def dirname_for_package_name(self, pkg_name): @@ -310,7 +351,7 @@ def __init__(self, root, namespace=repo_namespace): """ # Root directory, containing _repo.yaml and package dirs # Allow roots to by spack-relative by starting with '$spack' - self.root = substitute_spack_prefix(root) + self.root = canonicalize_path(root) # super-namespace for all packages in the Repo self.super_namespace = namespace @@ -330,7 +371,7 @@ def check(condition, msg): # Read configuration and validate namespace config = self._read_config() check('namespace' in config, '%s must define a namespace.' - % join_path(self.root, repo_config_name)) + % join_path(root, repo_config_name)) self.namespace = config['namespace'] check(re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', self.namespace), @@ -524,13 +565,22 @@ def extensions_for(self, extendee_spec): return [p for p in self.all_packages() if p.extends(extendee_spec)] - def dirname_for_package_name(self, pkg_name): + def _check_namespace(self, spec): + """Check that the spec's namespace is the same as this repository's.""" + if spec.namespace and spec.namespace != self.namespace: + raise UnknownNamespaceError(spec.namespace) + + + @_autospec + def dirname_for_package_name(self, spec): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" - return join_path(self.packages_path, pkg_name) + self._check_namespace(spec) + return join_path(self.packages_path, spec.name) - def filename_for_package_name(self, pkg_name): + @_autospec + def filename_for_package_name(self, spec): """Get the filename for the module we should load for a particular package. Packages for a Repo live in ``$root//package.py`` @@ -539,8 +589,8 @@ def filename_for_package_name(self, pkg_name): package doesn't exist yet, so callers will need to ensure the package exists before importing. """ - validate_module_name(pkg_name) - pkg_dir = self.dirname_for_package_name(pkg_name) + self._check_namespace(spec) + pkg_dir = self.dirname_for_package_name(spec.name) return join_path(pkg_dir, package_file_name) @@ -679,6 +729,13 @@ def __init__(self, name, repo=None): self.name = name +class UnknownNamespaceError(PackageLoadError): + """Raised when we encounter an unknown namespace""" + def __init__(self, namespace): + super(UnknownNamespaceError, self).__init__( + "Unknown namespace: %s" % namespace) + + class FailedConstructorError(PackageLoadError): """Raised when a package's class constructor fails.""" def __init__(self, name, exc_type, exc_obj, exc_tb): diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py index 26ca86c77fa..5025f150278 100644 --- a/lib/spack/spack/util/naming.py +++ b/lib/spack/spack/util/naming.py @@ -8,11 +8,15 @@ import spack __all__ = ['mod_to_class', 'spack_module_to_python_module', 'valid_module_name', + 'valid_fully_qualified_module_name', 'validate_fully_qualified_module_name', 'validate_module_name', 'possible_spack_module_names', 'NamespaceTrie'] # Valid module names can contain '-' but can't start with it. _valid_module_re = r'^\w[\w-]*$' +# Valid module names can contain '-' but can't start with it. +_valid_fully_qualified_module_re = r'^(\w[\w-]*)(\.\w[\w-]*)*$' + def mod_to_class(mod_name): """Convert a name from module style to class name style. Spack mostly @@ -75,16 +79,27 @@ def possible_spack_module_names(python_mod_name): def valid_module_name(mod_name): - """Return whether the mod_name is valid for use in Spack.""" + """Return whether mod_name is valid for use in Spack.""" return bool(re.match(_valid_module_re, mod_name)) +def valid_fully_qualified_module_name(mod_name): + """Return whether mod_name is a valid namespaced module name.""" + return bool(re.match(_valid_fully_qualified_module_re, mod_name)) + + def validate_module_name(mod_name): """Raise an exception if mod_name is not valid.""" if not valid_module_name(mod_name): raise InvalidModuleNameError(mod_name) +def validate_fully_qualified_module_name(mod_name): + """Raise an exception if mod_name is not a valid namespaced module name.""" + if not valid_fully_qualified_module_name(mod_name): + raise InvalidFullyQualifiedModuleNameError(mod_name) + + class InvalidModuleNameError(spack.error.SpackError): """Raised when we encounter a bad module name.""" def __init__(self, name): @@ -93,6 +108,14 @@ def __init__(self, name): self.name = name +class InvalidFullyQualifiedModuleNameError(spack.error.SpackError): + """Raised when we encounter a bad full package name.""" + def __init__(self, name): + super(InvalidFullyQualifiedModuleNameError, self).__init__( + "Invalid fully qualified package name: " + name) + self.name = name + + class NamespaceTrie(object): class Element(object): def __init__(self, value): From 4bab6f9fdb950b958d439d217f0aec671f336727 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 17 Jan 2016 18:55:36 -0800 Subject: [PATCH 326/910] Update cmake package to use http in all URLs. --- var/spack/packages/cmake/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/cmake/package.py b/var/spack/packages/cmake/package.py index 173f72742c4..f67ae21ebd0 100644 --- a/var/spack/packages/cmake/package.py +++ b/var/spack/packages/cmake/package.py @@ -31,12 +31,12 @@ class Cmake(Package): version('2.8.10.2', '097278785da7182ec0aea8769d06860c', url = 'http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz') - + version('3.0.2', 'db4c687a31444a929d2fdc36c4dfb95f', url = 'http://www.cmake.org/files/v3.0/cmake-3.0.2.tar.gz') - + version('3.4.0', 'cd3034e0a44256a0917e254167217fc8', - url = 'https://cmake.org/files/v3.4/cmake-3.4.0.tar.gz') + url = 'http://cmake.org/files/v3.4/cmake-3.4.0.tar.gz') variant('ncurses', default=True, description='Enables the build of the ncurses gui') From 58162ec17070db5476b3aeffe1ef40d676a62949 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 18 Jan 2016 16:03:41 -0600 Subject: [PATCH 327/910] Add verbs variant, remove deprecated configure flags --- var/spack/packages/mpich/package.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py index 00b7dfda751..128a1f44aef 100644 --- a/var/spack/packages/mpich/package.py +++ b/var/spack/packages/mpich/package.py @@ -41,6 +41,8 @@ class Mpich(Package): version('3.1', '5643dd176499bfb7d25079aaff25f2ec') version('3.0.4', '9c5d5d4fe1e17dd12153f40bc5b6dbc0') + variant('verbs', default=False, description='Build support for OpenFabrics verbs.') + provides('mpi@:3.0', when='@3:') provides('mpi@:1.3', when='@1:') @@ -56,14 +58,20 @@ def install(self, spec, prefix): config_args = ["--prefix=" + prefix, "--enable-shared"] + # Variants + if '+verbs' in spec: + config_args.append("--with-ibverbs") + else: + config_args.append("--without-ibverbs") + # TODO: Spack should make it so that you can't actually find # these compilers if they're "disabled" for the current # compiler configuration. if not self.compiler.f77: - config_args.append("--disable-f77") + config_args.append("--disable-fortran=f77") if not self.compiler.fc: - config_args.append("--disable-fc") + config_args.append("--disable-fortran=fc") configure(*config_args) make() From ad32f64ef6120ab2bc9da739f3032b2ba304a2f0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 17 Jan 2016 19:55:20 -0800 Subject: [PATCH 328/910] Allow completely empty config files. - Previous version would give validation error for an empty file. Now this is properly ignored. - Also includes bugfix in ConfigFormatError --- lib/spack/spack/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 193b3114349..6fecde99801 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -357,8 +357,8 @@ def _read_config_file(filename, schema): with open(filename) as f: data = syaml.load(f) - validate_section(data, schema) - + if data: + validate_section(data, schema) return data except MarkedYAMLError, e: @@ -514,7 +514,7 @@ def __init__(self, validation_error, data): # Try really hard to get the parent (which sometimes is not # set) This digs it out of the validated structure if it's not # on the validation_error. - if not parent_mark: + if path and not parent_mark: parent_path = list(path)[:-1] parent = get_path(parent_path, data) if path[-1] in parent: From cee7bfa9f0e5dc0ec8f8641327d1dba56dc4510d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 18 Jan 2016 00:01:30 -0800 Subject: [PATCH 329/910] Simplify output redirection in spack.util.executable - By default inherit parent's input/output descriptor - Only use pipes if we need to return output. - Allows subprocesses (like emacsclient) to detect terminal correctly --- lib/spack/spack/compiler.py | 2 +- lib/spack/spack/util/executable.py | 25 ++++++++++--------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index b9abf943e80..a665f6062d2 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -51,7 +51,7 @@ def _verify_executables(*paths): def get_compiler_version(compiler_path, version_arg, regex='(.*)'): if not compiler_path in _version_cache: compiler = Executable(compiler_path) - output = compiler(version_arg, return_output=True, error=None) + output = compiler(version_arg, return_output=True, error=os.devnull) match = re.search(regex, output) _version_cache[compiler_path] = match.group(1) if match else 'unknown' diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index 35bf0866162..ba765eb6629 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -60,16 +60,16 @@ def __call__(self, *args, **kwargs): fail_on_error = kwargs.get("fail_on_error", True) ignore_errors = kwargs.get("ignore_errors", ()) - output = kwargs.get("output", sys.stdout) - error = kwargs.get("error", sys.stderr) + # Default values of None says to keep parent's file descriptors. + output = kwargs.get("output", None) + error = kwargs.get("error", None) input = kwargs.get("input", None) def streamify(arg, mode): if isinstance(arg, basestring): return open(arg, mode), True - elif arg is None and mode != 'r': - return open(os.devnull, mode), True - return arg, False + else: + return arg, False output, ostream = streamify(output, 'w') error, estream = streamify(error, 'w') input, istream = streamify(input, 'r') @@ -92,19 +92,14 @@ def streamify(arg, mode): tty.debug(cmd_line) try: + if return_output: + output = subprocess.PIPE + proc = subprocess.Popen( - cmd, - stdin=input, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) + cmd, stdin=input, stderr=error, stdout=output) out, err = proc.communicate() - self.returncode = proc.returncode - if not return_output: - output.write(out) - error.write(err) - - rc = proc.returncode + rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" % proc.returncode, cmd_line) From 67ea7ce0ab2b89b19d73ea60285ba30bccebf4fb Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 18 Jan 2016 00:36:06 -0800 Subject: [PATCH 330/910] Fix LaunchMon on newer gcc versions. --- var/spack/repos/builtin/packages/launchmon/package.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/var/spack/repos/builtin/packages/launchmon/package.py b/var/spack/repos/builtin/packages/launchmon/package.py index f97384a249f..aec2fd6fa7e 100644 --- a/var/spack/repos/builtin/packages/launchmon/package.py +++ b/var/spack/repos/builtin/packages/launchmon/package.py @@ -37,6 +37,16 @@ class Launchmon(Package): depends_on('automake') depends_on('libtool') + + def patch(self): + # This patch makes libgcrypt compile correctly with newer gcc versions. + mf = FileFilter('tools/libgcrypt/tests/Makefile.in') + mf.filter(r'(basic_LDADD\s*=\s*.*)', r'\1 -lgpg-error') + mf.filter(r'(tsexp_LDADD\s*=\s*.*)', r'\1 -lgpg-error') + mf.filter(r'(keygen_LDADD\s*=\s*.*)', r'\1 -lgpg-error') + mf.filter(r'(benchmark_LDADD\s*=\s*.*)', r'\1 -lgpg-error') + + def install(self, spec, prefix): configure( "--prefix=" + prefix, From b7b4faff96dbc4d1559fd06d1e9f1327590f1cc7 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 18 Jan 2016 03:01:46 -0800 Subject: [PATCH 331/910] disable cairo-trace, which is incompatible with older libiberty.h. --- var/spack/repos/builtin/packages/cairo/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/cairo/package.py b/var/spack/repos/builtin/packages/cairo/package.py index e1ac8aaa7d7..8255e869bea 100644 --- a/var/spack/repos/builtin/packages/cairo/package.py +++ b/var/spack/repos/builtin/packages/cairo/package.py @@ -14,6 +14,7 @@ class Cairo(Package): def install(self, spec, prefix): configure("--prefix=%s" % prefix, + "--disable-trace", # can cause problems with libiberty "--enable-tee") make() make("install") From 047f1b9de9befd06c511b4d72a80a8b4c40a018a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 18 Jan 2016 03:02:21 -0800 Subject: [PATCH 332/910] disable parallel install for glib (found races) --- var/spack/repos/builtin/packages/glib/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/glib/package.py b/var/spack/repos/builtin/packages/glib/package.py index baca1a5a459..67ead5f9416 100644 --- a/var/spack/repos/builtin/packages/glib/package.py +++ b/var/spack/repos/builtin/packages/glib/package.py @@ -16,4 +16,4 @@ class Glib(Package): def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() - make("install") + make("install", parallel=False) From 01c5b53ba16a95ab77918d30dfa3a63f2ef2707f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 18 Jan 2016 03:02:58 -0800 Subject: [PATCH 333/910] Make libxcb compile with gcc 4.9. --- var/spack/repos/builtin/packages/libxcb/package.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/libxcb/package.py b/var/spack/repos/builtin/packages/libxcb/package.py index 16a5525c0d3..1dd5954c997 100644 --- a/var/spack/repos/builtin/packages/libxcb/package.py +++ b/var/spack/repos/builtin/packages/libxcb/package.py @@ -1,9 +1,9 @@ from spack import * class Libxcb(Package): - """The X protocol C-language Binding (XCB) is a replacement - for Xlib featuring a small footprint, latency hiding, direct - access to the protocol, improved threading support, and + """The X protocol C-language Binding (XCB) is a replacement + for Xlib featuring a small footprint, latency hiding, direct + access to the protocol, improved threading support, and extensibility.""" homepage = "http://xcb.freedesktop.org/" @@ -14,6 +14,10 @@ class Libxcb(Package): depends_on("python") depends_on("xcb-proto") + def patch(self): + filter_file('typedef struct xcb_auth_info_t {', 'typedef struct {', 'src/xcb.h') + + def install(self, spec, prefix): configure("--prefix=%s" % prefix) From f48fc0d8b184e26accc570910b8f4244d4d9e63f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 19 Jan 2016 09:43:28 -0600 Subject: [PATCH 334/910] Revert deprecated configure flags --- var/spack/repos/builtin/packages/mpich/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 128a1f44aef..c856cfe277f 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -68,10 +68,10 @@ def install(self, spec, prefix): # these compilers if they're "disabled" for the current # compiler configuration. if not self.compiler.f77: - config_args.append("--disable-fortran=f77") + config_args.append("--disable-f77") if not self.compiler.fc: - config_args.append("--disable-fortran=fc") + config_args.append("--disable-fc") configure(*config_args) make() From 0d42cdaffdf12fa3717135b07fade947dcb6b3b7 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 19 Jan 2016 10:09:25 -0600 Subject: [PATCH 335/910] Merge branch 'develop' into features/hdf --- var/spack/{ => repos/builtin}/packages/hdf/package.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename var/spack/{ => repos/builtin}/packages/hdf/package.py (100%) diff --git a/var/spack/packages/hdf/package.py b/var/spack/repos/builtin/packages/hdf/package.py similarity index 100% rename from var/spack/packages/hdf/package.py rename to var/spack/repos/builtin/packages/hdf/package.py From e121faffa1974bff9570eb8b7a2ba83fdd624f0e Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 19 Jan 2016 17:26:26 +0100 Subject: [PATCH 336/910] eigen : added package --- .../repos/builtin/packages/eigen/package.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 var/spack/repos/builtin/packages/eigen/package.py diff --git a/var/spack/repos/builtin/packages/eigen/package.py b/var/spack/repos/builtin/packages/eigen/package.py new file mode 100644 index 00000000000..44ee6819f58 --- /dev/null +++ b/var/spack/repos/builtin/packages/eigen/package.py @@ -0,0 +1,68 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by David Beckingsale, david@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Eigen(Package): + """ + Eigen is a C++ template library for linear algebra: matrices, vectors, numerical solvers, and related algorithms + """ + + homepage = 'http://eigen.tuxfamily.org/' + url = 'http://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2' + + version('3.2.7', 'cc1bacbad97558b97da6b77c9644f184', url='http://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2') + + variant('debug', default=False, description='Builds the library in debug mode') + + variant('metis', default=True, description='Enables metis backend') + variant('scotch', default=True, description='Enables scotch backend') + variant('fftw', default=True, description='Enables FFTW backend') + + # TODO : dependency on SuiteSparse, googlehash, superlu, adolc missing + + depends_on('metis', when='+metis') + depends_on('scotch', when='+scotch') + depends_on('fftw', when='+fftw') + + depends_on('mpfr@2.3.0:') # Eigen 3.2.7 requires at least 2.3.0 + depends_on('gmp') + + def install(self, spec, prefix): + + options = [] + options.extend(std_cmake_args) + + build_directory = join_path(self.stage.path, 'spack-build') + source_directory = self.stage.source_path + + if '+debug' in spec: + options.append('-DCMAKE_BUILD_TYPE:STRING=Debug') + + with working_dir(build_directory, create=True): + cmake(source_directory, *options) + make() + make("install") From 98706d31dca0773c51acfabb8f1a399d4720a875 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 19 Jan 2016 12:46:15 -0600 Subject: [PATCH 337/910] Various updates to MVAPICH2 package Use url_for_version function to specify urls Add version 2.2b Add dedicated MRAIL variant Fix typo Make fortran compiler flags more specific --- .../builtin/packages/mvapich2/package.py | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index 23a11b3171a..af5ed1b0888 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -4,15 +4,13 @@ class Mvapich2(Package): """MVAPICH2 is an MPI implementation for Infiniband networks.""" homepage = "http://mvapich.cse.ohio-state.edu/" + url = "http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.2b.tar.gz" - version('2.2a', 'b8ceb4fc5f5a97add9b3ff1b9cbe39d2', - url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.2a.tar.gz') + version('2.2b', '5651e8b7a72d7c77ca68da48f3a5d108') + version('2.2a', 'b8ceb4fc5f5a97add9b3ff1b9cbe39d2') + version('2.0', '9fbb68a4111a8b6338e476dc657388b4') + version('1.9', '5dc58ed08fd3142c260b70fe297e127c') - version('2.0', '9fbb68a4111a8b6338e476dc657388b4', - url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.0.tar.gz') - - version('1.9', '5dc58ed08fd3142c260b70fe297e127c', - url="http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz") patch('ad_lustre_rwcontig_open_source.patch', when='@1.9') provides('mpi@:2.2', when='@1.9') # MVAPICH2-1.9 supports MPI 2.2 @@ -41,16 +39,25 @@ class Mvapich2(Package): NEMESISIBTCP = 'nemesisibtcp' NEMESISIB = 'nemesisib' NEMESIS = 'nemesis' + MRAIL = 'mrail' SUPPORTED_NETWORKS = (PSM, SOCK, NEMESIS, NEMESISIB, NEMESISIBTCP) variant(PSM, default=False, description='Configures a build for QLogic PSM-CH3') variant(SOCK, default=False, description='Configures a build for TCP/IP-CH3') variant(NEMESISIBTCP, default=False, description='Configures a build for both OFA-IB-Nemesis and TCP/IP-Nemesis') variant(NEMESISIB, default=False, description='Configures a build for OFA-IB-Nemesis') variant(NEMESIS, default=False, description='Configures a build for TCP/IP-Nemesis') + variant(MRAIL, default=False, description='Configures a build for OFA-IB-CH3') ########## # FIXME : CUDA support is missing + def url_for_version(self, version): + base_url = "http://mvapich.cse.ohio-state.edu/download" + if version < Version('2.0'): + return "%s/mvapich2/mv2/mvapich2-%s.tar.gz" % (base_url, version) + else: + return "%s/mvapich/mv2/mvapich2-%s.tar.gz" % (base_url, version) + @staticmethod def enabled(x): """ @@ -117,7 +124,7 @@ def set_network_type(self, spec, configure_args): if count > 1: raise RuntimeError('network variants are mutually exclusive (only one can be selected at a time)') - # From here on I can suppose that ony one variant has been selected + # From here on I can suppose that only one variant has been selected if self.enabled(Mvapich2.PSM) in spec: network_options = ["--with-device=ch3:psm"] elif self.enabled(Mvapich2.SOCK) in spec: @@ -128,7 +135,7 @@ def set_network_type(self, spec, configure_args): network_options = ["--with-device=ch3:nemesis:ib"] elif self.enabled(Mvapich2.NEMESIS) in spec: network_options = ["--with-device=ch3:nemesis"] - else: + elif self.enabled(Mvapich2.MRAIL) in spec: network_options = ["--with-device=ch3:mrail", "--with-rdma=gen2"] configure_args.extend(network_options) @@ -141,7 +148,14 @@ def install(self, spec, prefix): "--enable-romio", "--disable-silent-rules", ] - if not self.compiler.f77 and not self.compiler.fc: + + if self.compiler.f77 and self.compiler.fc: + configure_args.append("--enable-fortran=all") + elif self.compiler.f77: + configure_args.append("--enable-fortran=f77") + elif self.compiler.fc: + configure_args.append("--enable-fortran=fc") + else: configure_args.append("--enable-fortran=none") # Set the type of the build (debug, release) From e6aa2610bc4314cab09de61b4f7b632409299fb2 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 19 Jan 2016 13:07:07 -0600 Subject: [PATCH 338/910] Add symlink for NAG Fortran Compiler --- lib/spack/env/nag/nagfor | 1 + 1 file changed, 1 insertion(+) create mode 120000 lib/spack/env/nag/nagfor diff --git a/lib/spack/env/nag/nagfor b/lib/spack/env/nag/nagfor new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/nag/nagfor @@ -0,0 +1 @@ +../cc \ No newline at end of file From 83de658ee42e08a27ce7d85d295955f4809c03dc Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 19 Jan 2016 13:28:35 -0600 Subject: [PATCH 339/910] Modify nag.py to match new compiler package structure --- lib/spack/spack/compilers/nag.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index 2dfc97af739..f1cc6be0d5c 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -1,3 +1,5 @@ +from spack.compiler import * + class Nag(Compiler): # Subclasses use possible names of C compiler cc_names = [] @@ -11,6 +13,13 @@ class Nag(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['nagfor'] + # Named wrapper links within spack.build_env_path + link_paths = { # Use default wrappers for C and C++, in case provided in compilers.yaml + 'cc' : 'cc', + 'cxx' : 'cxx', + 'f77' : 'nag/nagfor', + 'fc' : 'nag/nagfor' } + @classmethod def default_version(self, comp): """The '-V' option works for nag compilers. From 6c9800bcc395ecbc319f4a3f6242efee4b972470 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Tue, 19 Jan 2016 15:22:16 -0500 Subject: [PATCH 340/910] typos: fix some letter transposes --- lib/spack/spack/cmd/deactivate.py | 2 +- lib/spack/spack/database.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index a0c78bf755b..d6b23d6a08e 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -37,7 +37,7 @@ def setup_parser(subparser): help="Run deactivation even if spec is NOT currently activated.") subparser.add_argument( '-a', '--all', action='store_true', - help="Deactivate all extensions of an extendable pacakge, or " + help="Deactivate all extensions of an extendable package, or " "deactivate an extension AND its dependencies.") subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="spec of package extension to deactivate.") diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 0fa18db34b8..9cbe7de44a3 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -489,7 +489,7 @@ def remove(self, spec): 1. Marks the spec as not installed. 2. Removes the spec if it has no more dependents. 3. If removed, recursively updates dependencies' ref counts - and remvoes them if they are no longer needed. + and removes them if they are no longer needed. """ # Take a lock around the entire removal. From ab27fdc692193fa2deeacfbb2aabc1f6feb06f6d Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 20 Jan 2016 09:35:15 +0100 Subject: [PATCH 341/910] elpa : commit partial package. Blocked by missing scalapack --- .../repos/builtin/packages/elpa/package.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 var/spack/repos/builtin/packages/elpa/package.py diff --git a/var/spack/repos/builtin/packages/elpa/package.py b/var/spack/repos/builtin/packages/elpa/package.py new file mode 100644 index 00000000000..6825056a04b --- /dev/null +++ b/var/spack/repos/builtin/packages/elpa/package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Elpa(Package): + """ + Eigenvalue solvers for Petaflop-Applications (ELPA) + """ + + homepage = 'http://elpa.mpcdf.mpg.de/' + url = 'http://elpa.mpcdf.mpg.de/elpa-2015.11.001.tar.gz' + + version('2015.11.001', 'de0f35b7ee7c971fd0dca35c900b87e6', url='http://elpa.mpcdf.mpg.de/elpa-2015.11.001.tar.gz') + + depends_on('mpi') + depends_on('blas') + depends_on('lapack') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") From 72d5cdf9abaee156d80dd027911a6cadbfebd7bb Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 20 Jan 2016 11:19:36 +0100 Subject: [PATCH 342/910] fixed two minor typos --- lib/spack/docs/site_configuration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index a7211a9d95b..8ab4e89dfce 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -76,9 +76,9 @@ directory. Here's an example of an external configuration:: path: /opt/openmpi-1.6.5-intel This example lists three installations of OpenMPI, one built with gcc, -one built with gcc and debug information, and another built with OpenMPI. +one built with gcc and debug information, and another built with Intel. If Spack is asked to build a package that uses one of these MPIs as a -dependency, it link the package to the pre-installed OpenMPI in +dependency, it links the package to the pre-installed OpenMPI in the given directory. Each ``packages.yaml`` should begin with a ``packages:`` token, followed From 65f65ec4e47289e15e5d6a25913beb8e6bb4831b Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 20 Jan 2016 10:03:24 +0100 Subject: [PATCH 343/910] Adding MUMPS and ScaLAPACK --- var/spack/packages/mumps/Makefile.inc | 41 ++++++ var/spack/packages/mumps/package.py | 121 ++++++++++++++++++ .../packages/netlib-scalapack/package.py | 43 +++++++ 3 files changed, 205 insertions(+) create mode 100644 var/spack/packages/mumps/Makefile.inc create mode 100644 var/spack/packages/mumps/package.py create mode 100644 var/spack/packages/netlib-scalapack/package.py diff --git a/var/spack/packages/mumps/Makefile.inc b/var/spack/packages/mumps/Makefile.inc new file mode 100644 index 00000000000..4b6696db5c3 --- /dev/null +++ b/var/spack/packages/mumps/Makefile.inc @@ -0,0 +1,41 @@ +LPORDDIR = $(topdir)/PORD/lib/ +IPORD = -I$(topdir)/PORD/include/ +LPORD = -L$(LPORDDIR) -lpord + +ORDERINGSC = $(ORDERINGSF) +LORDERINGS = $(LMETIS) $(LPORD) $(LSCOTCH) +IORDERINGSF = $(ISCOTCH) +IORDERINGSC = $(IMETIS) $(IPORD) $(ISCOTCH) + +PLAT = +LIBEXT = .a +OUTC = -o +OUTF = -o +RM = /bin/rm -f +AR = ar vr +RANLIB = ranlib + +INCSEQ = -I$(topdir)/libseq +LIBSEQ = -L$(topdir)/libseq -lmpiseq + +INCPAR = +LIBPAR = $(SCALAP) + +LIBOTHERS = -lpthread + +#Preprocessor defs for calling Fortran from C (-DAdd_ or -DAdd__ or -DUPPER) +CDEFS = -DAdd_ + +#Sequential: +ifeq ($(MUMPS_TYPE),seq) +INCS = $(INCSEQ) +LIBS = $(LIBSEQ) +LIBSEQNEEDED = libseqneeded +endif + +#Parallel: +ifeq ($(MUMPS_TYPE),par) +INCS = $(INCPAR) +LIBS = $(LIBPAR) +LIBSEQNEEDED = +endif diff --git a/var/spack/packages/mumps/package.py b/var/spack/packages/mumps/package.py new file mode 100644 index 00000000000..b4e38c02de4 --- /dev/null +++ b/var/spack/packages/mumps/package.py @@ -0,0 +1,121 @@ +from spack import * +import os + + +class Mumps(Package): + """MUMPS: a MUltifrontal Massively Parallel sparse direct Solver""" + + homepage = "http://mumps.enseeiht.fr" + url = "http://mumps.enseeiht.fr/MUMPS_5.0.1.tar.gz" + + version('5.0.1', 'b477573fdcc87babe861f62316833db0') + + variant('mpi', default=True, description='Activate the compilation of MUMPS with the MPI support') + variant('scotch', default=False, description='Activate Scotch as a possible ordering library') + variant('ptscotch', default=False, description='Activate PT-Scotch as a possible ordering library') + variant('metis', default=False, description='Activate Metis as a possible ordering library') + variant('parmetis', default=False, description='Activate Parmetis as a possible ordering library') + variant('double', default=True, description='Activate dmumps') + variant('float', default=True, description='Activate smumps') + variant('complex', default=True, description='Activate cmumps and/or zmumps') + variant('idx64', default=False, description='Use int64_t/integer*8 as default index type') + variant('double', default=False, description='Use double precision floating point types') + + + depends_on('scotch + esmumps', when='~ptscotch+scotch') + depends_on('scotch + esmumps + mpi', when='+ptscotch') + depends_on('metis', when='~parmetis+metis') + depends_on('parmetis', when="+parmetis") + depends_on('blas') + depends_on('scalapack', when='+mpi') + depends_on('mpi', when='+mpi') + + def patch(self): + if ('+parmetis' in self.spec or '+ptscotch' in self.spec) and '+mpi' not in self.spec: + raise RuntimeError('You cannot use the variants parmetis or ptscotch without mpi') + + makefile_conf = ["LIBBLAS = -L%s -lblas" % self.spec['blas'].prefix.lib] + + orderings = ['-Dpord'] + + if '+ptscotch' in self.spec or '+scotch' in self.spec: + join_lib = ' -l%s' % ('pt' if '+ptscotch' in self.spec else '') + makefile_conf.extend( + ["ISCOTCH = -I%s" % self.spec['scotch'].prefix.include, + "LSCOTCH = -L%s %s%s" % (self.spec['scotch'].prefix.lib, + join_lib, + join_lib.join(['esmumps', 'scotch', 'scotcherr']))]) + orderings.append('-Dscotch') + if '+ptscotch' in self.spec: + orderings.append('-Dptscotch') + + if '+parmetis' in self.spec or '+metis' in self.spec: + libname = 'parmetis' if '+parmetis' in self.spec else 'metis' + makefile_conf.extend( + ["IMETIS = -I%s" % self.spec[libname].prefix.include, + "LMETIS = -L%s -l%s" % (self.spec[libname].prefix.lib, libname)]) + + orderings.append('-Dmetis') + if '+parmetis' in self.spec: + orderings.append('-Dparmetis') + + makefile_conf.append("ORDERINGSF = %s" % (' '.join(orderings))) + + if '+idx64' in self.spec: + makefile_conf.extend( + ['OPTF = -O -DALLOW_NON_INIT %s' % '-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8', + 'OPTL = -O ', + 'OPTC = -O -DINTSIZE64']) + else: + makefile_conf.extend( + ['OPTF = -O -DALLOW_NON_INIT', + 'OPTL = -O ', + 'OPTC = -O ']) + + + if '+mpi' in self.spec: + makefile_conf.extend( + ["CC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), + "FC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + "FL = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + "SCALAP = %s" % self.spec['scalapack'].fc_link]) + else: + makefile_conf.extend( + ["CC = cc", + "FC = fc", + "FL = fc", + "MUMPS_TYPE = seq"]) + + makefile_inc_template = join_path(os.path.dirname(self.module.__file__), + 'Makefile.inc') + with open(makefile_inc_template, "r") as fh: + makefile_conf.extend(fh.read().split('\n')) + + with working_dir('.'): + with open("Makefile.inc", "w") as fh: + makefile_inc = '\n'.join(makefile_conf) + fh.write(makefile_inc) + + + + def install(self, spec, prefix): + make_libs = [] + + # the coice to compile ?examples is to have kind of a sanity + # check on the libraries generated. + if '+float' in spec: + make_libs.append('sexamples') + if '+complex' in spec: + make_libs.append('cexamples') + + if '+double' in spec: + make_libs.append('dexamples') + if '+complex' in spec: + make_libs.append('zexamples') + + make(*make_libs) + + install_tree('lib', prefix.lib) + install_tree('include', prefix.include) + if '~mpi' in spec: + install('libseq/libmpiseq.a', prefix.lib) diff --git a/var/spack/packages/netlib-scalapack/package.py b/var/spack/packages/netlib-scalapack/package.py new file mode 100644 index 00000000000..69df4d9da75 --- /dev/null +++ b/var/spack/packages/netlib-scalapack/package.py @@ -0,0 +1,43 @@ +from spack import * + +class NetlibScalapack(Package): + """ScaLAPACK is a library of high-performance linear algebra routines for parallel distributed memory machines""" + + homepage = "http://www.netlib.org/scalapack/" + url = "http://www.netlib.org/scalapack/scalapack-2.0.2.tgz" + + version('2.0.2', '2f75e600a2ba155ed9ce974a1c4b536f') + version('2.0.1', '17b8cde589ea0423afe1ec43e7499161') + version('2.0.0', '9e76ae7b291be27faaad47cfc256cbfe') + + variant('shared', default=True, description='Build the shared library version') + + provides('scalapack') + + depends_on('mpi') + depends_on('blas') + depends_on('lapack') + + def install(self, spec, prefix): + options = [ + "-DBUILD_SHARED_LIBS:BOOL=%s" % 'ON' if '+shared' in spec else 'OFF', + "-DBUILD_STATIC_LIBS:BOOL=%s" % 'OFF' if '+shared' in spec else 'ON', + "-DCMAKE_C_FLAGS=-fPIC", + "-DCMAKE_Fortran_FLAGS=-fPIC", + ] + + options.extend(std_cmake_args) + + with working_dir('spack-build', create=True): + cmake('..', *options) + make() + make("install") + + def setup_dependent_environment(self, module, spec, dependent_spec): + # TODO treat OS that are not Linux... + lib_suffix = '.so' if '+shared' in spec['scalapack'] else '.a' + + spec['scalapack'].fc_link = '-L%s -lscalapack' % spec['scalapack'].prefix.lib + spec['scalapack'].cc_link = spec['scalapack'].fc_link + spec['scalapack'].libraries = [join_path(spec['scalapack'].prefix.lib, + 'libscalapack%s' % lib_suffix)] From 974fc65e0f15ac87d2422858d71e0c7624fb7726 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 20 Jan 2016 14:38:55 +0100 Subject: [PATCH 344/910] renaming packages to follow changes on develop --- var/spack/{ => repos/builtin}/packages/mumps/Makefile.inc | 0 var/spack/{ => repos/builtin}/packages/mumps/package.py | 0 .../{ => repos/builtin}/packages/netlib-scalapack/package.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename var/spack/{ => repos/builtin}/packages/mumps/Makefile.inc (100%) rename var/spack/{ => repos/builtin}/packages/mumps/package.py (100%) rename var/spack/{ => repos/builtin}/packages/netlib-scalapack/package.py (100%) diff --git a/var/spack/packages/mumps/Makefile.inc b/var/spack/repos/builtin/packages/mumps/Makefile.inc similarity index 100% rename from var/spack/packages/mumps/Makefile.inc rename to var/spack/repos/builtin/packages/mumps/Makefile.inc diff --git a/var/spack/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py similarity index 100% rename from var/spack/packages/mumps/package.py rename to var/spack/repos/builtin/packages/mumps/package.py diff --git a/var/spack/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py similarity index 100% rename from var/spack/packages/netlib-scalapack/package.py rename to var/spack/repos/builtin/packages/netlib-scalapack/package.py From 19caac69d8b3c8383c7b59c2b39ec3bf54406fa8 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 20 Jan 2016 15:22:49 +0100 Subject: [PATCH 345/910] openmpi : turned torque support into a variant (default false) hdf5 : fixed a few bugs, removed suspicious macro deinition, etc. --- .../repos/builtin/packages/hdf5/package.py | 84 +++++++++++++++---- .../repos/builtin/packages/openmpi/package.py | 24 +++--- 2 files changed, 79 insertions(+), 29 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 9a401643411..ac78d8e9617 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -1,5 +1,31 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + from spack import * + class Hdf5(Package): """HDF5 is a data model, library, and file format for storing and managing data. It supports an unlimited variety of datatypes, and is designed for @@ -7,7 +33,7 @@ class Hdf5(Package): """ homepage = "http://www.hdfgroup.org/HDF5/" - url = "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.13/src/hdf5-1.8.13.tar.gz" + url = "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.13/src/hdf5-1.8.13.tar.gz" list_url = "http://www.hdfgroup.org/ftp/HDF5/releases" list_depth = 3 @@ -15,26 +41,53 @@ class Hdf5(Package): version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') version('1.8.13', 'c03426e9e77d7766944654280b467289') + variant('debug', default=False, description='Builds a debug version of the library') + variant('cxx', default=True, description='Enable C++ support') variant('fortran', default=True, description='Enable Fortran support') + variant('unsupported', default=False, description='Enables unsupported configuration options') + variant('mpi', default=False, description='Enable MPI support') - variant('threadsafe', default=False, description='Enable multithreading') + variant('threadsafe', default=False, description='Enable thread-safe capabilities') depends_on("mpi", when='+mpi') depends_on("zlib") - # TODO: currently hard-coded to use OpenMPI + def validate(self, spec): + """ + Checks if incompatible variants have been activated at the same time + + :param spec: spec of the package + :raises RuntimeError: in case of inconsistencies + """ + if '+fortran' in spec and not self.compiler.fc: + msg = 'cannot build a fortran variant without a fortran compiler' + raise RuntimeError(msg) + + if '+threadsafe' in spec and ('+cxx' in spec or '+fortran' in spec): + raise RuntimeError("cannot use variant +threadsafe with either +cxx or +fortran") + def install(self, spec, prefix): + self.validate(spec) + # Handle compilation after spec validation extra_args = [] + if '+debug' in spec: + extra_args.append('--enable-debug=all') + else: + extra_args.append('--enable-production') + + if '+unsupported' in spec: + extra_args.append("--enable-unsupported") + if '+cxx' in spec: - extra_args.extend([ - '--enable-cxx' - ]) + extra_args.append('--enable-cxx') + if '+fortran' in spec: extra_args.extend([ '--enable-fortran', '--enable-fortran2003' ]) + if '+mpi' in spec: # The HDF5 configure script warns if cxx and mpi are enabled # together. There doesn't seem to be a real reason for this, except @@ -43,27 +96,26 @@ def install(self, spec, prefix): # this is not actually a problem. extra_args.extend([ "--enable-parallel", - "--enable-unsupported", "CC=%s" % spec['mpi'].prefix.bin + "/mpicc", - "CXX=%s" % spec['mpi'].prefix.bin + "/mpic++", - "FC=%s" % spec['mpi'].prefix.bin + "/mpifort", ]) - if '+threads' in spec: - if '+cxx' in spec or '+fortran' in spec: - die("Cannot use variant +threads with either +cxx or +fortran") + + if '+cxx' in spec: + extra_args.append("CXX=%s" % spec['mpi'].prefix.bin + "/mpic++") + + if '+fortran' in spec: + extra_args.append("FC=%s" % spec['mpi'].prefix.bin + "/mpifort") + + if '+threadsafe' in spec: extra_args.extend([ '--enable-threadsafe', '--disable-hl', - 'CPPFLAGS=-DHDatexit=""', - 'CFLAGS=-DHDatexit=""' ]) configure( "--prefix=%s" % prefix, "--with-zlib=%s" % spec['zlib'].prefix, - "--enable-shared", + "--enable-shared", # TODO : this should be enabled by default, remove it? *extra_args) - make() make("install") diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 463719f9dbe..1b7f1c2011d 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -19,43 +19,44 @@ class Openmpi(Package): version('1.10.1', 'f0fcd77ed345b7eafb431968124ba16e') version('1.10.0', '280cf952de68369cebaca886c5ce0304') - version('1.8.8', '0dab8e602372da1425e9242ae37faf8c') - version('1.6.5', '03aed2a4aa4d0b27196962a2a65fc475') + version('1.8.8', '0dab8e602372da1425e9242ae37faf8c') + version('1.6.5', '03aed2a4aa4d0b27196962a2a65fc475') patch('ad_lustre_rwcontig_open_source.patch', when="@1.6.5") patch('llnl-platforms.patch', when="@1.6.5") patch('configure.patch', when="@1.10.0:") - variant('psm', default=False, description='Build support for the PSM library.') + variant('psm', default=False, description='Build support for the PSM library.') variant('verbs', default=False, description='Build support for OpenFabrics verbs.') + # TODO : variant support for other schedulers is missing + variant('tm', default=False, description='Build TM (Torque, PBSPro, and compatible) support') + provides('mpi@:2.2', when='@1.6.5') provides('mpi@:3.0', when='@1.7.5:') - depends_on('hwloc') - def url_for_version(self, version): return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) - def setup_dependent_environment(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" - os.environ['OMPI_CC'] = 'cc' + os.environ['OMPI_CC'] = 'cc' os.environ['OMPI_CXX'] = 'c++' os.environ['OMPI_FC'] = 'f90' os.environ['OMPI_F77'] = 'f77' - def install(self, spec, prefix): config_args = ["--prefix=%s" % prefix, "--with-hwloc=%s" % spec['hwloc'].prefix, - "--with-tm", # necessary for Torque support "--enable-shared", "--enable-static"] # Variants + if '+tm' in spec: + config_args.append("--with-tm") # necessary for Torque support + if '+psm' in spec: config_args.append("--with-psm") @@ -85,7 +86,6 @@ def install(self, spec, prefix): self.filter_compilers() - def filter_compilers(self): """Run after install to make the MPI compilers use the compilers that Spack built the package with. @@ -94,7 +94,7 @@ def filter_compilers(self): to Spack's generic cc, c++ and f90. We want them to be bound to whatever compiler they were built with. """ - kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : False } + kwargs = {'ignore_absent': True, 'backup': False, 'string': False} dir = os.path.join(self.prefix, 'share/openmpi/') cc_wrappers = ['mpicc-vt-wrapper-data.txt', 'mpicc-wrapper-data.txt', @@ -132,5 +132,3 @@ def filter_compilers(self): if not os.path.islink(path): filter_file('compiler=.*', 'compiler=%s' % self.compiler.fc, path, **kwargs) - - From b71b478a3626ae9e1a0d3b5c7ab9945b5c478200 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 20 Jan 2016 18:31:02 +0100 Subject: [PATCH 346/910] Minor correction to install mumps+mpi --- .../repos/builtin/packages/mumps/Makefile.inc | 3 -- .../repos/builtin/packages/mumps/package.py | 32 +++++++++++++++---- .../packages/netlib-scalapack/package.py | 17 +++++++--- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/var/spack/repos/builtin/packages/mumps/Makefile.inc b/var/spack/repos/builtin/packages/mumps/Makefile.inc index 4b6696db5c3..2e6a041878c 100644 --- a/var/spack/repos/builtin/packages/mumps/Makefile.inc +++ b/var/spack/repos/builtin/packages/mumps/Makefile.inc @@ -23,9 +23,6 @@ LIBPAR = $(SCALAP) LIBOTHERS = -lpthread -#Preprocessor defs for calling Fortran from C (-DAdd_ or -DAdd__ or -DUPPER) -CDEFS = -DAdd_ - #Sequential: ifeq ($(MUMPS_TYPE),seq) INCS = $(INCSEQ) diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index b4e38c02de4..44a37903cc8 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -15,11 +15,10 @@ class Mumps(Package): variant('ptscotch', default=False, description='Activate PT-Scotch as a possible ordering library') variant('metis', default=False, description='Activate Metis as a possible ordering library') variant('parmetis', default=False, description='Activate Parmetis as a possible ordering library') - variant('double', default=True, description='Activate dmumps') - variant('float', default=True, description='Activate smumps') - variant('complex', default=True, description='Activate cmumps and/or zmumps') + variant('double', default=True, description='Activate the compilation of dmumps') + variant('float', default=True, description='Activate the compilation of smumps') + variant('complex', default=True, description='Activate the compilation of cmumps and/or zmumps') variant('idx64', default=False, description='Use int64_t/integer*8 as default index type') - variant('double', default=False, description='Use double precision floating point types') depends_on('scotch + esmumps', when='~ptscotch+scotch') @@ -27,10 +26,16 @@ class Mumps(Package): depends_on('metis', when='~parmetis+metis') depends_on('parmetis', when="+parmetis") depends_on('blas') + depends_on('lapack') depends_on('scalapack', when='+mpi') depends_on('mpi', when='+mpi') - - def patch(self): + + # this function is not a patch function because in case scalapack + # is needed it uses self.spec['scalapack'].fc_link set by the + # setup_dependent_environment in scalapck. This happen after patch + # end before install + # def patch(self): + def write_makefile_inc(self): if ('+parmetis' in self.spec or '+ptscotch' in self.spec) and '+mpi' not in self.spec: raise RuntimeError('You cannot use the variants parmetis or ptscotch without mpi') @@ -61,8 +66,13 @@ def patch(self): makefile_conf.append("ORDERINGSF = %s" % (' '.join(orderings))) + # TODO: test this part, it needs a full blas, scalapack and + # partitionning environment with 64bit integers if '+idx64' in self.spec: makefile_conf.extend( + # the fortran compilation flags most probably are + # working only for intel and gnu compilers this is + # perhaps something the compiler should provide ['OPTF = -O -DALLOW_NON_INIT %s' % '-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8', 'OPTL = -O ', 'OPTC = -O -DINTSIZE64']) @@ -78,7 +88,8 @@ def patch(self): ["CC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), "FC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), "FL = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), - "SCALAP = %s" % self.spec['scalapack'].fc_link]) + "SCALAP = %s" % self.spec['scalapack'].fc_link, + "MUMPS_TYPE = par"]) else: makefile_conf.extend( ["CC = cc", @@ -86,6 +97,11 @@ def patch(self): "FL = fc", "MUMPS_TYPE = seq"]) + # TODO: change the value to the correct one according to the + # compiler possible values are -DAdd_, -DAdd__ and/or -DUPPER + makefile_conf.append("CDEFS = -DAdd_") + + makefile_inc_template = join_path(os.path.dirname(self.module.__file__), 'Makefile.inc') with open(makefile_inc_template, "r") as fh: @@ -113,6 +129,8 @@ def install(self, spec, prefix): if '+complex' in spec: make_libs.append('zexamples') + self.write_makefile_inc() + make(*make_libs) install_tree('lib', prefix.lib) diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 69df4d9da75..5be91c4a40c 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -9,23 +9,30 @@ class NetlibScalapack(Package): version('2.0.2', '2f75e600a2ba155ed9ce974a1c4b536f') version('2.0.1', '17b8cde589ea0423afe1ec43e7499161') version('2.0.0', '9e76ae7b291be27faaad47cfc256cbfe') - + # versions before 2.0.0 are not using cmake and requires blacs as + # a separated package + variant('shared', default=True, description='Build the shared library version') - + variant('fpic', default=False, description="Build with -fpic compiler option") + provides('scalapack') depends_on('mpi') - depends_on('blas') depends_on('lapack') def install(self, spec, prefix): options = [ "-DBUILD_SHARED_LIBS:BOOL=%s" % 'ON' if '+shared' in spec else 'OFF', "-DBUILD_STATIC_LIBS:BOOL=%s" % 'OFF' if '+shared' in spec else 'ON', - "-DCMAKE_C_FLAGS=-fPIC", - "-DCMAKE_Fortran_FLAGS=-fPIC", + "-DUSE_OPTIMIZED_LAPACK_BLAS:BOOL=ON", # forces scalapack to use find_package(LAPACK) ] + if '+fpic' in spec: + options.extend([ + "-DCMAKE_C_FLAGS=-fPIC", + "-DCMAKE_Fortran_FLAGS=-fPIC" + ]) + options.extend(std_cmake_args) with working_dir('spack-build', create=True): From e25150296a4ef34a80a664565e3d26f00f7d999f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 20 Jan 2016 14:44:43 -0600 Subject: [PATCH 347/910] Redirect STDERR to STDOUT for compiler version This is necessary for the NAG Fortran compiler, which prints its version message to STDERR instead of STDOUT. This message was previously being ignored, preventing spack from finding the compiler's version or automatically adding it to the compilers.yaml configuration file. --- lib/spack/spack/compiler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index a665f6062d2..887e416dc58 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -24,6 +24,7 @@ ############################################################################## import os import re +import subprocess import itertools from datetime import datetime @@ -51,7 +52,7 @@ def _verify_executables(*paths): def get_compiler_version(compiler_path, version_arg, regex='(.*)'): if not compiler_path in _version_cache: compiler = Executable(compiler_path) - output = compiler(version_arg, return_output=True, error=os.devnull) + output = compiler(version_arg, return_output=True, error=subprocess.STDOUT) match = re.search(regex, output) _version_cache[compiler_path] = match.group(1) if match else 'unknown' From 088b6f0ea9953a3f007d207d06ca3150f83b3be4 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 20 Jan 2016 17:19:01 -0600 Subject: [PATCH 348/910] Fixed typo when running spack help --- lib/spack/spack/cmd/location.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 39c225e9b26..307ee8982d5 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -32,7 +32,7 @@ import spack import spack.cmd -description="Print out locations of various diectories used by Spack" +description="Print out locations of various directories used by Spack" def setup_parser(subparser): global directories From 785c01218ffb33b085c786f19b81ebbefcb3d0c6 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 21 Jan 2016 10:06:43 +0100 Subject: [PATCH 349/910] elpa : added missing dependencies --- var/spack/repos/builtin/packages/elpa/package.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/elpa/package.py b/var/spack/repos/builtin/packages/elpa/package.py index 6825056a04b..2ade5b0b37b 100644 --- a/var/spack/repos/builtin/packages/elpa/package.py +++ b/var/spack/repos/builtin/packages/elpa/package.py @@ -36,11 +36,20 @@ class Elpa(Package): version('2015.11.001', 'de0f35b7ee7c971fd0dca35c900b87e6', url='http://elpa.mpcdf.mpg.de/elpa-2015.11.001.tar.gz') + variant('openmp', default=False, description='Activates OpenMP support') + depends_on('mpi') depends_on('blas') depends_on('lapack') + depends_on('scalapack') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + + options = ["--prefix=%s" % prefix] + + if '+openmp' in spec: + options.append("--enable-openmp") + + configure(*options) make() make("install") From 950246b331c74700e01dc48a86f84bf47d528af3 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 21 Jan 2016 09:54:24 -0600 Subject: [PATCH 350/910] Remove constraint on dependency version --- var/spack/repos/builtin/packages/hdf/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/hdf/package.py b/var/spack/repos/builtin/packages/hdf/package.py index 7882665dbaf..43b59802301 100644 --- a/var/spack/repos/builtin/packages/hdf/package.py +++ b/var/spack/repos/builtin/packages/hdf/package.py @@ -12,7 +12,7 @@ class Hdf(Package): version('4.2.11', '063f9928f3a19cc21367b71c3b8bbf19') depends_on("jpeg") - depends_on("szip@2.1") + depends_on("szip") depends_on("zlib") From ea9d0c0263126c370119e5fc435652c7214e658a Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 21 Jan 2016 09:58:47 -0600 Subject: [PATCH 351/910] Remove constraints on dependency versions --- var/spack/repos/builtin/packages/mpc/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/mpc/package.py b/var/spack/repos/builtin/packages/mpc/package.py index 65c0ba7ea13..108fec678f1 100644 --- a/var/spack/repos/builtin/packages/mpc/package.py +++ b/var/spack/repos/builtin/packages/mpc/package.py @@ -34,8 +34,8 @@ class Mpc(Package): version('1.0.3', 'd6a1d5f8ddea3abd2cc3e98f58352d26') version('1.0.2', '68fadff3358fb3e7976c7a398a0af4c3') - depends_on("gmp@4.3.2:") - depends_on("mpfr@2.4.2:") + depends_on("gmp") + depends_on("mpfr") def url_for_version(self, version): if version < Version("1.0.1"): From 03d5ee6738e49c4402cffe7503ea92c25edf9720 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 21 Jan 2016 10:02:33 -0600 Subject: [PATCH 352/910] Remove constraint on dependency version --- var/spack/repos/builtin/packages/mpfr/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/mpfr/package.py b/var/spack/repos/builtin/packages/mpfr/package.py index 2758b0da2ed..a1bd7529cfe 100644 --- a/var/spack/repos/builtin/packages/mpfr/package.py +++ b/var/spack/repos/builtin/packages/mpfr/package.py @@ -33,7 +33,7 @@ class Mpfr(Package): version('3.1.3', '5fdfa3cfa5c86514ee4a241a1affa138') version('3.1.2', 'ee2c3ac63bf0c2359bf08fc3ee094c19') - depends_on('gmp@4.1.0:') + depends_on('gmp') def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 10848c2e9a7cd3771c469dfd5ddb235f76b229ba Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 21 Jan 2016 02:40:38 -0800 Subject: [PATCH 353/910] Refactor args for Executable.__call__ - simplify output, error, and input redirection - `return_output=True` is now `output=str` - `return_output=True` will still work for the time being but is deprecated. - previously you could say `return_output=True` and `output=`, which wouldn't actually write to the stream. Now you actually can't specify erroneous cases since there is only one parameter with mutually exclusive options.. --- lib/spack/spack/cmd/bootstrap.py | 2 +- lib/spack/spack/cmd/create.py | 2 +- lib/spack/spack/cmd/pkg.py | 2 +- lib/spack/spack/compiler.py | 3 +- lib/spack/spack/fetch_strategy.py | 6 +- lib/spack/spack/test/cc.py | 6 +- lib/spack/spack/test/make_executable.py | 48 +++++----- lib/spack/spack/test/mock_repo.py | 4 +- lib/spack/spack/test/svn_fetch.py | 2 +- lib/spack/spack/util/executable.py | 95 +++++++++++++++---- .../repos/builtin/packages/gcc/package.py | 6 +- 11 files changed, 116 insertions(+), 60 deletions(-) diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index e4ec7da35db..bdbd623b39a 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -42,7 +42,7 @@ def get_origin_url(): git = which('git', required=True) origin_url = git( '--git-dir=%s' % git_dir, 'config', '--get', 'remote.origin.url', - return_output=True) + output=str) return origin_url.strip() diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 7cea39cb55a..edcea0718cd 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -132,7 +132,7 @@ def __call__(self, stage): # Peek inside the tarball. tar = which('tar') output = tar( - "--exclude=*/*/*", "-tf", stage.archive_file, return_output=True) + "--exclude=*/*/*", "-tf", stage.archive_file, output=str) lines = output.split("\n") # Set the configure line to the one that matched. diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index 448f7628415..cf478d3763e 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -79,7 +79,7 @@ def list_packages(rev): git = get_git() relpath = spack.packages_path[len(spack.prefix + os.path.sep):] + os.path.sep output = git('ls-tree', '--full-tree', '--name-only', rev, relpath, - return_output=True) + output=str) return sorted(line[len(relpath):] for line in output.split('\n') if line) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 887e416dc58..12c02e0ea2b 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -24,7 +24,6 @@ ############################################################################## import os import re -import subprocess import itertools from datetime import datetime @@ -52,7 +51,7 @@ def _verify_executables(*paths): def get_compiler_version(compiler_path, version_arg, regex='(.*)'): if not compiler_path in _version_cache: compiler = Executable(compiler_path) - output = compiler(version_arg, return_output=True, error=subprocess.STDOUT) + output = compiler(version_arg, output=str, error=str) match = re.search(regex, output) _version_cache[compiler_path] = match.group(1) if match else 'unknown' diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 0657146bf63..337dd1e1988 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -154,7 +154,7 @@ def fetch(self): # Run curl but grab the mime type from the http headers headers = spack.curl( - *curl_args, return_output=True, fail_on_error=False) + *curl_args, output=str, fail_on_error=False) if spack.curl.returncode != 0: # clean up archive on failure. @@ -375,7 +375,7 @@ def __init__(self, **kwargs): @property def git_version(self): - vstring = self.git('--version', return_output=True).lstrip('git version ') + vstring = self.git('--version', output=str).lstrip('git version ') return Version(vstring) @@ -518,7 +518,7 @@ def fetch(self): def _remove_untracked_files(self): """Removes untracked files in an svn repository.""" - status = self.svn('status', '--no-ignore', return_output=True) + status = self.svn('status', '--no-ignore', output=str) self.svn('status', '--no-ignore') for line in status.split('\n'): if not re.match('^[I?]', line): diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 4188b8d5505..905af28a06d 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -65,17 +65,17 @@ def setUp(self): def check_cc(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.cc(*args, return_output=True).strip(), expected) + self.assertEqual(self.cc(*args, output=str).strip(), expected) def check_ld(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.ld(*args, return_output=True).strip(), expected) + self.assertEqual(self.ld(*args, output=str).strip(), expected) def check_cpp(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.cpp(*args, return_output=True).strip(), expected) + self.assertEqual(self.cpp(*args, output=str).strip(), expected) def test_vcheck_mode(self): diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index 09efec85805..d568a28d443 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -56,47 +56,47 @@ def tearDown(self): def test_make_normal(self): make = MakeExecutable('make', 8) - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') def test_make_explicit(self): make = MakeExecutable('make', 8) - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') def test_make_one_job(self): make = MakeExecutable('make', 1) - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') def test_make_parallel_false(self): make = MakeExecutable('make', 8) - self.assertEqual(make(parallel=False, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=False, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=False, output=str).strip(), '') + self.assertEqual(make('install', parallel=False, output=str).strip(), 'install') def test_make_parallel_disabled(self): make = MakeExecutable('make', 8) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true' - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') os.environ['SPACK_NO_PARALLEL_MAKE'] = '1' - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') # These don't disable (false and random string) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false' - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar' - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') del os.environ['SPACK_NO_PARALLEL_MAKE'] @@ -106,20 +106,20 @@ def test_make_parallel_precedence(self): # These should work os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true' - self.assertEqual(make(parallel=True, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=True, output=str).strip(), '') + self.assertEqual(make('install', parallel=True, output=str).strip(), 'install') os.environ['SPACK_NO_PARALLEL_MAKE'] = '1' - self.assertEqual(make(parallel=True, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=True, output=str).strip(), '') + self.assertEqual(make('install', parallel=True, output=str).strip(), 'install') # These don't disable (false and random string) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false' - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar' - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') del os.environ['SPACK_NO_PARALLEL_MAKE'] diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index c454b1f106c..9738ba4e728 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -141,7 +141,7 @@ def __init__(self): self.url = self.path def rev_hash(self, rev): - return git('rev-parse', rev, return_output=True).strip() + return git('rev-parse', rev, output=str).strip() class MockSvnRepo(MockVCSRepo): @@ -193,4 +193,4 @@ def __init__(self): def get_rev(self): """Get current mercurial revision.""" - return hg('id', '-i', return_output=True).strip() + return hg('id', '-i', output=str).strip() diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 2ee4748fdbd..7d150b42f41 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -65,7 +65,7 @@ def tearDown(self): def assert_rev(self, rev): """Check that the current revision is equal to the supplied rev.""" def get_rev(): - output = svn('info', return_output=True) + output = svn('info', output=str) self.assertTrue("Revision" in output) for line in output.split('\n'): match = re.match(r'Revision: (\d+)', line) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index ba765eb6629..fc27b789d06 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -55,24 +55,80 @@ def command(self): def __call__(self, *args, **kwargs): - """Run the executable with subprocess.check_output, return output.""" - return_output = kwargs.get("return_output", False) - fail_on_error = kwargs.get("fail_on_error", True) - ignore_errors = kwargs.get("ignore_errors", ()) + """Run this executable in a subprocess. + + Arguments + args + command line arguments to the executable to run. + + Optional arguments + + fail_on_error + + Raise an exception if the subprocess returns an + error. Default is True. When not set, the return code is + avaiale as `exe.returncode`. + + ignore_errors + + An optional list/tuple of error codes that can be + *ignored*. i.e., if these codes are returned, this will + not raise an exception when `fail_on_error` is `True`. + + output, error + + These arguments allow you to specify new stdout and stderr + values. They default to `None`, which means the + subprocess will inherit the parent's file descriptors. + + You can set these to: + - python streams, e.g. open Python file objects, or os.devnull; + - filenames, which will be automatically opened for writing; or + - `str`, as in the Python string type. If you set these to `str`, + output and error will be written to pipes and returned as + a string. If both `output` and `error` are set to `str`, + then one string is returned containing output concatenated + with error. + + input + + Same as output, error, but `str` is not an allowed value. + + Deprecated arguments + + return_output[=False] + + Setting this to True is the same as setting output=str. + This argument may be removed in future Spack versions. + + """ + fail_on_error = kwargs.pop("fail_on_error", True) + ignore_errors = kwargs.pop("ignore_errors", ()) + + # TODO: This is deprecated. Remove in a future version. + return_output = kwargs.pop("return_output", False) # Default values of None says to keep parent's file descriptors. - output = kwargs.get("output", None) - error = kwargs.get("error", None) - input = kwargs.get("input", None) + if return_output: + output = str + else: + output = kwargs.pop("output", None) + + error = kwargs.pop("error", None) + input = kwargs.pop("input", None) + if input is str: + raise ValueError("Cannot use `str` as input stream.") def streamify(arg, mode): if isinstance(arg, basestring): return open(arg, mode), True + elif arg is str: + return subprocess.PIPE, False else: return arg, False - output, ostream = streamify(output, 'w') - error, estream = streamify(error, 'w') - input, istream = streamify(input, 'r') + ostream, close_ostream = streamify(output, 'w') + estream, close_estream = streamify(error, 'w') + istream, close_istream = streamify(input, 'r') # if they just want to ignore one error code, make it a tuple. if isinstance(ignore_errors, int): @@ -92,19 +148,20 @@ def streamify(arg, mode): tty.debug(cmd_line) try: - if return_output: - output = subprocess.PIPE - proc = subprocess.Popen( - cmd, stdin=input, stderr=error, stdout=output) + cmd, stdin=istream, stderr=estream, stdout=ostream) out, err = proc.communicate() rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" % proc.returncode, cmd_line) - if return_output: - return out + + if output is str or error is str: + result = '' + if output is str: result += out + if error is str: result += err + return result except OSError, e: raise ProcessError( @@ -119,9 +176,9 @@ def streamify(arg, mode): % (proc.returncode, cmd_line)) finally: - if ostream: output.close() - if estream: error.close() - if istream: input.close() + if close_ostream: output.close() + if close_estream: error.close() + if close_istream: input.close() def __eq__(self, other): diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 61b16f3fd8d..3e5895cfb83 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -50,7 +50,7 @@ class Gcc(Package): version('4.5.4', '27e459c2566b8209ab064570e1b378f7') variant('gold', default=True, description="Build the gold linker plugin for ld-based LTO") - + depends_on("mpfr") depends_on("gmp") depends_on("mpc") # when @4.5: @@ -102,7 +102,7 @@ def install(self, spec, prefix): configure(*options) make() make("install") - + self.write_rpath_specs() @@ -121,7 +121,7 @@ def write_rpath_specs(self): return gcc = Executable(join_path(self.prefix.bin, 'gcc')) - lines = gcc('-dumpspecs', return_output=True).strip().split("\n") + lines = gcc('-dumpspecs', output=str).strip().split("\n") specs_file = join_path(self.spec_dir, 'specs') with closing(open(specs_file, 'w')) as out: for line in lines: From 6b3f023d98c7d34ab859e53b2e0953f529325f6e Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 21 Jan 2016 14:05:16 -0500 Subject: [PATCH 354/910] Update OpenMPI to 1.10.2 --- var/spack/repos/builtin/packages/openmpi/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 1b7f1c2011d..9d57f49427b 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -17,6 +17,7 @@ class Openmpi(Package): list_url = "http://www.open-mpi.org/software/ompi/" list_depth = 3 + version('1.10.2', 'b2f43d9635d2d52826e5ef9feb97fd4c') version('1.10.1', 'f0fcd77ed345b7eafb431968124ba16e') version('1.10.0', '280cf952de68369cebaca886c5ce0304') version('1.8.8', '0dab8e602372da1425e9242ae37faf8c') From 2f70c842b8bc7f897b5a18df10c72ae6a7277fbd Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 21 Jan 2016 14:13:39 -0500 Subject: [PATCH 355/910] Disable patch that was merged into upstream in 1.10.2 --- var/spack/repos/builtin/packages/openmpi/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 9d57f49427b..e4484af8c5e 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -25,7 +25,7 @@ class Openmpi(Package): patch('ad_lustre_rwcontig_open_source.patch', when="@1.6.5") patch('llnl-platforms.patch', when="@1.6.5") - patch('configure.patch', when="@1.10.0:") + patch('configure.patch', when="@1.10.0:1.10.1") variant('psm', default=False, description='Build support for the PSM library.') variant('verbs', default=False, description='Build support for OpenFabrics verbs.') From ca3edc36738f561c3a27369835da80152b5f9f78 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 21 Jan 2016 13:20:39 -0600 Subject: [PATCH 356/910] Add symlink cxx to point to cc --- lib/spack/env/cxx | 1 + 1 file changed, 1 insertion(+) create mode 120000 lib/spack/env/cxx diff --git a/lib/spack/env/cxx b/lib/spack/env/cxx new file mode 120000 index 00000000000..2652f5f42c0 --- /dev/null +++ b/lib/spack/env/cxx @@ -0,0 +1 @@ +cc \ No newline at end of file From 571386cdf4c6249958008d3a3f0bd24192fd111a Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 21 Jan 2016 13:55:59 -0600 Subject: [PATCH 357/910] Remove cxx symlink, tell NAG to look for c++ symlink --- lib/spack/env/cxx | 1 - lib/spack/spack/compilers/nag.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 120000 lib/spack/env/cxx diff --git a/lib/spack/env/cxx b/lib/spack/env/cxx deleted file mode 120000 index 2652f5f42c0..00000000000 --- a/lib/spack/env/cxx +++ /dev/null @@ -1 +0,0 @@ -cc \ No newline at end of file diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index f1cc6be0d5c..527a05a090c 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -16,7 +16,7 @@ class Nag(Compiler): # Named wrapper links within spack.build_env_path link_paths = { # Use default wrappers for C and C++, in case provided in compilers.yaml 'cc' : 'cc', - 'cxx' : 'cxx', + 'cxx' : 'c++', 'f77' : 'nag/nagfor', 'fc' : 'nag/nagfor' } From 2e58bc31138ee172ea55ad652aaf49b09fe6e135 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 21 Jan 2016 19:57:49 -0800 Subject: [PATCH 358/910] Sticking with additive approach but now most libraries are installed by default. --- .../repos/builtin/packages/boost/package.py | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 1992d4d39a5..f31bc445b8d 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -45,28 +45,36 @@ class Boost(Package): version('1.34.1', '2d938467e8a448a2c9763e0a9f8ca7e5') version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') - libs = ['chrono', + default_install_libs = set(['chrono', 'date_time', 'filesystem', - 'iostreams', + 'graph', + 'iostreams', + 'log', + 'math', 'random', 'regex', 'serialization', 'signals', 'system', 'thread', - 'wave', - 'mpi', - 'python'] + 'wave']) - for lib in libs: - variant(lib, default=False, description="Compile with {0} library" - .format(lib)) + # These are not installed by default because they pull in many dependencies + # and/or because there is a great deal of customization possible (and it + # would be difficult or tedious to choose sensible defaults here). + default_noinstall_libs = set(['mpi', 'python']) + + all_libs = default_install_libs | default_noinstall_libs + + for lib in all_libs: + variant(lib, default=(lib in default_install_libs), + description="Compile with {0} library".format(lib)) variant('debug', default=False, description='Switch to the debug version of Boost') variant('shared', default=True, description="Additionally build shared libraries") variant('multithreaded', default=True, description="Build multi-threaded versions of libraries") - variant('singlethreaded', default=False, description="Build single-threaded versions of libraries") + variant('singlethreaded', default=True, description="Build single-threaded versions of libraries") variant('regex_icu', default=False, description="Include regex ICU support (by default false even if regex library is compiled)") depends_on('icu', when='+regex_icu') From 4a55b97d11739a0ee6ca5ddcbdf35826ca292469 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 Jan 2016 12:37:12 -0800 Subject: [PATCH 359/910] Fixed reference --- var/spack/repos/builtin/packages/boost/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index f31bc445b8d..467e9a61c65 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -163,7 +163,7 @@ def determine_b2_options(self, spec, options): def install(self, spec, prefix): withLibs = list() - for lib in Boost.libs: + for lib in Boost.all_libs: if "+{0}".format(lib) in spec: withLibs.append(lib) if not withLibs: From 10de5a3ec76fbf1244c43a843ae26a67fc879ddb Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Fri, 22 Jan 2016 13:02:18 -0800 Subject: [PATCH 360/910] Patch fix for boost@1.60.0%gcc@4.4.7 --- .../builtin/packages/boost/boost_11856.patch | 34 +++++++++++++++++++ .../repos/builtin/packages/boost/package.py | 3 ++ 2 files changed, 37 insertions(+) create mode 100644 var/spack/repos/builtin/packages/boost/boost_11856.patch diff --git a/var/spack/repos/builtin/packages/boost/boost_11856.patch b/var/spack/repos/builtin/packages/boost/boost_11856.patch new file mode 100644 index 00000000000..3b4052ca187 --- /dev/null +++ b/var/spack/repos/builtin/packages/boost/boost_11856.patch @@ -0,0 +1,34 @@ +--- a/libs/container/src/pool_resource.cpp 2015-11-06 12:49:55.000000000 -0800 ++++ b/libs/container/src/pool_resource.cpp 2015-12-22 07:54:36.202131121 -0800 +@@ -32,11 +32,11 @@ + class pool_data_t + : public block_slist_base<> + { +- typedef block_slist_base<> block_slist_base; ++ typedef block_slist_base<> block_slist_base_t; + + public: + explicit pool_data_t(std::size_t initial_blocks_per_chunk) +- : block_slist_base(), next_blocks_per_chunk(initial_blocks_per_chunk) ++ : block_slist_base_t(), next_blocks_per_chunk(initial_blocks_per_chunk) + { slist_algo::init_header(&free_slist); } + + void *allocate_block() BOOST_NOEXCEPT +@@ -59,7 +59,7 @@ + void release(memory_resource &upstream) + { + slist_algo::init_header(&free_slist); +- this->block_slist_base::release(upstream); ++ this->block_slist_base_t::release(upstream); + next_blocks_per_chunk = pool_options_minimum_max_blocks_per_chunk; + } + +@@ -72,7 +72,7 @@ + + //Minimum block size is at least max_align, so all pools allocate sizes that are multiple of max_align, + //meaning that all blocks are max_align-aligned. +- char *p = static_cast(block_slist_base::allocate(blocks_per_chunk*pool_block, mr)); ++ char *p = static_cast(block_slist_base_t::allocate(blocks_per_chunk*pool_block, mr)); + + //Create header types. This is no-throw + for(std::size_t i = 0, max = blocks_per_chunk; i != max; ++i){ diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 3427b74ad68..e3fb516be9e 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -54,6 +54,9 @@ class Boost(Package): depends_on('bzip2', when='+compression') depends_on('zlib', when='+compression') + # Patch fix from https://svn.boost.org/trac/boost/ticket/11856 + patch('boost_11856.patch', when='@1.60.0%gcc@4.4.7') + def url_for_version(self, version): """Handle Boost's weird URLs, which write the version two different ways.""" parts = [str(p) for p in Version(version)] From 9f99ee61c733e8fee8ae4058fb9198288af40fc6 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 Jan 2016 13:25:45 -0800 Subject: [PATCH 361/910] 1. added default install libs (atomic, test, locale, program_options) 2. clarify comment for default_noinstall_libs 3. renamed regex_icu variant to icu_support (both the locale and regex libs can use it) 4. explicitly set b2 install ICU_PATH when regex_icu is activated --- .../repos/builtin/packages/boost/package.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 467e9a61c65..1403ea44118 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -45,24 +45,28 @@ class Boost(Package): version('1.34.1', '2d938467e8a448a2c9763e0a9f8ca7e5') version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') - default_install_libs = set(['chrono', + default_install_libs = set(['atomic', + 'chrono', 'date_time', 'filesystem', 'graph', 'iostreams', + 'locale', 'log', 'math', + 'program_options', 'random', 'regex', 'serialization', 'signals', 'system', + 'test', 'thread', 'wave']) - # These are not installed by default because they pull in many dependencies - # and/or because there is a great deal of customization possible (and it - # would be difficult or tedious to choose sensible defaults here). + # mpi/python are not installed by default because they pull in many + # dependencies and/or because there is a great deal of customization + # possible (and it would be difficult to choose sensible defaults) default_noinstall_libs = set(['mpi', 'python']) all_libs = default_install_libs | default_noinstall_libs @@ -75,9 +79,9 @@ class Boost(Package): variant('shared', default=True, description="Additionally build shared libraries") variant('multithreaded', default=True, description="Build multi-threaded versions of libraries") variant('singlethreaded', default=True, description="Build single-threaded versions of libraries") - variant('regex_icu', default=False, description="Include regex ICU support (by default false even if regex library is compiled)") + variant('icu_support', default=False, description="Include ICU support (for regex/locale libraries)") - depends_on('icu', when='+regex_icu') + depends_on('icu', when='+icu_support') depends_on('python', when='+python') depends_on('mpi', when='+mpi') depends_on('bzip2', when='+iostreams') @@ -134,6 +138,9 @@ def determine_b2_options(self, spec, options): else: options.append('variant=release') + if '+icu_support' in spec: + options.extend(['-s', 'ICU_PATH=%s' % spec['icu'].prefix]) + if '+iostreams' in spec: options.extend([ '-s', 'BZIP2_INCLUDE=%s' % spec['bzip2'].prefix.include, From a653d2f5e2f296ff66eb41ffb9f2f6e9b9bdb3b5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 Jan 2016 13:43:16 -0800 Subject: [PATCH 362/910] Slightly more robust approach for setting defaults for noinstall_libs --- var/spack/repos/builtin/packages/boost/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 1403ea44118..a30cd7cc35b 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -72,7 +72,7 @@ class Boost(Package): all_libs = default_install_libs | default_noinstall_libs for lib in all_libs: - variant(lib, default=(lib in default_install_libs), + variant(lib, default=(lib not in default_noinstall_libs), description="Compile with {0} library".format(lib)) variant('debug', default=False, description='Switch to the debug version of Boost') From 64a954922504d08d2e109ae93be06c5095243b67 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 24 Jan 2016 15:31:07 -0800 Subject: [PATCH 363/910] Minor cleanup. 80 char width for command description. --- lib/spack/spack/cmd/test-install.py | 40 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 74017f59fb3..656873a2f03 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -37,20 +37,20 @@ from spack.fetch_strategy import FetchError import spack.cmd -description = "Treat package installations as unit tests and output formatted test results" +description = "Run package installation as a unit test, output formatted results." def setup_parser(subparser): subparser.add_argument( '-j', '--jobs', action='store', type=int, help="Explicitly set number of make jobs. Default is #cpus.") - + subparser.add_argument( '-n', '--no-checksum', action='store_true', dest='no_checksum', help="Do not check packages against checksum") - + subparser.add_argument( '-o', '--output', action='store', help="test output goes in this file") - + subparser.add_argument( 'package', nargs=argparse.REMAINDER, help="spec of package to install") @@ -59,10 +59,10 @@ class JunitResultFormat(object): def __init__(self): self.root = ET.Element('testsuite') self.tests = [] - + def add_test(self, buildId, testResult, buildInfo=None): self.tests.append((buildId, testResult, buildInfo)) - + def write_to(self, stream): self.root.set('tests', '{0}'.format(len(self.tests))) for buildId, testResult, buildInfo in self.tests: @@ -84,25 +84,25 @@ class TestResult(object): PASSED = 0 FAILED = 1 SKIPPED = 2 - + class BuildId(object): def __init__(self, spec): self.name = spec.name self.version = spec.version self.hashId = spec.dag_hash() - + def stringId(self): return "-".join(str(x) for x in (self.name, self.version, self.hashId)) def __hash__(self): return hash((self.name, self.version, self.hashId)) - + def __eq__(self, other): if not isinstance(other, BuildId): return False - - return ((self.name, self.version, self.hashId) == + + return ((self.name, self.version, self.hashId) == (other.name, other.version, other.hashId)) @@ -114,12 +114,12 @@ def fetch_log(path): def failed_dependencies(spec): - return set(childSpec for childSpec in spec.dependencies.itervalues() if not + return set(childSpec for childSpec in spec.dependencies.itervalues() if not spack.repo.get(childSpec).installed) def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): - # Post-order traversal is not strictly required but it makes sense to output + # Post-order traversal is not strictly required but it makes sense to output # tests for dependencies first. for spec in topSpec.traverse(order='post'): if spec not in newInstalls: @@ -143,12 +143,12 @@ def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): re.search('error:', line, re.IGNORECASE)) errOutput = errMessages if errMessages else lines[-10:] errOutput = '\n'.join(itertools.chain( - [spec.to_yaml(), "Errors:"], errOutput, + [spec.to_yaml(), "Errors:"], errOutput, ["Build Log:", package.build_log_path])) else: result = TestResult.PASSED errOutput = None - + bId = BuildId(spec) output.add_test(bId, result, errOutput) @@ -163,18 +163,18 @@ def test_install(parser, args): if args.no_checksum: spack.do_checksum = False # TODO: remove this global. - + specs = spack.cmd.parse_specs(args.package, concretize=True) if len(specs) > 1: tty.die("Only 1 top-level package can be specified") topSpec = iter(specs).next() - + newInstalls = set() for spec in topSpec.traverse(): package = spack.repo.get(spec) if not package.installed: newInstalls.add(spec) - + if not args.output: bId = BuildId(topSpec) outputDir = join_path(os.getcwd(), "test-output") @@ -183,7 +183,7 @@ def test_install(parser, args): outputFpath = join_path(outputDir, "test-{0}.xml".format(bId.stringId())) else: outputFpath = args.output - + for spec in topSpec.traverse(order='post'): # Calling do_install for the top-level package would be sufficient but # this attempts to keep going if any package fails (other packages which @@ -202,7 +202,7 @@ def test_install(parser, args): pass except FetchError: pass - + jrf = JunitResultFormat() handled = {} create_test_output(topSpec, newInstalls, jrf) From be354e85c9d168b5764f8fec6cd1da53a5b4a7fe Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 24 Jan 2016 16:16:43 -0800 Subject: [PATCH 364/910] Better errors for mkdirp failure in mirror. --- lib/spack/spack/mirror.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 1d9b0e7ef25..d4f1bb89fec 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -147,7 +147,11 @@ def create(path, specs, **kwargs): # Get the absolute path of the root before we start jumping around. mirror_root = os.path.abspath(path) if not os.path.isdir(mirror_root): - mkdirp(mirror_root) + try: + mkdirp(mirror_root) + except OSError as e: + raise MirrorError( + "Cannot create directory '%s':" % mirror_root, str(e)) # Things to keep track of while parsing specs. present = [] @@ -164,7 +168,11 @@ def create(path, specs, **kwargs): # create a subdirectory for the current package@version archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec))) subdir = os.path.dirname(archive_path) - mkdirp(subdir) + try: + mkdirp(subdir) + except OSError as e: + raise MirrorError( + "Cannot create directory '%s':" % subdir, str(e)) if os.path.exists(archive_path): tty.msg("Already added %s" % spec.format("$_$@")) From fe50593c66d8cf7ee527fa13928990595d8b0a29 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 24 Jan 2016 20:14:21 -0800 Subject: [PATCH 365/910] Minor line width reductions. --- lib/spack/spack/cmd/mirror.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 885483a8406..92b92e07af6 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -53,11 +53,13 @@ def setup_parser(subparser): create_parser.add_argument('-d', '--directory', default=None, help="Directory in which to create mirror.") create_parser.add_argument( - 'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror") + 'specs', nargs=argparse.REMAINDER, + help="Specs of packages to put in mirror") create_parser.add_argument( '-f', '--file', help="File with specs of packages to put in mirror.") create_parser.add_argument( - '-D', '--dependencies', action='store_true', help="Also fetch all dependencies") + '-D', '--dependencies', action='store_true', + help="Also fetch all dependencies") create_parser.add_argument( '-o', '--one-version-per-spec', action='store_const', const=1, default=0, help="Only fetch one 'preferred' version per spec, not all known versions.") @@ -74,7 +76,8 @@ def setup_parser(subparser): help="Configuration scope to modify.") # Remove - remove_parser = sp.add_parser('remove', aliases=['rm'], help=mirror_remove.__doc__) + remove_parser = sp.add_parser('remove', aliases=['rm'], + help=mirror_remove.__doc__) remove_parser.add_argument('name') remove_parser.add_argument( '--scope', choices=scopes, default=spack.cmd.default_modify_scope, From d3ff8ca00f7f47d365ea88ed717fa7940a1346c6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 24 Jan 2016 20:14:40 -0800 Subject: [PATCH 366/910] Fixes #382: Issues with spack fetch. - urljoin() was compliant with RFC 1808 but not with my understanding of how paths should be joined. - updated path joining logic to comply. --- lib/spack/spack/stage.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 31635a854a5..19b7d03664d 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -252,7 +252,13 @@ def fetch(self): self.skip_checksum_for_mirror = True if self.mirror_path: mirrors = spack.config.get_config('mirrors') - urls = [urljoin(u, self.mirror_path) for name, u in mirrors.items()] + + # Join URLs of mirror roots with mirror paths. Because + # urljoin() will strip everything past the final '/' in + # the root, so we add a '/' if it is not present. + mirror_roots = [root if root.endswith('/') else root + '/' + for root in mirrors.values()] + urls = [urljoin(root, self.mirror_path) for root in mirror_roots] # If this archive is normally fetched from a tarball URL, # then use the same digest. `spack mirror` ensures that From a48d0a494f8524c789e94a09c182ff1b6e465b8c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 25 Jan 2016 00:32:14 -0800 Subject: [PATCH 367/910] Refactor mock_repo: add a destroy() method. - classes using mock_repo need not know about its stage now. --- lib/spack/spack/test/git_fetch.py | 5 +---- lib/spack/spack/test/hg_fetch.py | 5 +---- lib/spack/spack/test/install.py | 4 +--- lib/spack/spack/test/mirror.py | 23 ++++++++++------------- lib/spack/spack/test/mock_repo.py | 6 ++++++ lib/spack/spack/test/svn_fetch.py | 5 +---- 6 files changed, 20 insertions(+), 28 deletions(-) diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 3813079065b..d84433176a8 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -56,10 +56,7 @@ def setUp(self): def tearDown(self): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() - + self.repo.destroy() self.pkg.do_clean() diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index ee8327aec84..bbcb64e4c1a 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -53,10 +53,7 @@ def setUp(self): def tearDown(self): """Destroy the stage space used by this test.""" super(HgFetchTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() - + self.repo.destroy() self.pkg.do_clean() diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 628329a4237..c09bd24c8e1 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -59,9 +59,7 @@ def setUp(self): def tearDown(self): super(InstallTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() + self.repo.destroy() # Turn checksumming back on spack.do_checksum = True diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 04e9e3db2ef..0093b5d82f1 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -44,8 +44,16 @@ def setUp(self): self.repos = {} + def tearDown(self): + """Destroy all the stages created by the repos in setup.""" + super(MirrorTest, self).tearDown() + for repo in self.repos.values(): + repo.destroy() + self.repos.clear() + + def set_up_package(self, name, MockRepoClass, url_attr): - """Use this to set up a mock package to be mirrored. + """Set up a mock package to be mirrored. Each package needs us to: 1. Set up a mock repo/archive to fetch from. 2. Point the package's version args at that repo. @@ -65,17 +73,6 @@ def set_up_package(self, name, MockRepoClass, url_attr): pkg.versions[v][url_attr] = repo.url - def tearDown(self): - """Destroy all the stages created by the repos in setup.""" - super(MirrorTest, self).tearDown() - - for name, repo in self.repos.items(): - if repo.stage: - pass #repo.stage.destroy() - - self.repos.clear() - - def check_mirror(self): stage = Stage('spack-mirror-test') mirror_root = join_path(stage.path, 'test-mirror') @@ -129,7 +126,7 @@ def check_mirror(self): self.assertTrue(all(l in exclude for l in dcmp.left_only)) finally: - pass #stage.destroy() + stage.destroy() def test_git_mirror(self): diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index 9738ba4e728..ed94023b0eb 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -55,6 +55,12 @@ def __init__(self, stage_name, repo_name): mkdirp(self.path) + def destroy(self): + """Destroy resources associated with this mock repo.""" + if self.stage: + self.stage.destroy() + + class MockArchive(MockRepo): """Creates a very simple archive directory with a configure script and a makefile that installs to a prefix. Tars it up into an archive.""" diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 7d150b42f41..454a7f1d1ff 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -55,10 +55,7 @@ def setUp(self): def tearDown(self): """Destroy the stage space used by this test.""" super(SvnFetchTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() - + self.repo.destroy() self.pkg.do_clean() From 5502fd1054e665d5deb028217f554640bedf8514 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 25 Jan 2016 01:59:02 -0800 Subject: [PATCH 368/910] More thorough mirror test: test full round-trip - Old test: did not attempt to actually fetch mirrored packages. - New test: 1. Creates a temporary mirror, 2. Registers it with spack, 3. Fetches from it, and 4. Verifies that the fetched archive matches the original - This test will hopefully mean that `spack mirror` is less brittle from now on. --- lib/spack/spack/cmd/mirror.py | 1 + lib/spack/spack/package.py | 14 ++++------ lib/spack/spack/stage.py | 10 ++++--- lib/spack/spack/test/mirror.py | 49 +++++++++++++++++----------------- 4 files changed, 37 insertions(+), 37 deletions(-) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 92b92e07af6..a71ad16b658 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -172,6 +172,7 @@ def mirror_create(args): specs = [Spec(n) for n in spack.repo.all_package_names()] specs.sort(key=lambda s: s.format("$_$@").lower()) + # If the user asked for dependencies, traverse spec DAG get them. if args.dependencies: new_specs = set() for spec in specs: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index fba2912b757..8cb947c276c 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -631,7 +631,7 @@ def remove_prefix(self): spack.install_layout.remove_install_directory(self.spec) - def do_fetch(self): + def do_fetch(self, mirror_only=False): """Creates a stage directory and downloads the taball for this package. Working directory will be set to the stage directory. """ @@ -656,7 +656,7 @@ def do_fetch(self): raise FetchError( "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg) - self.stage.fetch() + self.stage.fetch(mirror_only) ########## # Fetch resources @@ -677,7 +677,8 @@ def do_fetch(self): if spack.do_checksum and self.version in self.versions: self.stage.check() - def do_stage(self): + + def do_stage(self, mirror_only=False): """Unpacks the fetched tarball, then changes into the expanded tarball directory.""" if not self.spec.concrete: @@ -691,8 +692,7 @@ def _expand_archive(stage, name=self.name): else: tty.msg("Already staged %s in %s." % (name, stage.path)) - - self.do_fetch() + self.do_fetch(mirror_only) _expand_archive(self.stage) ########## @@ -835,10 +835,6 @@ def _resource_stage(self, resource): resource_stage_folder = '-'.join(pieces) return resource_stage_folder - def _build_logger(self, log_path): - """Create a context manager to log build output.""" - - def do_install(self, keep_prefix=False, keep_stage=False, ignore_deps=False, diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 19b7d03664d..79c9030e20f 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -97,7 +97,6 @@ def __init__(self, url_or_fetch_strategy, **kwargs): self.name = kwargs.get('name') self.mirror_path = kwargs.get('mirror_path') - self.tmp_root = find_tmp_root() self.path = None @@ -240,11 +239,13 @@ def chdir(self): tty.die("Setup failed: no such directory: " + self.path) - def fetch(self): + def fetch(self, mirror_only=False): """Downloads an archive or checks out code from a repository.""" self.chdir() - fetchers = [self.default_fetcher] + fetchers = [] + if not mirror_only: + fetchers.append(self.default_fetcher) # TODO: move mirror logic out of here and clean it up! # TODO: Or @alalazo may have some ideas about how to use a @@ -267,10 +268,11 @@ def fetch(self): if isinstance(self.default_fetcher, fs.URLFetchStrategy): digest = self.default_fetcher.digest - # Have to skip the checkesum for things archived from + # Have to skip the checksum for things archived from # repositories. How can this be made safer? self.skip_checksum_for_mirror = not bool(digest) + # Add URL strategies for all the mirrors with the digest for url in urls: fetchers.insert(0, fs.URLFetchStrategy(url, digest)) diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 0093b5d82f1..046ec56604c 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -77,6 +77,10 @@ def check_mirror(self): stage = Stage('spack-mirror-test') mirror_root = join_path(stage.path, 'test-mirror') + # register mirror with spack config + mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } + spack.config.update_config('mirrors', mirrors) + try: os.chdir(stage.path) spack.mirror.create( @@ -85,7 +89,7 @@ def check_mirror(self): # Stage directory exists self.assertTrue(os.path.isdir(mirror_root)) - # subdirs for each package + # check that there are subdirs for each package for name in self.repos: subdir = join_path(mirror_root, name) self.assertTrue(os.path.isdir(subdir)) @@ -93,38 +97,35 @@ def check_mirror(self): files = os.listdir(subdir) self.assertEqual(len(files), 1) - # Decompress archive in the mirror - archive = files[0] - archive_path = join_path(subdir, archive) - decomp = decompressor_for(archive_path) + # Now try to fetch each package. + for name, mock_repo in self.repos.items(): + spec = Spec(name).concretized() + pkg = spec.package - with working_dir(subdir): - decomp(archive_path) - - # Find the untarred archive directory. - files = os.listdir(subdir) - self.assertEqual(len(files), 2) - self.assertTrue(archive in files) - files.remove(archive) - - expanded_archive = join_path(subdir, files[0]) - self.assertTrue(os.path.isdir(expanded_archive)) + saved_checksum_setting = spack.do_checksum + try: + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) # Compare the original repo with the expanded archive - repo = self.repos[name] - if not 'svn' in name: - original_path = repo.path - else: - co = 'checked_out' - svn('checkout', repo.url, co) - original_path = join_path(subdir, co) + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) - dcmp = dircmp(original_path, expanded_archive) + dcmp = dircmp(original_path, pkg.stage.source_path) # make sure there are no new files in the expanded tarball self.assertFalse(dcmp.right_only) + + # and that all original files are present. self.assertTrue(all(l in exclude for l in dcmp.left_only)) + finally: + spack.do_checksum = saved_checksum_setting + pkg.do_clean() finally: stage.destroy() From 41cd8f8e6d78f45c31145652a518b9d6d7d9a9c3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 25 Jan 2016 02:57:23 -0800 Subject: [PATCH 369/910] Fixes #74: minor fixes to spack mirror. --- lib/spack/spack/cmd/mirror.py | 4 +++- lib/spack/spack/mirror.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index a71ad16b658..8e9438c1a37 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -144,15 +144,17 @@ def mirror_list(args): def _read_specs_from_file(filename): + specs = [] with open(filename, "r") as stream: for i, string in enumerate(stream): try: s = Spec(string) s.package - args.specs.append(s) + specs.append(s) except SpackError, e: tty.die("Parse error in %s, line %d:" % (args.file, i+1), ">>> " + string, str(e)) + return specs def mirror_create(args): diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index d4f1bb89fec..341cc4cb885 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -78,6 +78,7 @@ def get_matching_versions(specs, **kwargs): continue num_versions = kwargs.get('num_versions', 0) + matching_spec = [] for i, v in enumerate(reversed(sorted(pkg.versions))): # Generate no more than num_versions versions for each spec. if num_versions and i >= num_versions: @@ -88,7 +89,11 @@ def get_matching_versions(specs, **kwargs): s = Spec(pkg.name) s.versions = VersionList([v]) s.variants = spec.variants.copy() - matching.append(s) + matching_spec.append(s) + + if not matching_spec: + tty.warn("No known version matches spec: %s" % spec) + matching.extend(matching_spec) return matching From fb5274145aac9266435f3e16f19ce3e30ba1550f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 21 Jan 2016 11:06:27 -0600 Subject: [PATCH 370/910] Fix regex to find version on OS X --- lib/spack/spack/compilers/clang.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 340051019cc..998d4e59323 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -54,4 +54,4 @@ def default_version(self, comp): Thread model: posix """ return get_compiler_version( - comp, '--version', r'(?:clang version|based on LLVM) ([^ )]+)') + comp, '--version', r'(?:clang version|based on LLVM|LLVM version) ([^ )]+)') From 004c99ab2ffa0a9e4635f3dbf9a9beaceb6727d3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 25 Jan 2016 03:41:28 -0800 Subject: [PATCH 371/910] Fixes #259: Apple clang compiler detection - Clang compilers are now versioned with the raw LLVM version or, on macs, with Apple's version with a suffix. --- lib/spack/spack/compilers/clang.py | 32 +++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 998d4e59323..e406d86a245 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -22,7 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import re +import spack.compiler as cpr from spack.compiler import * +from spack.util.executable import * class Clang(Compiler): # Subclasses use possible names of C compiler @@ -47,11 +50,34 @@ class Clang(Compiler): @classmethod def default_version(self, comp): """The '--version' option works for clang compilers. - Output looks like this:: + On most platforms, output looks like this:: clang version 3.1 (trunk 149096) Target: x86_64-unknown-linux-gnu Thread model: posix + + On Mac OS X, it looks like this: + + Apple LLVM version 7.0.2 (clang-700.1.81) + Target: x86_64-apple-darwin15.2.0 + Thread model: posix + """ - return get_compiler_version( - comp, '--version', r'(?:clang version|based on LLVM|LLVM version) ([^ )]+)') + if comp not in cpr._version_cache: + compiler = Executable(comp) + output = compiler('--version', output=str, error=str) + + ver = 'unknown' + match = re.search(r'^Apple LLVM version ([^ )]+)', output) + if match: + # Apple's LLVM compiler has its own versions, so suffix them. + ver = match.group(1) + '-apple' + else: + # Normal clang compiler versions are left as-is + match = re.search(r'^clang version ([^ )]+)', output) + if match: + ver = match.group(1) + + cpr._version_cache[comp] = ver + + return cpr._version_cache[comp] From 323bba99f18d0d774e76feb7f0d28a87fac2d93f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 25 Jan 2016 09:29:51 -0600 Subject: [PATCH 372/910] Fix variable name bug --- bin/spack | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/spack b/bin/spack index 93263217ac1..31165bba9d1 100755 --- a/bin/spack +++ b/bin/spack @@ -42,9 +42,9 @@ SPACK_EXTERNAL_LIBS = os.path.join(SPACK_LIB_PATH, "external") sys.path.insert(0, SPACK_EXTERNAL_LIBS) import warnings -# Avoid warnings when nose is installed with the python exe being used to run -# spack. Note this must be done after Spack's external libs directory is added -# to sys.path. +# Avoid warnings when nose is installed with the python exe being used to run +# spack. Note this must be done after Spack's external libs directory is added +# to sys.path. with warnings.catch_warnings(): warnings.filterwarnings("ignore", ".*nose was already imported") import nose @@ -62,7 +62,7 @@ for pyc_file in orphaned_pyc_files: try: os.remove(pyc_file) except OSError as e: - print "WARNING: Spack may fail mysteriously. Couldn't remove orphaned .pyc file: %s" % pyc + print "WARNING: Spack may fail mysteriously. Couldn't remove orphaned .pyc file: %s" % pyc_file # If there is no working directory, use the spack prefix. try: From 1d18f2031c086b4f1e5bb52744c75b232cbf2e7f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 25 Jan 2016 14:57:33 -0600 Subject: [PATCH 373/910] Add szip variant --- var/spack/repos/builtin/packages/hdf5/package.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index ac78d8e9617..5321a191f0e 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -48,9 +48,11 @@ class Hdf5(Package): variant('unsupported', default=False, description='Enables unsupported configuration options') variant('mpi', default=False, description='Enable MPI support') + variant('szip', default=False, description='Enable szip support') variant('threadsafe', default=False, description='Enable thread-safe capabilities') depends_on("mpi", when='+mpi') + depends_on("szip", when='+szip') depends_on("zlib") def validate(self, spec): @@ -105,6 +107,9 @@ def install(self, spec, prefix): if '+fortran' in spec: extra_args.append("FC=%s" % spec['mpi'].prefix.bin + "/mpifort") + if '+szip' in spec: + extra_args.append("--with-szlib=%s" % spec['szip'].prefix) + if '+threadsafe' in spec: extra_args.extend([ '--enable-threadsafe', From 093b8317998c395ed3e2dee7f19d1bdcea1b0560 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 26 Jan 2016 12:08:28 +0100 Subject: [PATCH 374/910] Fixes issues introduced after merge with conflicts --- lib/spack/spack/package.py | 3 +++ lib/spack/spack/stage.py | 43 +++++++++++++++++++------------------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 06aecf11bd5..14cbc0dbce6 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -804,6 +804,9 @@ def _get_resources(self): for when_spec, resource_list in self.resources.items(): if when_spec in self.spec: resources.extend(resource_list) + # Sorts the resources by the length of the string representing their destination. Since any nested resource + # must contain another resource's name in its path, it seems that should work + resources = sorted(resources, key=lambda res: len(res.destination)) return resources def _resource_stage(self, resource): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 1deac1137c4..6ba301d95f4 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -1,4 +1,4 @@ -############################################################################## +1 ############################################################################## # Copyright (c) 2013, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import re +import errno import shutil import tempfile from urlparse import urljoin @@ -38,7 +38,6 @@ import spack.fetch_strategy as fs import spack.error - STAGE_PREFIX = 'spack-stage-' @@ -95,7 +94,7 @@ def __init__(self, url_or_fetch_strategy, **kwargs): raise ValueError("Can't construct Stage without url or fetch strategy") self.fetcher.set_stage(self) self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. - self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. + self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. self.name = kwargs.get('name') self.mirror_path = kwargs.get('mirror_path') @@ -104,7 +103,6 @@ def __init__(self, url_or_fetch_strategy, **kwargs): self.path = None self._setup() - def _cleanup_dead_links(self): """Remove any dead links in the stage directory.""" for file in os.listdir(spack.stage_path): @@ -114,7 +112,6 @@ def _cleanup_dead_links(self): if not os.path.exists(path): os.unlink(path) - def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we looked at path. Returns True if path already exists and is ok. @@ -132,7 +129,7 @@ def _need_to_create_path(self): # Path looks ok, but need to check the target of the link. if os.path.islink(self.path): real_path = os.path.realpath(self.path) - real_tmp = os.path.realpath(self.tmp_root) + real_tmp = os.path.realpath(self.tmp_root) if spack.use_tmp_stage: # If we're using a tmp dir, it's a link, and it points at the right spot, @@ -151,7 +148,6 @@ def _need_to_create_path(self): return False - def _setup(self): """Creates the stage directory. If spack.use_tmp_stage is False, the stage directory is created @@ -200,7 +196,6 @@ def _setup(self): # Make sure we can actually do something with the stage we made. ensure_access(self.path) - @property def archive_file(self): """Path to the source archive within this stage directory.""" @@ -217,7 +212,6 @@ def archive_file(self): else: return None - @property def source_path(self): """Returns the path to the expanded/checked out source code @@ -232,7 +226,6 @@ def source_path(self): return p return None - def chdir(self): """Changes directory to the stage path. Or dies if it is not set up.""" if os.path.isdir(self.path): @@ -240,7 +233,6 @@ def chdir(self): else: tty.die("Setup failed: no such directory: " + self.path) - def fetch(self, mirror_only=False): """Downloads an archive or checks out code from a repository.""" self.chdir() @@ -293,7 +285,6 @@ def fetch(self, mirror_only=False): self.fetcher = self.default_fetcher raise fs.FetchError(errMessage, None) - def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" @@ -307,7 +298,6 @@ def check(self): else: self.fetcher.check() - def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded archive. Fail if the stage is not set up or if the archive is not yet @@ -320,7 +310,6 @@ def expand_archive(self): else: tty.msg("Already staged %s in %s." % (self.name, self.path)) - def chdir_to_source(self): """Changes directory to the expanded archive directory. Dies with an error if there was no expanded archive. @@ -333,14 +322,12 @@ def chdir_to_source(self): if not os.listdir(path): tty.die("Archive was empty for %s" % self.name) - def restage(self): """Removes the expanded archive path if it exists, then re-expands the archive. """ self.fetcher.reset() - def destroy(self): """Remove this stage directory.""" remove_linked_tree(self.path) @@ -367,11 +354,24 @@ def expand_archive(self): placement = {'': placement} # Make the paths in the dictionary absolute and link for key, value in placement.iteritems(): - link_path = join_path(root_stage.source_path, resource.destination, value) + target_path = join_path(root_stage.source_path, resource.destination) + destination_path = join_path(target_path, value) source_path = join_path(self.source_path, key) - if not os.path.exists(link_path): + + try: + os.makedirs(target_path) + except OSError as err: + if err.errno == errno.EEXIST and os.path.isdir(target_path): + pass + else: + raise + + if not os.path.exists(destination_path): # Create a symlink - os.symlink(source_path, link_path) + tty.info('Moving resource stage\n\tsource : {stage}\n\tdestination : {destination}'.format( + stage=source_path, destination=destination_path + )) + shutil.move(source_path, destination_path) @pattern.composite(method_list=['fetch', 'check', 'expand_archive', 'restage', 'destroy']) @@ -380,6 +380,7 @@ class StageComposite: Composite for Stage type objects. The first item in this composite is considered to be the root package, and operations that return a value are forwarded to it. """ + @property def source_path(self): return self[0].source_path @@ -394,6 +395,7 @@ def chdir_to_source(self): class DIYStage(object): """Simple class that allows any directory to be a spack stage.""" + def __init__(self, path): self.archive_file = None self.path = path @@ -431,7 +433,6 @@ def _get_mirrors(): return [val for name, val in config.iteritems()] - def ensure_access(file=spack.stage_path): """Ensure we can access a directory and die with an error if we can't.""" if not can_access(file): From 5e3b7a424765ee64f71192bdca129c104a1246f6 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 26 Jan 2016 14:11:23 +0100 Subject: [PATCH 375/910] Fixes issues introduced after merge with conflicts --- lib/spack/spack/package.py | 2 +- lib/spack/spack/stage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 14cbc0dbce6..a1b8d12ec29 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -702,7 +702,7 @@ def do_stage(self, mirror_only=False): if not self.spec.concrete: raise ValueError("Can only stage concrete packages.") - self.do_fetch() + self.do_fetch(mirror_only) self.stage.expand_archive() self.stage.chdir_to_source() diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 6ba301d95f4..f217450d42d 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -1,4 +1,4 @@ -1 ############################################################################## +############################################################################## # Copyright (c) 2013, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # From ee6f69a227a4ee005759a64bdaa542745972d8e0 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 26 Jan 2016 14:25:16 +0100 Subject: [PATCH 376/910] Unit tests : hack to make them pass while a decision on how to proceed is made --- lib/spack/spack/test/mirror.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 046ec56604c..f83cc8090c9 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -102,6 +102,7 @@ def check_mirror(self): spec = Spec(name).concretized() pkg = spec.package + pkg._stage = None saved_checksum_setting = spack.do_checksum try: # Stage the archive from the mirror and cd to it. From 7891346a4ff939bf14c1c3a688f50a2be9b5dbbe Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Tue, 26 Jan 2016 09:52:01 -0800 Subject: [PATCH 377/910] fix for issue 398, added extensions_for in RepoPath --- lib/spack/spack/repository.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 31596cee7a5..b5df1168b66 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -233,6 +233,11 @@ def providers_for(self, vpkg_spec): return providers + @_autospec + def extensions_for(self, extendee_spec): + return [p for p in self.all_packages() if p.extends(extendee_spec)] + + def find_module(self, fullname, path=None): """Implements precedence for overlaid namespaces. From c6bb00085f89b6db37a03674ba09714adab1f2f0 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 26 Jan 2016 12:04:48 -0600 Subject: [PATCH 378/910] Extensive modifications to NetCDF package --- .../repos/builtin/packages/netcdf/package.py | 50 ++++++++++++++----- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 239644d894c..93c4410146f 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -2,27 +2,53 @@ class Netcdf(Package): """NetCDF is a set of software libraries and self-describing, machine-independent - data formats that support the creation, access, and sharing of array-oriented - scientific data.""" + data formats that support the creation, access, and sharing of array-oriented + scientific data.""" homepage = "http://www.unidata.ucar.edu/software/netcdf/" url = "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.3.3.tar.gz" + version('4.4.0', 'f01cb26a0126dd9a6224e76472d25f6c') version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') + variant('fortran', default=False, description="Download and install NetCDF-Fortran") + variant('hdf4', default=False, description="Enable HDF4 support") + patch('netcdf-4.3.3-mpi.patch') # Dependencies: - depends_on("cmake @2.8.12:") - # >HDF5 - depends_on("hdf5") + depends_on("curl") # required for DAP support + depends_on("hdf", when='+hdf4') + depends_on("hdf5") # required for NetCDF-4 support + depends_on("zlib") # required for NetCDF-4 support def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('..', - "-DCMAKE_INSTALL_PREFIX:PATH=%s" % prefix, - "-DENABLE_DAP:BOOL=OFF", # Disable DAP. - "-DBUILD_SHARED_LIBS:BOOL=OFF") # Don't build shared libraries (use static libs). + config_args = [ + "--enable-fsync", + "--enable-v2", + "--enable-utilities", + "--enable-shared", + "--enable-static", + "--enable-largefile", + # necessary for HDF5 support + "--enable-netcdf-4", + "--enable-dynamic-loading", + # necessary for DAP support + "--enable-dap" + ] - make() - make("install") + # HDF4 support + if '+hdf4' in spec: + config_args.append("--enable-hdf4") + + # Fortran support + # In version 4.2+, NetCDF-C and NetCDF-Fortran have split. + # They can be installed separately, but this bootstrap procedure + # should be able to install both at the same time. + # Note: this is a new experimental feature + if '+fortran' in spec: + config_args.append("--enable-remote-fortran-bootstrap") + + configure(*config_args) + make() + make("install") From 81ccba202c601edacaf483eacc7373d1947e4875 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 26 Jan 2016 15:38:22 -0600 Subject: [PATCH 379/910] Make szip a variant --- .../repos/builtin/packages/hdf/package.py | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf/package.py b/var/spack/repos/builtin/packages/hdf/package.py index 43b59802301..1ecb1671836 100644 --- a/var/spack/repos/builtin/packages/hdf/package.py +++ b/var/spack/repos/builtin/packages/hdf/package.py @@ -11,8 +11,10 @@ class Hdf(Package): version('4.2.11', '063f9928f3a19cc21367b71c3b8bbf19') + variant('szip', default=False, description="Enable szip support") + depends_on("jpeg") - depends_on("szip") + depends_on("szip", when='+szip') depends_on("zlib") @@ -21,15 +23,22 @@ def url_for_version(self, version): def install(self, spec, prefix): - configure('--prefix=%s' % prefix, - '--with-jpeg=%s' % spec['jpeg'].prefix, - '--with-szlib=%s' % spec['szip'].prefix, - '--with-zlib=%s' % spec['zlib'].prefix, - '--disable-netcdf', - '--enable-fortran', - '--disable-shared', - '--enable-static', - '--enable-production') + config_args = [ + '--prefix=%s' % prefix, + '--with-jpeg=%s' % spec['jpeg'].prefix, + '--with-zlib=%s' % spec['zlib'].prefix, + '--disable-netcdf', + '--enable-fortran', + '--disable-shared', + '--enable-static', + '--enable-production' + ] + + # SZip support + if '+szip' in spec: + config_args.append('--with-szlib=%s' % spec['szip'].prefix) + + configure(*config_args) make() make("install") From 23af31cb1c773cd5a01f93a4896d794ddf908bae Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Tue, 26 Jan 2016 17:55:32 -0800 Subject: [PATCH 380/910] adding the cmocka unit testing library --- .../repos/builtin/packages/cmocka/package.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 var/spack/repos/builtin/packages/cmocka/package.py diff --git a/var/spack/repos/builtin/packages/cmocka/package.py b/var/spack/repos/builtin/packages/cmocka/package.py new file mode 100644 index 00000000000..7377016a6b0 --- /dev/null +++ b/var/spack/repos/builtin/packages/cmocka/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Cmocka(Package): + """Unit-testing framework in pure C""" + homepage = "https://cmocka.org/" + url = "https://cmocka.org/files/1.0/cmocka-1.0.1.tar.xz" + + version('1.0.1', 'ed861e501a21a92b2af63e466df2015e') + parallel = False + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + + make() + make("install") From f1d8f30342f21d772b2bf4fc76a9a8b262e479c5 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 27 Jan 2016 10:54:41 +0100 Subject: [PATCH 381/910] Bug fix: precedence in % operator seems to change depending in the python version --- var/spack/repos/builtin/packages/netlib-scalapack/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 5be91c4a40c..22d538560e8 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -22,8 +22,8 @@ class NetlibScalapack(Package): def install(self, spec, prefix): options = [ - "-DBUILD_SHARED_LIBS:BOOL=%s" % 'ON' if '+shared' in spec else 'OFF', - "-DBUILD_STATIC_LIBS:BOOL=%s" % 'OFF' if '+shared' in spec else 'ON', + "-DBUILD_SHARED_LIBS:BOOL=%s" % ('ON' if '+shared' in spec else 'OFF'), + "-DBUILD_STATIC_LIBS:BOOL=%s" % ('OFF' if '+shared' in spec else 'ON'), "-DUSE_OPTIMIZED_LAPACK_BLAS:BOOL=ON", # forces scalapack to use find_package(LAPACK) ] From f7f192e12b0387537652ba4914e6e484d29ef728 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 27 Jan 2016 17:12:24 +0100 Subject: [PATCH 382/910] Added unit tests for util.pattern --- lib/spack/spack/test/__init__.py | 1 + lib/spack/spack/test/pattern.py | 101 +++++++++++++++++++++++++++++++ lib/spack/spack/util/pattern.py | 2 +- 3 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 lib/spack/spack/test/pattern.py diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index a569cbbf359..4b9a361d4bf 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -48,6 +48,7 @@ 'package_sanity', 'config', 'directory_layout', + 'pattern', 'python_version', 'git_fetch', 'svn_fetch', diff --git a/lib/spack/spack/test/pattern.py b/lib/spack/spack/test/pattern.py new file mode 100644 index 00000000000..64fc9187f9a --- /dev/null +++ b/lib/spack/spack/test/pattern.py @@ -0,0 +1,101 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import unittest + +import spack.util.pattern as pattern + + +class CompositeTest(unittest.TestCase): + + def setUp(self): + class Base: + counter = 0 + + def add(self): + raise NotImplemented('add not implemented') + + def subtract(self): + raise NotImplemented('subtract not implemented') + + class One(Base): + def add(self): + Base.counter += 1 + + def subtract(self): + Base.counter -= 1 + + class Two(Base): + def add(self): + Base.counter += 2 + + def subtract(self): + Base.counter -= 2 + + self.Base = Base + self.One = One + self.Two = Two + + def test_composite_from_method_list(self): + + @pattern.composite(method_list=['add', 'subtract']) + class CompositeFromMethodList: + pass + + composite = CompositeFromMethodList() + composite.append(self.One()) + composite.append(self.Two()) + composite.add() + self.assertEqual(self.Base.counter, 3) + composite.pop() + composite.subtract() + self.assertEqual(self.Base.counter, 2) + + def test_composite_from_interface(self): + + @pattern.composite(interface=self.Base) + class CompositeFromInterface: + pass + + composite = CompositeFromInterface() + composite.append(self.One()) + composite.append(self.Two()) + composite.add() + self.assertEqual(self.Base.counter, 3) + composite.pop() + composite.subtract() + self.assertEqual(self.Base.counter, 2) + + def test_error_conditions(self): + + with self.assertRaises(TypeError): + @pattern.composite(interface=self.Base, container=2) + class CompositeFromInterface: + pass + + with self.assertRaises(TypeError): + @pattern.composite() + class CompositeFromInterface: + pass diff --git a/lib/spack/spack/util/pattern.py b/lib/spack/spack/util/pattern.py index 73c1e26aa5d..17a126498bf 100644 --- a/lib/spack/spack/util/pattern.py +++ b/lib/spack/spack/util/pattern.py @@ -53,7 +53,7 @@ def composite(interface=None, method_list=None, container=list): raise TypeError("Either 'interface' or 'method_list' must be defined on a call to composite") def cls_decorator(cls): - # Retrieve the base class of the composite. Inspect its the methods and decide which ones will be overridden + # Retrieve the base class of the composite. Inspect its methods and decide which ones will be overridden def no_special_no_private(x): return inspect.ismethod(x) and not x.__name__.startswith('_') From 47035671e80f071c970dc66b15ad7ced2b87329d Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 27 Jan 2016 17:22:12 +0100 Subject: [PATCH 383/910] unit tests : now compliant with 2.6 --- lib/spack/spack/test/pattern.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/pattern.py b/lib/spack/spack/test/pattern.py index 64fc9187f9a..6c783c6a5f7 100644 --- a/lib/spack/spack/test/pattern.py +++ b/lib/spack/spack/test/pattern.py @@ -90,12 +90,15 @@ class CompositeFromInterface: def test_error_conditions(self): - with self.assertRaises(TypeError): + def wrong_container(): @pattern.composite(interface=self.Base, container=2) class CompositeFromInterface: pass - with self.assertRaises(TypeError): + def no_methods(): @pattern.composite() class CompositeFromInterface: pass + + self.assertRaises(TypeError, wrong_container) + self.assertRaises(TypeError, no_methods) From 6122642b818a6f1cfa999482949fed7154b2dbf0 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 27 Jan 2016 16:16:33 -0600 Subject: [PATCH 384/910] More NetCDF changes --- .../repos/builtin/packages/hdf/package.py | 9 ++--- .../packages/netcdf/netcdf-4.3.3-mpi.patch | 25 -------------- .../repos/builtin/packages/netcdf/package.py | 34 ++++++++++++++++--- 3 files changed, 35 insertions(+), 33 deletions(-) delete mode 100644 var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch diff --git a/var/spack/repos/builtin/packages/hdf/package.py b/var/spack/repos/builtin/packages/hdf/package.py index 1ecb1671836..ac6435f2a24 100644 --- a/var/spack/repos/builtin/packages/hdf/package.py +++ b/var/spack/repos/builtin/packages/hdf/package.py @@ -24,12 +24,13 @@ def url_for_version(self, version): def install(self, spec, prefix): config_args = [ + 'CFLAGS=-fPIC', '--prefix=%s' % prefix, - '--with-jpeg=%s' % spec['jpeg'].prefix, - '--with-zlib=%s' % spec['zlib'].prefix, - '--disable-netcdf', + '--with-jpeg=%s' % spec['jpeg'].prefix, + '--with-zlib=%s' % spec['zlib'].prefix, + '--disable-netcdf', # must be disabled to build NetCDF with HDF4 support '--enable-fortran', - '--disable-shared', + '--disable-shared', # fortran and shared libraries are not compatible '--enable-static', '--enable-production' ] diff --git a/var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch b/var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch deleted file mode 100644 index 46dda5fc9de..00000000000 --- a/var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch +++ /dev/null @@ -1,25 +0,0 @@ -diff -Nur netcdf-4.3.3/CMakeLists.txt netcdf-4.3.3.mpi/CMakeLists.txt ---- netcdf-4.3.3/CMakeLists.txt 2015-02-12 16:44:35.000000000 -0500 -+++ netcdf-4.3.3.mpi/CMakeLists.txt 2015-10-14 16:44:41.176300658 -0400 -@@ -753,6 +753,7 @@ - SET(USE_PARALLEL OFF CACHE BOOL "") - MESSAGE(STATUS "Cannot find HDF5 library built with parallel support. Disabling parallel build.") - ELSE() -+ FIND_PACKAGE(MPI REQUIRED) - SET(USE_PARALLEL ON CACHE BOOL "") - SET(STATUS_PARALLEL "ON") - ENDIF() -diff -Nur netcdf-4.3.3/liblib/CMakeLists.txt netcdf-4.3.3.mpi/liblib/CMakeLists.txt ---- netcdf-4.3.3/liblib/CMakeLists.txt 2015-02-12 16:44:35.000000000 -0500 -+++ netcdf-4.3.3.mpi/liblib/CMakeLists.txt 2015-10-14 16:44:57.757793634 -0400 -@@ -71,6 +71,10 @@ - SET(TLL_LIBS ${TLL_LIBS} ${CURL_LIBRARY}) - ENDIF() - -+IF(USE_PARALLEL) -+ SET(TLL_LIBS ${TLL_LIBS} ${MPI_C_LIBRARIES}) -+ENDIF() -+ - IF(USE_HDF4) - SET(TLL_LIBS ${TLL_LIBS} ${HDF4_LIBRARIES}) - ENDIF() diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 93c4410146f..3cd0b2ee7a8 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -12,9 +12,7 @@ class Netcdf(Package): version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') variant('fortran', default=False, description="Download and install NetCDF-Fortran") - variant('hdf4', default=False, description="Enable HDF4 support") - - patch('netcdf-4.3.3-mpi.patch') + variant('hdf4', default=False, description="Enable HDF4 support") # Dependencies: depends_on("curl") # required for DAP support @@ -23,7 +21,13 @@ class Netcdf(Package): depends_on("zlib") # required for NetCDF-4 support def install(self, spec, prefix): + # Environment variables + CPPFLAGS = [] + LDFLAGS = [] + LIBS = [] + config_args = [ + "--prefix=%s" % prefix, "--enable-fsync", "--enable-v2", "--enable-utilities", @@ -37,18 +41,40 @@ def install(self, spec, prefix): "--enable-dap" ] + CPPFLAGS.append("-I%s/include" % spec['hdf5'].prefix) + LDFLAGS.append( "-L%s/lib" % spec['hdf5'].prefix) + # HDF4 support + # As of NetCDF 4.1.3, "--with-hdf4=..." is no longer a valid option + # You must use the environment variables CPPFLAGS and LDFLAGS if '+hdf4' in spec: config_args.append("--enable-hdf4") + CPPFLAGS.append("-I%s/include" % spec['hdf'].prefix) + LDFLAGS.append( "-L%s/lib" % spec['hdf'].prefix) + LIBS.append( "-l%s" % "jpeg") + + if 'szip' in spec: + CPPFLAGS.append("-I%s/include" % spec['szip'].prefix) + LDFLAGS.append( "-L%s/lib" % spec['szip'].prefix) + LIBS.append( "-l%s" % "sz") # Fortran support # In version 4.2+, NetCDF-C and NetCDF-Fortran have split. # They can be installed separately, but this bootstrap procedure # should be able to install both at the same time. - # Note: this is a new experimental feature + # Note: this is a new experimental feature. if '+fortran' in spec: config_args.append("--enable-remote-fortran-bootstrap") + config_args.append('CPPFLAGS=%s' % ' '.join(CPPFLAGS)) + config_args.append('LDFLAGS=%s' % ' '.join(LDFLAGS)) + config_args.append('LIBS=%s' % ' '.join(LIBS)) + configure(*config_args) make() make("install") + + # After installing NetCDF-C, install NetCDF-Fortran + if '+fortran' in spec: + make("build-netcdf-fortran") + make("install-netcdf-fortran") From 07bb6fef01bfe48aa22c39e53b75e4c779ac0c2e Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 28 Jan 2016 10:58:56 +0100 Subject: [PATCH 385/910] resource directive : now works with all the fetch strategies available --- lib/spack/spack/directives.py | 2 +- lib/spack/spack/fetch_strategy.py | 19 ++++++++++++++++--- lib/spack/spack/package.py | 1 - .../repos/builtin/packages/llvm/package.py | 19 +++++++++++++++++++ 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 0b98211cb9e..5745adce63c 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -296,8 +296,8 @@ def resource(pkg, **kwargs): raise RuntimeError(message) when_spec = parse_anonymous_spec(when, pkg.name) resources = pkg.resources.setdefault(when_spec, []) - fetcher = from_kwargs(**kwargs) name = kwargs.get('name') + fetcher = from_kwargs(**kwargs) resources.append(Resource(name, fetcher, destination, placement)) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index b2ff587a606..83a2dbb59c5 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -44,6 +44,7 @@ import sys import re import shutil +import copy from functools import wraps import llnl.util.tty as tty from llnl.util.filesystem import * @@ -370,8 +371,12 @@ class GitFetchStrategy(VCSFetchStrategy): required_attributes = ('git',) def __init__(self, **kwargs): + # Discards the keywords in kwargs that may conflict with the next call to __init__ + forwarded_args = copy.copy(kwargs) + forwarded_args.pop('name', None) + super(GitFetchStrategy, self).__init__( - 'git', 'tag', 'branch', 'commit', **kwargs) + 'git', 'tag', 'branch', 'commit', **forwarded_args) self._git = None @property @@ -479,8 +484,12 @@ class SvnFetchStrategy(VCSFetchStrategy): required_attributes = ['svn'] def __init__(self, **kwargs): + # Discards the keywords in kwargs that may conflict with the next call to __init__ + forwarded_args = copy.copy(kwargs) + forwarded_args.pop('name', None) + super(SvnFetchStrategy, self).__init__( - 'svn', 'revision', **kwargs) + 'svn', 'revision', **forwarded_args) self._svn = None if self.revision is not None: self.revision = str(self.revision) @@ -556,8 +565,12 @@ class HgFetchStrategy(VCSFetchStrategy): required_attributes = ['hg'] def __init__(self, **kwargs): + # Discards the keywords in kwargs that may conflict with the next call to __init__ + forwarded_args = copy.copy(kwargs) + forwarded_args.pop('name', None) + super(HgFetchStrategy, self).__init__( - 'hg', 'revision', **kwargs) + 'hg', 'revision', **forwarded_args) self._hg = None @property diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index a1b8d12ec29..8019b29cba5 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -435,7 +435,6 @@ def url_for_version(self, version): def _make_resource_stage(self, root_stage, fetcher, resource): resource_stage_folder = self._resource_stage(resource) - # FIXME : works only for URLFetchStrategy resource_mirror = join_path(self.name, os.path.basename(fetcher.url)) stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource, name=resource_stage_folder, mirror_path=resource_mirror) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index a2b2c6ecccb..1805d3ded80 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -171,6 +171,25 @@ class Llvm(Package): when='@%(version)s' % release, placement=resources[name].get('placement', None)) + # SVN - current develop + version('develop', svn='http://llvm.org/svn/llvm-project/llvm/trunk') + resource(name='clang', svn='http://llvm.org/svn/llvm-project/cfe/trunk', + destination='tools', when='@develop', placement='clang') + resource(name='compiler-rt', svn='http://llvm.org/svn/llvm-project/compiler-rt/trunk', + destination='projects', when='@develop', placement='compiler-rt') + resource(name='openmp', svn='http://llvm.org/svn/llvm-project/openmp/trunk', + destination='projects', when='@develop', placement='openmp') + resource(name='libcxx', svn='http://llvm.org/svn/llvm-project/libcxx/trunk', + destination='projects', when='@develop', placement='libcxx') + resource(name='libcxxabi', svn='http://llvm.org/svn/llvm-project/libcxxabi/trunk', + destination='projects', when='@develop', placement='libcxxabi') + resource(name='polly', svn='http://llvm.org/svn/llvm-project/polly/trunk', + destination='tools', when='@develop', placement='polly') + resource(name='lldb', svn='http://llvm.org/svn/llvm-project/lldb/trunk', + destination='tools', when='@develop', placement='lldb') + + + def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag cmake_args = [ arg for arg in std_cmake_args if 'BUILD_TYPE' not in arg ] From 603f5e69ee43e1249e413c966cde3e02ba10a1a3 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 28 Jan 2016 11:30:28 +0100 Subject: [PATCH 386/910] removed outdated code : os detection (will be handled by platforms) and openssl modifications --- lib/spack/spack/architecture.py | 89 ------------------- .../repos/builtin/packages/openssl/package.py | 15 ---- 2 files changed, 104 deletions(-) delete mode 100644 lib/spack/spack/architecture.py diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py deleted file mode 100644 index 2701fab90c9..00000000000 --- a/lib/spack/spack/architecture.py +++ /dev/null @@ -1,89 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import os -import re -import platform - -from llnl.util.lang import memoized - -import spack -import spack.error as serr - - -class InvalidSysTypeError(serr.SpackError): - def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__( - "Invalid sys_type value for Spack: " + sys_type) - - -class NoSysTypeError(serr.SpackError): - def __init__(self): - super(NoSysTypeError, self).__init__( - "Could not determine sys_type for this machine.") - - -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" - if not hasattr(spack, "sys_type"): - return None - elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() - else: - return spack.sys_type - - -def get_sys_type_from_environment(): - """Return $SYS_TYPE or None if it's not defined.""" - return os.environ.get('SYS_TYPE') - - -def get_sys_type_from_platform(): - """Return the architecture from Python's platform module.""" - sys_type = platform.system() + '-' + platform.machine() - sys_type = re.sub(r'[^\w-]', '_', sys_type) - return sys_type.lower() - - -@memoized -def sys_type(): - """Returns a SysType for the current machine.""" - methods = [get_sys_type_from_spack_globals, - get_sys_type_from_environment, - get_sys_type_from_platform] - - # search for a method that doesn't return None - sys_type = None - for method in methods: - sys_type = method() - if sys_type: break - - # Couldn't determine the sys_type for this machine. - if sys_type is None: - return "unknown_arch" - - if not isinstance(sys_type, basestring): - raise InvalidSysTypeError(sys_type) - - return sys_type diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 8f0427796be..bbb169ec6bc 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -1,7 +1,5 @@ from spack import * -import spack.util.architecture as arch - class Openssl(Package): """The OpenSSL Project is a collaborative effort to develop a robust, commercial-grade, full-featured, and Open Source @@ -40,16 +38,3 @@ def install(self, spec, prefix): make() make("install") - - @when(arch.os_is_in('darwin')) - def install(self, spec, prefix): - perl = which('perl') - perl("./Configure", - "--prefix=%s" % prefix, - "--openssldir=%s/etc/openssl" % prefix, - "zlib", - "no-krb5", - "shared", - "darwin64-x86_64-cc") - filter_file(r'-arch x86_64', '', 'Makefile') - From 0cf03518f3edf8b70331b1b5f2eae171d0a0dccf Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 28 Jan 2016 12:06:30 +0100 Subject: [PATCH 387/910] reverted deletion of the wrong `architecture.py` --- lib/spack/spack/architecture.py | 89 ++++++++++++++++++++++++++++ lib/spack/spack/util/architecture.py | 46 -------------- 2 files changed, 89 insertions(+), 46 deletions(-) create mode 100644 lib/spack/spack/architecture.py delete mode 100644 lib/spack/spack/util/architecture.py diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py new file mode 100644 index 00000000000..2701fab90c9 --- /dev/null +++ b/lib/spack/spack/architecture.py @@ -0,0 +1,89 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import re +import platform + +from llnl.util.lang import memoized + +import spack +import spack.error as serr + + +class InvalidSysTypeError(serr.SpackError): + def __init__(self, sys_type): + super(InvalidSysTypeError, self).__init__( + "Invalid sys_type value for Spack: " + sys_type) + + +class NoSysTypeError(serr.SpackError): + def __init__(self): + super(NoSysTypeError, self).__init__( + "Could not determine sys_type for this machine.") + + +def get_sys_type_from_spack_globals(): + """Return the SYS_TYPE from spack globals, or None if it isn't set.""" + if not hasattr(spack, "sys_type"): + return None + elif hasattr(spack.sys_type, "__call__"): + return spack.sys_type() + else: + return spack.sys_type + + +def get_sys_type_from_environment(): + """Return $SYS_TYPE or None if it's not defined.""" + return os.environ.get('SYS_TYPE') + + +def get_sys_type_from_platform(): + """Return the architecture from Python's platform module.""" + sys_type = platform.system() + '-' + platform.machine() + sys_type = re.sub(r'[^\w-]', '_', sys_type) + return sys_type.lower() + + +@memoized +def sys_type(): + """Returns a SysType for the current machine.""" + methods = [get_sys_type_from_spack_globals, + get_sys_type_from_environment, + get_sys_type_from_platform] + + # search for a method that doesn't return None + sys_type = None + for method in methods: + sys_type = method() + if sys_type: break + + # Couldn't determine the sys_type for this machine. + if sys_type is None: + return "unknown_arch" + + if not isinstance(sys_type, basestring): + raise InvalidSysTypeError(sys_type) + + return sys_type diff --git a/lib/spack/spack/util/architecture.py b/lib/spack/spack/util/architecture.py deleted file mode 100644 index a020c74a7ac..00000000000 --- a/lib/spack/spack/util/architecture.py +++ /dev/null @@ -1,46 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## - -import sys - - -def os_is_in(*args): - """ - Return True if any element in the list is equal to sys.platform, False otherwise - - :param args: list of names to be checked - :return: True or False - """ - return any(map(lambda x: x == sys.platform, args)) - - -def os_is_not_in(*args): - """ - Return True if none of the elements in the list is equal to sys.platform, False otherwise - - :param args: list of names to be checked - :return: True or False - """ - return not os_is_in(*args) From 5850d8530ef0570e508fb1ebd53428e98cc761a3 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Thu, 28 Jan 2016 14:22:28 +0100 Subject: [PATCH 388/910] Adding the stager to checksum any url that spack can handle --- lib/spack/spack/cmd/md5.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index ef1e4f34756..b8ffb5dec79 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -30,8 +30,9 @@ from llnl.util.filesystem import * import spack.util.crypto +from spack.stage import Stage, FailedDownloadError -description = "Calculate md5 checksums for files." +description = "Calculate md5 checksums for files/urls." def setup_parser(subparser): setup_parser.parser = subparser @@ -45,9 +46,20 @@ def md5(parser, args): for f in args.files: if not os.path.isfile(f): - tty.die("Not a file: %s" % f) - if not can_access(f): - tty.die("Cannot read file: %s" % f) + stage = Stage(f) + try: + stage.fetch() + checksum = spack.util.crypto.checksum(hashlib.md5, stage.archive_file) + print "%s %s" % (checksum, f) + except FailedDownloadError, e: + tty.msg("Failed to fetch %s" % url) + continue - checksum = spack.util.crypto.checksum(hashlib.md5, f) - print "%s %s" % (checksum, f) + finally: + stage.destroy() + else: + if not can_access(f): + tty.die("Cannot read file: %s" % f) + + checksum = spack.util.crypto.checksum(hashlib.md5, f) + print "%s %s" % (checksum, f) From 824546d343917fb24c1b94af0eeaea79371efb46 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Thu, 28 Jan 2016 14:43:30 +0100 Subject: [PATCH 389/910] correcting a bug when url and files are mixed --- lib/spack/spack/cmd/md5.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index b8ffb5dec79..879ef9f7b74 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -23,8 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import hashlib import argparse +import hashlib + +from contextlib import contextmanager import llnl.util.tty as tty from llnl.util.filesystem import * @@ -34,6 +36,19 @@ description = "Calculate md5 checksums for files/urls." +@contextmanager +def stager(url): + _cwd = os.getcwd() + _stager = Stage(url) + try: + _stager.fetch() + yield _stager + except FailedDownloadError: + tty.msg("Failed to fetch %s" % url) + finally: + _stager.destroy() + os.chdir(_cwd) # the Stage class changes the current working dir so it has to be restored + def setup_parser(subparser): setup_parser.parser = subparser subparser.add_argument('files', nargs=argparse.REMAINDER, @@ -46,17 +61,9 @@ def md5(parser, args): for f in args.files: if not os.path.isfile(f): - stage = Stage(f) - try: - stage.fetch() + with stager(f) as stage: checksum = spack.util.crypto.checksum(hashlib.md5, stage.archive_file) print "%s %s" % (checksum, f) - except FailedDownloadError, e: - tty.msg("Failed to fetch %s" % url) - continue - - finally: - stage.destroy() else: if not can_access(f): tty.die("Cannot read file: %s" % f) From 06f3cc33baeb1c8a8e99b0e0e602f398b4d221bd Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 28 Jan 2016 11:41:12 -0600 Subject: [PATCH 390/910] Modify url settings for fish package --- var/spack/repos/builtin/packages/fish/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/fish/package.py b/var/spack/repos/builtin/packages/fish/package.py index 12255587057..b5a4a2d209e 100644 --- a/var/spack/repos/builtin/packages/fish/package.py +++ b/var/spack/repos/builtin/packages/fish/package.py @@ -7,7 +7,8 @@ class Fish(Package): homepage = "http://fishshell.com/" url = "http://fishshell.com/files/2.2.0/fish-2.2.0.tar.gz" - list_url = homepage + list_url = "http://fishshell.com/files/" + list_depth = 2 version('2.2.0', 'a76339fd14ce2ec229283c53e805faac48c3e99d9e3ede9d82c0554acfc7b77a') From 0e52c30bb8fbe954bdb2f74353865317cac18b85 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Thu, 28 Jan 2016 10:42:46 -0800 Subject: [PATCH 391/910] added py-wheel package --- .../repos/builtin/packages/py-wheel/package.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-wheel/package.py diff --git a/var/spack/repos/builtin/packages/py-wheel/package.py b/var/spack/repos/builtin/packages/py-wheel/package.py new file mode 100644 index 00000000000..3118e745199 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-wheel/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyWheel(Package): + """A built-package format for Python.""" + + homepage = "https://pypi.python.org/pypi/wheel" + url = "https://pypi.python.org/packages/source/w/wheel/wheel-0.26.0.tar.gz" + + version('0.26.0', '4cfc6e7e3dc7377d0164914623922a10') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 7e65f4da82cb9176247c1aa9ab72f40116cc6a8e Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Thu, 28 Jan 2016 13:22:56 -0800 Subject: [PATCH 392/910] Add the Caliper package --- .../repos/builtin/packages/caliper/package.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 var/spack/repos/builtin/packages/caliper/package.py diff --git a/var/spack/repos/builtin/packages/caliper/package.py b/var/spack/repos/builtin/packages/caliper/package.py new file mode 100644 index 00000000000..b14a562aa8e --- /dev/null +++ b/var/spack/repos/builtin/packages/caliper/package.py @@ -0,0 +1,25 @@ +from spack import * + +class Caliper(Package): + """ + Caliper is a generic context annotation system. It gives programmers the + ability to provide arbitrary program context information to (performance) + tools at runtime. + """ + + homepage = "https://github.com/LLNL/Caliper" + url = "" + + version('master', git='ssh://git@cz-stash.llnl.gov:7999/piper/caliper.git') + + variant('mpi', default=False, description='Enable MPI function wrappers.') + + depends_on('libunwind') + depends_on('papi') + depends_on('mpi', when='+mpi') + + def install(self, spec, prefix): + with working_dir('build', create=True): + cmake('..', *std_cmake_args) + make() + make("install") From 4f340315341a125bd01d0c844efc83454e531ca5 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 28 Jan 2016 15:41:58 -0600 Subject: [PATCH 393/910] Add PnetCDF and M4 packages --- .../repos/builtin/packages/m4/package.py | 13 ++++++++++++ .../packages/parallel-netcdf/package.py | 20 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 var/spack/repos/builtin/packages/m4/package.py create mode 100644 var/spack/repos/builtin/packages/parallel-netcdf/package.py diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py new file mode 100644 index 00000000000..5d76d8866bc --- /dev/null +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -0,0 +1,13 @@ +from spack import * + +class M4(Package): + """GNU M4 is an implementation of the traditional Unix macro processor.""" + homepage = "https://www.gnu.org/software/m4/m4.html" + url = "ftp://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz" + + version('1.4.17', 'a5e9954b1dae036762f7b13673a2cf76') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/parallel-netcdf/package.py b/var/spack/repos/builtin/packages/parallel-netcdf/package.py new file mode 100644 index 00000000000..62a8f7ca0ba --- /dev/null +++ b/var/spack/repos/builtin/packages/parallel-netcdf/package.py @@ -0,0 +1,20 @@ +from spack import * + +class ParallelNetcdf(Package): + """Parallel netCDF (PnetCDF) is a library providing high-performance + parallel I/O while still maintaining file-format compatibility with + Unidata's NetCDF.""" + + homepage = "https://trac.mcs.anl.gov/projects/parallel-netcdf" + url = "http://cucis.ece.northwestern.edu/projects/PnetCDF/Release/parallel-netcdf-1.6.1.tar.gz" + + version('1.6.1', '62a094eb952f9d1e15f07d56e535052604f1ac34') + + depends_on("m4") + depends_on("mpi") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--with-mpi=%s" % spec['mpi'].prefix) + make() + make("install") From d9548c01afe555ee58019b2ef5cb8938cd71cee3 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Thu, 28 Jan 2016 15:47:37 -0700 Subject: [PATCH 394/910] Correct package URL --- var/spack/repos/builtin/packages/caliper/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/caliper/package.py b/var/spack/repos/builtin/packages/caliper/package.py index b14a562aa8e..d51b4a4dd57 100644 --- a/var/spack/repos/builtin/packages/caliper/package.py +++ b/var/spack/repos/builtin/packages/caliper/package.py @@ -10,7 +10,7 @@ class Caliper(Package): homepage = "https://github.com/LLNL/Caliper" url = "" - version('master', git='ssh://git@cz-stash.llnl.gov:7999/piper/caliper.git') + version('master', git='ssh://git@github.com:LLNL/Caliper.git') variant('mpi', default=False, description='Enable MPI function wrappers.') From 2bea7f8d69064f893a21cfc53f3d781b572e2f79 Mon Sep 17 00:00:00 2001 From: Abhishek Kulkarni Date: Fri, 29 Jan 2016 00:29:20 -0500 Subject: [PATCH 395/910] Add the HPX-5 package. --- .../repos/builtin/packages/hpx/package.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 var/spack/repos/builtin/packages/hpx/package.py diff --git a/var/spack/repos/builtin/packages/hpx/package.py b/var/spack/repos/builtin/packages/hpx/package.py new file mode 100644 index 00000000000..665d5c74073 --- /dev/null +++ b/var/spack/repos/builtin/packages/hpx/package.py @@ -0,0 +1,27 @@ +from spack import * +import os + +class Hpx(Package): + """The HPX-5 Runtime System. HPX-5 (High Performance ParalleX) is an + open source, portable, performance-oriented runtime developed at + CREST (Indiana University). HPX-5 provides a distributed + programming model allowing programs to run unmodified on systems + from a single SMP to large clusters and supercomputers with + thousands of nodes. HPX-5 supports a wide variety of Intel and ARM + platforms. It is being used by a broad range of scientific + applications enabling scientists to write code that performs and + scales better than contemporary runtimes.""" + homepage = "http://hpx.crest.iu.edu" + url = "http://hpx.crest.iu.edu/release/hpx-2.0.0.tar.gz" + + version('2.0.0', '3d2ff3aab6c46481f9ec65c5b2bfe7a6') + version('1.3.0', '2260ecc7f850e71a4d365a43017d8cee') + version('1.2.0', '4972005f85566af4afe8b71afbf1480f') + version('1.1.0', '646afb460ecb7e0eea713a634933ce4f') + version('1.0.0', '8020822adf6090bd59ed7fe465f6c6cb') + + def install(self, spec, prefix): + os.chdir("./hpx/") + configure('--prefix=%s' % prefix) + make() + make("install") From b574c4ad5190bad375b9b8b38857222d5e79682a Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 29 Jan 2016 08:57:07 +0100 Subject: [PATCH 396/910] trilinos : updated package to meet changes in dependencies --- .../repos/builtin/packages/netcdf/package.py | 7 +++- .../builtin/packages/trilinos/package.py | 36 +++++++++++-------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 3cd0b2ee7a8..579282e7f76 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -11,13 +11,15 @@ class Netcdf(Package): version('4.4.0', 'f01cb26a0126dd9a6224e76472d25f6c') version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') + variant('mpi', default=True, description='Enables MPI parallelism') variant('fortran', default=False, description="Download and install NetCDF-Fortran") variant('hdf4', default=False, description="Enable HDF4 support") # Dependencies: depends_on("curl") # required for DAP support depends_on("hdf", when='+hdf4') - depends_on("hdf5") # required for NetCDF-4 support + depends_on("hdf5+mpi~cxx", when='+mpi') # required for NetCDF-4 support + depends_on("hdf5~mpi", when='~mpi') # required for NetCDF-4 support depends_on("zlib") # required for NetCDF-4 support def install(self, spec, prefix): @@ -41,6 +43,9 @@ def install(self, spec, prefix): "--enable-dap" ] + if '+mpi' in spec: + config_args.append('--enable-parallel') + CPPFLAGS.append("-I%s/include" % spec['hdf5'].prefix) LDFLAGS.append( "-L%s/lib" % spec['hdf5'].prefix) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 7c43f796a4e..e6b97022d92 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -10,6 +10,7 @@ class Trilinos(Package): homepage = "https://trilinos.org/" url = "http://trilinos.csbsju.edu/download/files/trilinos-12.2.1-Source.tar.gz" + version('12.4.2', '7c830f7f0f68b8ad324690603baf404e') version('12.2.1', '6161926ea247863c690e927687f83be9') version('12.0.1', 'bd99741d047471e127b8296b2ec08017') version('11.14.3', '2f4f83f8333e4233c57d0f01c4b57426') @@ -17,33 +18,38 @@ class Trilinos(Package): version('11.14.1', '40febc57f76668be8b6a77b7607bb67f') variant('mpi', default=True, description='Add a dependency on MPI and enables MPI dependent packages') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Builds a debug version of the libraries') # Everything should be compiled with -fpic depends_on('blas') depends_on('lapack') depends_on('boost') - depends_on('netcdf') depends_on('matio') depends_on('glm') depends_on('swig') + + # MPI related dependencies depends_on('mpi', when='+mpi') + depends_on('netcdf+mpi', when='+mpi') + depends_on('netcdf~mpi', when='~mpi') + + depends_on('python') # Needs py-numpy activated def install(self, spec, prefix): - - options = [ - '-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', - '-DTrilinos_ENABLE_TESTS:BOOL=OFF', - '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', - '-DBUILD_SHARED_LIBS:BOOL=ON', - '-DBLAS_LIBRARY_DIRS:PATH=%s' % spec['blas'].prefix, - '-DLAPACK_LIBRARY_DIRS:PATH=%s' % spec['lapack'].prefix - ] - if '+mpi' in spec: - mpi_options = ['-DTPL_ENABLE_MPI:BOOL=ON'] - options.extend(mpi_options) - - # -DCMAKE_INSTALL_PREFIX and all the likes... + options = [] options.extend(std_cmake_args) + + options.extend(['-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', + '-DTrilinos_ENABLE_TESTS:BOOL=OFF', + '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', + '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), + '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), + '-DTPL_ENABLE_MPI:BOOL=%s' % ('ON' if '+mpi' in spec else 'OFF'), + '-DBLAS_LIBRARY_DIRS:PATH=%s' % spec['blas'].prefix, + '-DLAPACK_LIBRARY_DIRS:PATH=%s' % spec['lapack'].prefix + ]) + with working_dir('spack-build', create=True): cmake('..', *options) make() From cd547939138c00a45a2252f073ce21d3742d5dd0 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Fri, 29 Jan 2016 11:42:06 +0100 Subject: [PATCH 397/910] Cleaning up --- var/spack/repos/builtin/packages/openssl/package.py | 7 ++++--- var/spack/repos/builtin/packages/trilinos/package.py | 8 +++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index bbb169ec6bc..3b790eca66b 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -9,9 +9,10 @@ class Openssl(Package): homepage = "http://www.openssl.org" url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz" - version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') - version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') - version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') + version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') # Not available + version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') # Not available + version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') # Not available + version('1.0.2f', 'b3bf73f507172be9292ea2a8c28b659d') depends_on("zlib") parallel = False diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index e6b97022d92..edc40476e30 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -17,7 +17,6 @@ class Trilinos(Package): version('11.14.2', 'a43590cf896c677890d75bfe75bc6254') version('11.14.1', '40febc57f76668be8b6a77b7607bb67f') - variant('mpi', default=True, description='Add a dependency on MPI and enables MPI dependent packages') variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds a debug version of the libraries') @@ -30,9 +29,8 @@ class Trilinos(Package): depends_on('swig') # MPI related dependencies - depends_on('mpi', when='+mpi') - depends_on('netcdf+mpi', when='+mpi') - depends_on('netcdf~mpi', when='~mpi') + depends_on('mpi') + depends_on('netcdf+mpi') depends_on('python') # Needs py-numpy activated @@ -45,7 +43,7 @@ def install(self, spec, prefix): '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), - '-DTPL_ENABLE_MPI:BOOL=%s' % ('ON' if '+mpi' in spec else 'OFF'), + '-DTPL_ENABLE_MPI:STRING=ON', '-DBLAS_LIBRARY_DIRS:PATH=%s' % spec['blas'].prefix, '-DLAPACK_LIBRARY_DIRS:PATH=%s' % spec['lapack'].prefix ]) From 74225544c6ae27e3ae07d37fe15935364c665f6e Mon Sep 17 00:00:00 2001 From: Abhishek Kulkarni Date: Fri, 29 Jan 2016 10:22:52 -0500 Subject: [PATCH 398/910] Rename the hpx package to hpx-5. --- .../builtin/packages/{hpx => hpx5}/package.py | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) rename var/spack/repos/builtin/packages/{hpx => hpx5}/package.py (57%) diff --git a/var/spack/repos/builtin/packages/hpx/package.py b/var/spack/repos/builtin/packages/hpx5/package.py similarity index 57% rename from var/spack/repos/builtin/packages/hpx/package.py rename to var/spack/repos/builtin/packages/hpx5/package.py index 665d5c74073..3dae3c4170c 100644 --- a/var/spack/repos/builtin/packages/hpx/package.py +++ b/var/spack/repos/builtin/packages/hpx5/package.py @@ -1,7 +1,7 @@ from spack import * import os -class Hpx(Package): +class Hpx5(Package): """The HPX-5 Runtime System. HPX-5 (High Performance ParalleX) is an open source, portable, performance-oriented runtime developed at CREST (Indiana University). HPX-5 provides a distributed @@ -20,8 +20,33 @@ class Hpx(Package): version('1.1.0', '646afb460ecb7e0eea713a634933ce4f') version('1.0.0', '8020822adf6090bd59ed7fe465f6c6cb') + variant('debug', default=False, description='Build a debug version of HPX-5') + variant('photon', default=False, description='Enable Photon support') + variant('mpi', default=False, description='Enable MPI support') + + depends_on("mpi", when='+mpi') + depends_on("mpi", when='+photon') + def install(self, spec, prefix): + extra_args = [] + if '+debug' in spec: + extra_args.extend([ + '--enable-debug', + 'CFLAGS=-g -O0' + ]) + else: + extra_args.append('CFLAGS=-O3') + + if '+mpi' in spec: + extra_args.append('--enable-mpi') + + if '+photon' in spec: + extra_args.extend([ + '--enable-mpi', + '--enable-photon' + ]) + os.chdir("./hpx/") - configure('--prefix=%s' % prefix) + configure('--prefix=%s' % prefix, *extra_args) make() make("install") From d14d50beb6dbb71c75a53902e70601c353496e09 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 29 Jan 2016 09:47:57 -0600 Subject: [PATCH 399/910] Checksum fix for hwloc --- var/spack/repos/builtin/packages/hwloc/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/hwloc/package.py b/var/spack/repos/builtin/packages/hwloc/package.py index 60b315119ba..ab7205646ee 100644 --- a/var/spack/repos/builtin/packages/hwloc/package.py +++ b/var/spack/repos/builtin/packages/hwloc/package.py @@ -17,8 +17,8 @@ class Hwloc(Package): list_url = "http://www.open-mpi.org/software/hwloc/" list_depth = 3 - version('1.11.2', '486169cbe111cdea57be12638828ebbf') - version('1.11.1', '002742efd3a8431f98d6315365a2b543') + version('1.11.2', 'e4ca55c2a5c5656da4a4e37c8fc51b23') + version('1.11.1', 'feb4e416a1b25963ed565d8b42252fdc') version('1.9', '1f9f9155682fe8946a97c08896109508') depends_on('libpciaccess') From 3bf6fed7b3536d3b38a9855c2129956b24a77529 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Fri, 29 Jan 2016 11:29:27 -0800 Subject: [PATCH 400/910] updated openssl version --- var/spack/repos/builtin/packages/openssl/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index bbb169ec6bc..05de35fca06 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -12,6 +12,7 @@ class Openssl(Package): version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') + version('1.0.2f', 'b3bf73f507172be9292ea2a8c28b659d') depends_on("zlib") parallel = False From 360abb070a639711f740aa7ef73a84ab76aff14f Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 1 Feb 2016 08:33:03 +0100 Subject: [PATCH 401/910] netcdf : fixed typo in configure option openssl : smarter URL computation --- .../repos/builtin/packages/netcdf/package.py | 3 +- .../repos/builtin/packages/openssl/package.py | 28 +++++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 579282e7f76..89b40f4a90d 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -1,5 +1,6 @@ from spack import * + class Netcdf(Package): """NetCDF is a set of software libraries and self-describing, machine-independent data formats that support the creation, access, and sharing of array-oriented @@ -44,7 +45,7 @@ def install(self, spec, prefix): ] if '+mpi' in spec: - config_args.append('--enable-parallel') + config_args.append('--enable-parallel4') CPPFLAGS.append("-I%s/include" % spec['hdf5'].prefix) LDFLAGS.append( "-L%s/lib" % spec['hdf5'].prefix) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 3b790eca66b..345fce0669e 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -1,3 +1,5 @@ +import urllib + from spack import * class Openssl(Package): @@ -9,9 +11,10 @@ class Openssl(Package): homepage = "http://www.openssl.org" url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz" - version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') # Not available - version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') # Not available - version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') # Not available + version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') + version('1.0.1r', '1abd905e079542ccae948af37e393d28') + version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') + version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') version('1.0.2f', 'b3bf73f507172be9292ea2a8c28b659d') depends_on("zlib") @@ -39,3 +42,22 @@ def install(self, spec, prefix): make() make("install") + + def url_for_version(self, version): + # This URL is computed pinging the place where the latest version is stored. To avoid slowdown + # due to repeated pinging, we store the URL in a private attribute + openssl_url = getattr(self, '_openssl_url', None) + + if openssl_url is None: + latest = 'http://www.openssl.org/source/openssl-{version}.tar.gz' + older = 'http://www.openssl.org/source/old/{version_number}/openssl-{version_full}.tar.gz' + # Try to use the url where the latest tarballs are stored. If the url does not exist (404), then + # return the url for older format + version_number = '.'.join([str(x) for x in version[:-1]]) + older_url = older.format(version_number=version_number, version_full=version) + latest_url = latest.format(version=version) + response = urllib.urlopen(latest.format(version=version)) + openssl_url = older_url if response.getcode() == 404 else latest_url + self._openssl_url = openssl_url + + return openssl_url From e67507478a9f662d7adfa891bcb2c7b830b69f5c Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 1 Feb 2016 10:55:39 +0100 Subject: [PATCH 402/910] opencv : reverted url_for_version modifications --- .../repos/builtin/packages/openssl/package.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 345fce0669e..36a33204fd8 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -42,22 +42,3 @@ def install(self, spec, prefix): make() make("install") - - def url_for_version(self, version): - # This URL is computed pinging the place where the latest version is stored. To avoid slowdown - # due to repeated pinging, we store the URL in a private attribute - openssl_url = getattr(self, '_openssl_url', None) - - if openssl_url is None: - latest = 'http://www.openssl.org/source/openssl-{version}.tar.gz' - older = 'http://www.openssl.org/source/old/{version_number}/openssl-{version_full}.tar.gz' - # Try to use the url where the latest tarballs are stored. If the url does not exist (404), then - # return the url for older format - version_number = '.'.join([str(x) for x in version[:-1]]) - older_url = older.format(version_number=version_number, version_full=version) - latest_url = latest.format(version=version) - response = urllib.urlopen(latest.format(version=version)) - openssl_url = older_url if response.getcode() == 404 else latest_url - self._openssl_url = openssl_url - - return openssl_url From c8d2275c06f05b8ddf2463c5aa110d1f44692038 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 1 Feb 2016 10:56:48 +0100 Subject: [PATCH 403/910] opencv : reverted url_for_version modifications --- var/spack/repos/builtin/packages/openssl/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 36a33204fd8..a225e30f6c2 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -1,5 +1,3 @@ -import urllib - from spack import * class Openssl(Package): From 2b140b9a34852cbada6a3c1045b3b021d5eabeb8 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 1 Feb 2016 11:46:33 -0500 Subject: [PATCH 404/910] hdf5: default +unsupported to on This flag can end up being required if a dependency tree ends up needing hdf5+cxx+parallel, but nothing turns it on. Since the core spack code puts this together, it ends up that the end user needs to specify it manually. Instead, just assume an unsupported configuration (since these *are* useful setups) and if anyone wants to seek upstream support, masking the flag can be done. --- var/spack/repos/builtin/packages/hdf5/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 5321a191f0e..80f79539c09 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -45,7 +45,7 @@ class Hdf5(Package): variant('cxx', default=True, description='Enable C++ support') variant('fortran', default=True, description='Enable Fortran support') - variant('unsupported', default=False, description='Enables unsupported configuration options') + variant('unsupported', default=True, description='Enables unsupported configuration options') variant('mpi', default=False, description='Enable MPI support') variant('szip', default=False, description='Enable szip support') From 422e87badbd073a9ca19e8c2e781a945d3a8e511 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 2 Feb 2016 10:57:58 -0600 Subject: [PATCH 405/910] Add latest libgpg-error version --- var/spack/repos/builtin/packages/libgpg-error/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/libgpg-error/package.py b/var/spack/repos/builtin/packages/libgpg-error/package.py index 6c1d1a10a7e..dd5fc2408a9 100644 --- a/var/spack/repos/builtin/packages/libgpg-error/package.py +++ b/var/spack/repos/builtin/packages/libgpg-error/package.py @@ -9,6 +9,7 @@ class LibgpgError(Package): homepage = "https://www.gnupg.org/related_software/libgpg-error" url = "ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.18.tar.bz2" + version('1.21', 'ab0b5aba6d0a185b41d07bda804fd8b2') version('1.18', '12312802d2065774b787cbfc22cc04e9') def install(self, spec, prefix): From ab3698e3a45e96b8cdd8d99220011350fa0489d1 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 2 Feb 2016 11:42:31 -0600 Subject: [PATCH 406/910] Add latest qhull version --- var/spack/repos/builtin/packages/qhull/package.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/qhull/package.py b/var/spack/repos/builtin/packages/qhull/package.py index f6712ced38c..bdca6db15dd 100644 --- a/var/spack/repos/builtin/packages/qhull/package.py +++ b/var/spack/repos/builtin/packages/qhull/package.py @@ -8,20 +8,18 @@ class Qhull(Package): implements the Quickhull algorithm for computing the convex hull. It handles roundoff errors from floating point arithmetic. It computes volumes, surface areas, and - approximations to the convex hull. - - Qhull does not support triangulation of non-convex surfaces, - mesh generation of non-convex objects, medium-sized inputs in - 9-D and higher, alpha shapes, weighted Voronoi diagrams, - Voronoi volumes, or constrained Delaunay triangulations.""" + approximations to the convex hull.""" homepage = "http://www.qhull.org" + version('7.2.0', 'e6270733a826a6a7c32b796e005ec3dc', + url="http://www.qhull.org/download/qhull-2015-src-7.2.0.tgz") + version('1.0', 'd0f978c0d8dfb2e919caefa56ea2953c', url="http://www.qhull.org/download/qhull-2012.1-src.tgz") # https://github.com/qhull/qhull/pull/5 - patch('qhull-iterator.patch') + patch('qhull-iterator.patch', when='@1.0') def install(self, spec, prefix): with working_dir('spack-build', create=True): From d2fb1522a3524315a036cd5d46a99b2e05f1ad62 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 2 Feb 2016 12:54:24 -0600 Subject: [PATCH 407/910] Add Phonopy and PyYAML Python modules --- .../builtin/packages/py-phonopy/package.py | 18 ++++++++++++++++++ .../builtin/packages/py-pyyaml/package.py | 13 +++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-phonopy/package.py create mode 100644 var/spack/repos/builtin/packages/py-pyyaml/package.py diff --git a/var/spack/repos/builtin/packages/py-phonopy/package.py b/var/spack/repos/builtin/packages/py-phonopy/package.py new file mode 100644 index 00000000000..6d10fea74ff --- /dev/null +++ b/var/spack/repos/builtin/packages/py-phonopy/package.py @@ -0,0 +1,18 @@ +from spack import * + +class PyPhonopy(Package): + """Phonopy is an open source package for phonon + calculations at harmonic and quasi-harmonic levels.""" + homepage = "http://atztogo.github.io/phonopy/index.html" + url = "http://sourceforge.net/projects/phonopy/files/phonopy/phonopy-1.10/phonopy-1.10.0.tar.gz" + + version('1.10.0', '973ed1bcea46e21b9bf747aab9061ff6') + + extends('python') + depends_on('py-numpy') + depends_on('py-scipy') + depends_on('py-matplotlib') + depends_on('py-pyyaml') + + def install(self, spec, prefix): + python('setup.py', 'install', '--home=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pyyaml/package.py b/var/spack/repos/builtin/packages/py-pyyaml/package.py new file mode 100644 index 00000000000..cae42f6e591 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pyyaml/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyPyyaml(Package): + """PyYAML is a YAML parser and emitter for Python.""" + homepage = "http://pyyaml.org/wiki/PyYAML" + url = "http://pyyaml.org/download/pyyaml/PyYAML-3.11.tar.gz" + + version('3.11', 'f50e08ef0fe55178479d3a618efe21db') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 587d356d6ec8820cf04043b3650ac66b82a651b2 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 3 Feb 2016 08:58:34 +0100 Subject: [PATCH 408/910] openssl : added logic to version computation. The package now warns user if he depends on an old version of the library --- .../repos/builtin/packages/openssl/package.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 05de35fca06..c73102f05d3 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -1,3 +1,6 @@ +import urllib +import llnl.util.tty as tty + from spack import * class Openssl(Package): @@ -10,6 +13,7 @@ class Openssl(Package): url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz" version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') + version('1.0.1r', '1abd905e079542ccae948af37e393d28') version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') version('1.0.2f', 'b3bf73f507172be9292ea2a8c28b659d') @@ -17,6 +21,42 @@ class Openssl(Package): depends_on("zlib") parallel = False + def url_for_version(self, version): + # This URL is computed pinging the place where the latest version is stored. To avoid slowdown + # due to repeated pinging, we store the URL in a private class attribute to do the job only once per version + openssl_urls = getattr(Openssl, '_openssl_url', {}) + openssl_url = openssl_urls.get(version, None) + # Same idea, but just to avoid issuing the same message multiple times + warnings_given_to_user = getattr(Openssl, '_warnings_given', {}) + if openssl_url is None: + latest = 'http://www.openssl.org/source/openssl-{version}.tar.gz' + older = 'http://www.openssl.org/source/old/{version_number}/openssl-{version_full}.tar.gz' + # Try to use the url where the latest tarballs are stored. If the url does not exist (404), then + # return the url for older format + version_number = '.'.join([str(x) for x in version[:-1]]) + older_url = older.format(version_number=version_number, version_full=version) + latest_url = latest.format(version=version) + response = urllib.urlopen(latest.format(version=version)) + if response.getcode() == 404: + openssl_url = older_url + # Checks if we already warned the user for this particular version of OpenSSL. + # If not we display a warning message and mark this version + if not warnings_given_to_user.get(version, False): + tty.warn('This installation depends on an old version of OpenSSL, which may have known security issues. ') + tty.warn('Consider updating to the latest version of this package.') + tty.warn('More details at {homepage}'.format(homepage=Openssl.homepage)) + warnings_given_to_user[version] = True + else: + openssl_url = latest_url + # Store the computed URL + openssl_urls[version] = openssl_url + # Store the updated dictionary of URLS + Openssl._openssl_url = openssl_urls + # Store the updated dictionary of warnings + Openssl._warnings_given = warnings_given_to_user + + return openssl_url + def install(self, spec, prefix): # OpenSSL uses a variable APPS in its Makefile. If it happens to be set # in the environment, then this will override what is set in the From 44c1b06609925307e0e522b10daecf1dbe2eddc4 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 1 Feb 2016 10:53:27 +0100 Subject: [PATCH 409/910] opencv : added package --- .../repos/builtin/packages/opencv/package.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 var/spack/repos/builtin/packages/opencv/package.py diff --git a/var/spack/repos/builtin/packages/opencv/package.py b/var/spack/repos/builtin/packages/opencv/package.py new file mode 100644 index 00000000000..99b555323f6 --- /dev/null +++ b/var/spack/repos/builtin/packages/opencv/package.py @@ -0,0 +1,50 @@ +from spack import * + + +class Opencv(Package): + """ + OpenCV is released under a BSD license and hence it's free for both academic and commercial use. It has C++, C, + Python and Java interfaces and supports Windows, Linux, Mac OS, iOS and Android. OpenCV was designed for + computational efficiency and with a strong focus on real-time applications. Written in optimized C/C++, the library + can take advantage of multi-core processing. Enabled with OpenCL, it can take advantage of the hardware + acceleration of the underlying heterogeneous compute platform. Adopted all around the world, OpenCV has more than + 47 thousand people of user community and estimated number of downloads exceeding 9 million. Usage ranges from + interactive art, to mines inspection, stitching maps on the web or through advanced robotics. + """ + homepage = 'http://opencv.org/' + url = 'https://github.com/Itseez/opencv/archive/3.1.0.tar.gz' + + version('3.1.0', '70e1dd07f0aa06606f1bc0e3fa15abd3') + + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Builds a debug version of the libraries') + + variant('eigen', default=True, description='Activates support for eigen') + variant('ipp', default=True, description='Activates support for IPP') + + depends_on('zlib') + depends_on('libpng') + depends_on('libjpeg-turbo') + depends_on('libtiff') + + depends_on('python') + depends_on('py-numpy') + + depends_on('eigen', when='+eigen') + + # FIXME : GUI extensions missing + # FIXME : CUDA extensions still missing + + def install(self, spec, prefix): + cmake_options = [] + cmake_options.extend(std_cmake_args) + + cmake_options.extend(['-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), + '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), + '-DENABLE_PRECOMPILED_HEADERS:BOOL=OFF', + '-DWITH_IPP:BOOL=%s' % ('ON' if '+ipp' in spec else 'OFF')]) + + with working_dir('spack_build', create=True): + cmake('..', *cmake_options) + make('VERBOSE=1') + make("install") From b6434d9581b8beb2457504c008dca5a035cce182 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 31 Jan 2016 12:09:43 -0800 Subject: [PATCH 410/910] Update mailmap --- .mailmap | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.mailmap b/.mailmap index 1b99da32b5b..39ec1832413 100644 --- a/.mailmap +++ b/.mailmap @@ -1,13 +1,20 @@ Todd Gamblin George Todd Gamblin +Todd Gamblin Todd Gamblin Adam Moody Adam T. Moody Alfredo Gimenez Alfredo Gimenez David Boehme David Boehme David Boehme David Boehme Kevin Brandstatter Kevin Brandstatter Luc Jaulmes Luc Jaulmes -Saravan Pantham Saravan Pantham Saravan Pantham Tom Scogland Tom Scogland Tom Scogland Tom Scogland Joachim Protze jprotze Gregory L. Lee Gregory L. Lee +Gregory L. Lee Gregory L. Lee +Gregory L. Lee Gregory L. Lee +Gregory L. Lee Gregory L. Lee Gregory L. Lee Gregory Lee +Massimiliano Culpo Massimiliano Culpo +Massimiliano Culpo alalazo +Mark Miller miller86 From e1b3c286ef3817c63b9566d6b64ea330fc739903 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 3 Feb 2016 16:28:09 -0600 Subject: [PATCH 411/910] Add url for cmake, simplify url_for_version --- var/spack/repos/builtin/packages/cmake/package.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index 408a90460b9..e20c1e4aeb0 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -23,13 +23,14 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import llnl.util.tty as tty class Cmake(Package): """A cross-platform, open-source build system. CMake is a family of tools designed to build, test and package software.""" homepage = 'https://www.cmake.org' + url = 'https://cmake.org/files/v3.4/cmake-3.4.3.tar.gz' + version('3.4.3', '4cb3ff35b2472aae70f542116d616e63') version('3.4.0', 'cd3034e0a44256a0917e254167217fc8') version('3.3.1', '52638576f4e1e621fed6c3410d3a1b12') version('3.0.2', 'db4c687a31444a929d2fdc36c4dfb95f') @@ -40,12 +41,7 @@ class Cmake(Package): def url_for_version(self, version): """Handle CMake's version-based custom URLs.""" - parts = [str(p) for p in Version(version)] - if len(parts) < 3: - tty.error("Version '%s'does not match CMake's version naming scheme (z.y.x)." % version) - version_short = ".".join(parts[:2]) - version_full = ".".join(parts) - return "http://www.cmake.org/files/v%s/cmake-%s.tar.gz" % (version_short,version_full) + return 'https://cmake.org/files/v%s/cmake-%s.tar.gz' % (version.up_to(2), version) def install(self, spec, prefix): From 031c292136d18889e5cd6e8facb68bf2c50ec083 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 4 Feb 2016 11:19:25 +0100 Subject: [PATCH 412/910] dakota : basic installation --- .../repos/builtin/packages/dakota/package.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 var/spack/repos/builtin/packages/dakota/package.py diff --git a/var/spack/repos/builtin/packages/dakota/package.py b/var/spack/repos/builtin/packages/dakota/package.py new file mode 100644 index 00000000000..5ec82ef83d0 --- /dev/null +++ b/var/spack/repos/builtin/packages/dakota/package.py @@ -0,0 +1,55 @@ +from spack import * + + +class Dakota(Package): + """ + The Dakota toolkit provides a flexible, extensible interface between analysis codes and iterative systems + analysis methods. Dakota contains algorithms for: + + - optimization with gradient and non gradient-based methods; + - uncertainty quantification with sampling, reliability, stochastic expansion, and epistemic methods; + - parameter estimation with nonlinear least squares methods; + - sensitivity/variance analysis with design of experiments and parameter study methods. + + These capabilities may be used on their own or as components within advanced strategies such as hybrid optimization, + surrogate-based optimization, mixed integer nonlinear programming, or optimization under uncertainty. + """ + + homepage = 'https://dakota.sandia.gov/' + url = 'https://dakota.sandia.gov/sites/default/files/distributions/public/dakota-6.3-public.src.tar.gz' + _url_str = 'https://dakota.sandia.gov/sites/default/files/distributions/public/dakota-{version}-public.src.tar.gz' + + version('6.3', '05a58d209fae604af234c894c3f73f6d') + + variant('debug', default=False, description='Builds a debug version of the libraries') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('mpi', default=True, description='Activates MPI support') + + depends_on('blas') + depends_on('lapack') + depends_on('mpi', when='+mpi') + + depends_on('python') + depends_on('boost') + + def url_for_version(self, version): + return Dakota._url_str.format(version=version) + + def install(self, spec, prefix): + options = [] + options.extend(std_cmake_args) + + options.extend(['-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), + '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF')]) + + if '+mpi' in spec: + options.extend(['-DDAKOTA_HAVE_MPI:BOOL=ON', + '-DMPI_CXX_COMPILER:STRING=%s' % join_path(spec['mpi'].prefix.bin, 'mpicxx')]) + + build_directory = join_path(self.stage.path, 'spack-build') + source_directory = self.stage.source_path + + with working_dir(build_directory, create=True): + cmake(source_directory, *options) + make() + make("install") From e3a0e1881dce3b251608d4252c7aafaa2da34104 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 4 Feb 2016 16:00:39 +0100 Subject: [PATCH 413/910] suitesparse : basic implementation --- .../builtin/packages/SuiteSparse/package.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 var/spack/repos/builtin/packages/SuiteSparse/package.py diff --git a/var/spack/repos/builtin/packages/SuiteSparse/package.py b/var/spack/repos/builtin/packages/SuiteSparse/package.py new file mode 100644 index 00000000000..6e130d118fe --- /dev/null +++ b/var/spack/repos/builtin/packages/SuiteSparse/package.py @@ -0,0 +1,27 @@ +from spack import * + + +class Suitesparse(Package): + """ + SuiteSparse is a suite of sparse matrix algorithms + """ + homepage = 'http://faculty.cse.tamu.edu/davis/suitesparse.html' + url = 'http://faculty.cse.tamu.edu/davis/SuiteSparse/SuiteSparse-4.5.1.tar.gz' + + version('4.5.1', 'f0ea9aad8d2d1ffec66a5b6bfeff5319') + + depends_on('blas') + depends_on('lapack') + + depends_on('metis@5.1.0', when='@4.5.1') + + def install(self, spec, prefix): + # The build system of SuiteSparse is quite old-fashioned + # It's basically a plain Makefile which include an header (SuiteSparse_config/SuiteSparse_config.mk) + # with a lot of convoluted logic in it. + # Any kind of customization will need to go through filtering of that file + + # FIXME : this actually uses the current workaround + # FIXME : (blas / lapack always provide libblas and liblapack as aliases) + make('install', 'INSTALL=%s' % prefix, 'BLAS=-lblas', 'LAPACK=-llapack') + From e2a95d4b5025a9965ab231e0c43fe9bebd88ca92 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 4 Feb 2016 16:27:26 +0100 Subject: [PATCH 414/910] eigen : added support for SuiteSparse --- var/spack/repos/builtin/packages/eigen/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/eigen/package.py b/var/spack/repos/builtin/packages/eigen/package.py index 44ee6819f58..e40046b4529 100644 --- a/var/spack/repos/builtin/packages/eigen/package.py +++ b/var/spack/repos/builtin/packages/eigen/package.py @@ -41,13 +41,14 @@ class Eigen(Package): variant('metis', default=True, description='Enables metis backend') variant('scotch', default=True, description='Enables scotch backend') variant('fftw', default=True, description='Enables FFTW backend') + variant('suitesparse', default=True, description='Enables SuiteSparse support') - # TODO : dependency on SuiteSparse, googlehash, superlu, adolc missing + # TODO : dependency on googlehash, superlu, adolc missing depends_on('metis', when='+metis') depends_on('scotch', when='+scotch') depends_on('fftw', when='+fftw') - + depends_on('SuiteSparse', when='+suitesparse') depends_on('mpfr@2.3.0:') # Eigen 3.2.7 requires at least 2.3.0 depends_on('gmp') From 71a25a109a8c9f23ce9aab0743ebf4516020519d Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 5 Feb 2016 11:00:37 +0100 Subject: [PATCH 415/910] arpack-ng : added package --- .../builtin/packages/arpack-ng/package.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 var/spack/repos/builtin/packages/arpack-ng/package.py diff --git a/var/spack/repos/builtin/packages/arpack-ng/package.py b/var/spack/repos/builtin/packages/arpack-ng/package.py new file mode 100644 index 00000000000..0b49d142024 --- /dev/null +++ b/var/spack/repos/builtin/packages/arpack-ng/package.py @@ -0,0 +1,57 @@ +from spack import * + + +class ArpackNg(Package): + """ + ARPACK-NG is a collection of Fortran77 subroutines designed to solve large scale eigenvalue problems. + + Important Features: + + * Reverse Communication Interface. + * Single and Double Precision Real Arithmetic Versions for Symmetric, + Non-symmetric, Standard or Generalized Problems. + * Single and Double Precision Complex Arithmetic Versions for Standard or + Generalized Problems. + * Routines for Banded Matrices - Standard or Generalized Problems. + * Routines for The Singular Value Decomposition. + * Example driver routines that may be used as templates to implement numerous + Shift-Invert strategies for all problem types, data types and precision. + + This project is a joint project between Debian, Octave and Scilab in order to + provide a common and maintained version of arpack. + + Indeed, no single release has been published by Rice university for the last + few years and since many software (Octave, Scilab, R, Matlab...) forked it and + implemented their own modifications, arpack-ng aims to tackle this by providing + a common repository and maintained versions. + + arpack-ng is replacing arpack almost everywhere. + """ + homepage = 'https://github.com/opencollab/arpack-ng' + url = 'https://github.com/opencollab/arpack-ng/archive/3.3.0.tar.gz' + + version('3.3.0', 'ed3648a23f0a868a43ef44c97a21bad5') + + variant('shared', default=True, description='Enables the build of shared libraries') + variant('mpi', default=False, description='Activates MPI support') + + depends_on('blas') + depends_on('lapack') + depends_on('mpi', when='+mpi') + + def install(self, spec, prefix): + # Apparently autotools are not bootstrapped + bootstrap = Executable('./bootstrap') + + options = ['--prefix=%s' % prefix] + + if '+mpi' in spec: + options.append('--enable-mpi') + + if '~shared' in spec: + options.append('--enable-shared=no') + + bootstrap() + configure(*options) + make() + make('install') From f7134990bd9551d6361fb57db7095fa3bf44e092 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 9 Feb 2016 08:57:27 -0800 Subject: [PATCH 416/910] Fix #430: edit -f fails with UnknownPackageError - Recent external repo refactoring made `repo_for_pkg` raise an error when the package was not known. - Correct behavior is to return the highest precedence repo. --- lib/spack/spack/repository.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index b5df1168b66..f58cd52125b 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -300,8 +300,11 @@ def repo_for_pkg(self, spec): for repo in self.repos: if spec.name in repo: return repo - else: - raise UnknownPackageError(spec.name) + + # If the package isn't in any repo, return the one with + # highest precedence. This is for commands like `spack edit` + # that can operate on packages that don't exist yet. + return self.first_repo() @_autospec From 25e2be3efda2f70cf8d6c4e7b6089fbe9124c0d3 Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Wed, 10 Feb 2016 13:28:01 +0100 Subject: [PATCH 417/910] RPATH was collecting all lib from all dependencies, when dep tree is deep like in ParaView the path was growing too much, showing install problems now just direct dependencies rpath is added --- lib/spack/spack/build_environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index b2db83acb7b..1b877780801 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -237,9 +237,9 @@ def set_module_variables_for_package(pkg, m): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" rpaths = [pkg.prefix.lib, pkg.prefix.lib64] - rpaths.extend(d.prefix.lib for d in pkg.spec.traverse(root=False) + rpaths.extend(d.prefix.lib for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib)) - rpaths.extend(d.prefix.lib64 for d in pkg.spec.traverse(root=False) + rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) return rpaths From f729cf621ad463c953a81f4963b87535508583da Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Wed, 10 Feb 2016 13:29:36 +0100 Subject: [PATCH 418/910] fixing qhull: seems that dependency on cmake is essential when it is used --- var/spack/repos/builtin/packages/qhull/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/qhull/package.py b/var/spack/repos/builtin/packages/qhull/package.py index bdca6db15dd..8f7c2f31b12 100644 --- a/var/spack/repos/builtin/packages/qhull/package.py +++ b/var/spack/repos/builtin/packages/qhull/package.py @@ -20,6 +20,8 @@ class Qhull(Package): # https://github.com/qhull/qhull/pull/5 patch('qhull-iterator.patch', when='@1.0') + + depends_on('cmake') def install(self, spec, prefix): with working_dir('spack-build', create=True): From fe8aca630c65e1f0324bfebeca11ca1cb3d4183c Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Wed, 10 Feb 2016 14:58:00 +0100 Subject: [PATCH 419/910] add a variant to Qt for make dependency on mesa swlwctable --- var/spack/repos/builtin/packages/qt/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index e8d843519de..91afa420c17 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -23,6 +23,7 @@ class Qt(Package): version('3.3.8b', '9f05b4125cfe477cc52c9742c3c09009', url="http://download.qt.io/archive/qt/3/qt-x11-free-3.3.8b.tar.gz") + variant('mesa', default=False, description='depend on mesa') # Add patch for compile issues with qt3 found with use in the OpenSpeedShop project variant('krellpatch', default=False, description="build with openspeedshop based patch.") patch('qt3krell.patch', when='@3.3.8b+krellpatch') @@ -48,7 +49,7 @@ class Qt(Package): # depends_on("icu4c") # OpenGL hardware acceleration - depends_on("mesa", when='@4:') + depends_on("mesa", when='@4:+mesa') depends_on("libxcb") From a1bd65550e343c25ffcaf22aadf7a1875b08d853 Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Wed, 10 Feb 2016 15:17:54 +0100 Subject: [PATCH 420/910] fix a reported hash mismatch on netcdf-4.4.0 tar download --- var/spack/repos/builtin/packages/netcdf/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 89b40f4a90d..41a0d2b6f92 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -9,7 +9,7 @@ class Netcdf(Package): homepage = "http://www.unidata.ucar.edu/software/netcdf/" url = "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.3.3.tar.gz" - version('4.4.0', 'f01cb26a0126dd9a6224e76472d25f6c') + version('4.4.0', 'cffda0cbd97fdb3a06e9274f7aef438e') version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') variant('mpi', default=True, description='Enables MPI parallelism') From d7b3ed08ab15caa3aac95700d84bfb52d8e1ac2f Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Wed, 10 Feb 2016 15:40:05 +0100 Subject: [PATCH 421/910] add variant to select OpenGL2 Paraview backend --- var/spack/repos/builtin/packages/paraview/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index aaab352e662..e43bdd44930 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -14,6 +14,7 @@ class Paraview(Package): variant('osmesa', default=False, description='Enable OSMesa support') variant('qt', default=False, description='Enable Qt support') + variant('opengl2', default=False, description='Enable OPengl2 backend') depends_on('python', when='+python') depends_on('py-numpy', when='+python') From 157ec210a74c2897abc093c14824975c49944f16 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 11 Feb 2016 13:05:31 +0100 Subject: [PATCH 422/910] espresso : synchronized with current develop --- var/spack/{ => repos/builtin}/packages/espresso/package.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename var/spack/{ => repos/builtin}/packages/espresso/package.py (100%) diff --git a/var/spack/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py similarity index 100% rename from var/spack/packages/espresso/package.py rename to var/spack/repos/builtin/packages/espresso/package.py From ca3cdb445825126776c5269481540d3afac02c9f Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 11 Feb 2016 17:45:09 +0100 Subject: [PATCH 423/910] espresso : current working tree --- .../builtin/packages/espresso/package.py | 73 +++++++++---------- 1 file changed, 33 insertions(+), 40 deletions(-) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index ce5dcc2accc..56b8c056b8a 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -1,53 +1,44 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install espresso -# -# You can always get back here to change things with: -# -# spack edit espresso -# -# See the spack documentation for more information on building -# packages. -# +import llnl.util.tty as tty + from spack import * + class Espresso(Package): - """FIXME: put a proper description of your package here.""" - # FIXME: add a proper url for your package's homepage here. - homepage = "http://quantum-espresso.org" - url = "http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz" + """ + QE is an integrated suite of Open-Source computer codes for electronic-structure calculations and materials + modeling at the nanoscale. It is based on density-functional theory, plane waves, and pseudopotentials. + """ + homepage = 'http://quantum-espresso.org' + url = 'http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz' version('5.3.0', '6848fcfaeb118587d6be36bd10b7f2c3') + variant('mpi', default=True, description='Build Quantum-ESPRESSO with mpi support') - variant('openmp', default=False, description='Build Quantum-ESPRESSO with mpi openmp') - variant('scalapack', default=False, description='Build Quantum-ESPRESSO with mpi openmp') + variant('openmp', default=False, description='Enables openMP support') + variant('scalapack', default=False, description='Enables scalapack support') + variant('elpa', default=True, description='Use elpa as an eigenvalue solver') + depends_on('blas') + depends_on('lapack') - # FIXME: Add dependencies if this package requires them. - # depends_on("foo") depends_on('mpi', when='+mpi') + depends_on('elpa', when='+elpa') + depends_on('scalapack', when='+scalapack') - -# def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. -# configure('--prefix=%s' % prefix) - - # FIXME: Add logic to build and install here -# make() -# make("install") + def check_variants(self, spec): + error = 'you cannot ask for \'+{variant}\' when \'+mpi\' is not active' + if '+scalapack' in spec and '~mpi' in spec: + raise RuntimeError(error.format(variant='scalapack')) + if '+elpa' in spec and '~mpi' in spec: + raise RuntimeError(error.format(variant='elpa')) def install(self, spec, prefix): - # TAU isn't happy with directories that have '@' in the path. Sigh. + self.check_variants(spec) - # TAU configure, despite the name , seems to be a manually written script (nothing related to autotools). - # As such it has a few #peculiarities# that make this build quite hackish. - options = ["-prefix=%s" % prefix, - "--enable-parallel"] + options = ['-prefix=%s' % prefix] + + if '+mpi' in spec: + options.append('--enable-parallel') if '+openmp' in spec: options.append('--enable-openmp') @@ -55,7 +46,9 @@ def install(self, spec, prefix): if '+scalapack' in spec: options.append('--with-scalapack=yes') - configure(*options) - make("all") - make("install") + if '+elpa' in spec: + options.append('--with-elpa=%s' % spec['elpa'].prefix) + configure(*options) + make('all') + make('install') From b272a8881becf205c09ac11228dd035b99fa0fcd Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 11 Feb 2016 18:18:05 +0100 Subject: [PATCH 424/910] espresso : fixed dependency handling --- var/spack/repos/builtin/packages/espresso/package.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index 56b8c056b8a..df37bb0d71d 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -1,5 +1,3 @@ -import llnl.util.tty as tty - from spack import * @@ -15,21 +13,21 @@ class Espresso(Package): variant('mpi', default=True, description='Build Quantum-ESPRESSO with mpi support') variant('openmp', default=False, description='Enables openMP support') - variant('scalapack', default=False, description='Enables scalapack support') + variant('scalapack', default=True, description='Enables scalapack support') variant('elpa', default=True, description='Use elpa as an eigenvalue solver') depends_on('blas') depends_on('lapack') depends_on('mpi', when='+mpi') - depends_on('elpa', when='+elpa') - depends_on('scalapack', when='+scalapack') + depends_on('elpa', when='+elpa+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation + depends_on('scalapack', when='+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation def check_variants(self, spec): error = 'you cannot ask for \'+{variant}\' when \'+mpi\' is not active' if '+scalapack' in spec and '~mpi' in spec: raise RuntimeError(error.format(variant='scalapack')) - if '+elpa' in spec and '~mpi' in spec: + if '+elpa' in spec and ('~mpi' in spec or '~scalapack' in spec): raise RuntimeError(error.format(variant='elpa')) def install(self, spec, prefix): From e8704433debe35d7893c0c20672d06973226c4e8 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 11 Feb 2016 18:57:40 +0100 Subject: [PATCH 425/910] espresso : added directories to search path --- var/spack/repos/builtin/packages/espresso/package.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index df37bb0d71d..f85257b3ccf 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -47,6 +47,14 @@ def install(self, spec, prefix): if '+elpa' in spec: options.append('--with-elpa=%s' % spec['elpa'].prefix) + # Add a list of directories to search + search_list = [] + for name, dependency_spec in spec.dependencies.iteritems(): + print name + search_list.extend([dependency_spec.prefix.lib, + dependency_spec.prefix.lib64]) + search_list = " ".join(search_list) + options.append('LIBDIRS=%s' % search_list) configure(*options) make('all') make('install') From d7f674ce9678ddb3b964ad457b29ed058e2cb3c7 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 14 Jan 2016 17:41:44 -0500 Subject: [PATCH 426/910] libedit depends on ncurses --- var/spack/repos/builtin/packages/libedit/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/libedit/package.py b/var/spack/repos/builtin/packages/libedit/package.py index bcd5212b9ea..faed8bad374 100644 --- a/var/spack/repos/builtin/packages/libedit/package.py +++ b/var/spack/repos/builtin/packages/libedit/package.py @@ -7,6 +7,8 @@ class Libedit(Package): version('3.1', '43cdb5df3061d78b5e9d59109871b4f6', url="http://thrysoee.dk/editline/libedit-20150325-3.1.tar.gz") + depends_on('ncurses') + def install(self, spec, prefix): configure('--prefix=%s' % prefix) From 70985170e5f248d0d6a1b6245bd5581185ecaa64 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Fri, 12 Feb 2016 12:08:31 +0100 Subject: [PATCH 427/910] qe : compiled on rhle6 --- .../repos/builtin/packages/espresso/package.py | 15 ++++++++++----- var/spack/repos/builtin/packages/mpich/package.py | 8 ++++---- .../repos/builtin/packages/openblas/package.py | 2 ++ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index f85257b3ccf..a2bf58f585b 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -1,5 +1,6 @@ from spack import * +import os class Espresso(Package): """ @@ -20,9 +21,10 @@ class Espresso(Package): depends_on('lapack') depends_on('mpi', when='+mpi') - depends_on('elpa', when='+elpa+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation + depends_on('fftw~mpi', when='~mpi') + depends_on('fftw+mpi', when='+mpi') depends_on('scalapack', when='+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation - + def check_variants(self, spec): error = 'you cannot ask for \'+{variant}\' when \'+mpi\' is not active' if '+scalapack' in spec and '~mpi' in spec: @@ -45,16 +47,19 @@ def install(self, spec, prefix): options.append('--with-scalapack=yes') if '+elpa' in spec: - options.append('--with-elpa=%s' % spec['elpa'].prefix) + options.append('--with-elpa=yes') # Add a list of directories to search search_list = [] for name, dependency_spec in spec.dependencies.iteritems(): - print name search_list.extend([dependency_spec.prefix.lib, dependency_spec.prefix.lib64]) + search_list = " ".join(search_list) options.append('LIBDIRS=%s' % search_list) + options.append('F90=%s' % os.environ['FC']) + configure(*options) make('all') - make('install') + make('install') + diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index c856cfe277f..26d3bc0c945 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -48,10 +48,10 @@ class Mpich(Package): def setup_dependent_environment(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" - os.environ['MPICH_CC'] = 'cc' - os.environ['MPICH_CXX'] = 'c++' - os.environ['MPICH_F77'] = 'f77' - os.environ['MPICH_F90'] = 'f90' + os.environ['MPICH_CC'] = os.environ['CC'] + os.environ['MPICH_CXX'] = os.environ['CXX'] + os.environ['MPICH_F77'] = os.environ['F77'] + os.environ['MPICH_FC'] = os.environ['FC'] def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 9c8fa1c694b..3c909360a41 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -19,7 +19,9 @@ def install(self, spec, prefix): with working_dir(prefix.lib): symlink('libopenblas.a', 'blas.a') symlink('libopenblas.a', 'libblas.a') + symlink('libopenblas.so', 'libblas.so') # Lapack virtual package should provide liblapack.a with working_dir(prefix.lib): symlink('libopenblas.a', 'liblapack.a') + symlink('libopenblas.so', 'liblapack.so') From 247a4bc75108aa2106c1b191345a0c3387a8afac Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 12 Feb 2016 12:15:09 +0100 Subject: [PATCH 428/910] mpich : added back MPI_F90 --- var/spack/repos/builtin/packages/mpich/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 26d3bc0c945..c517defa83e 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -51,6 +51,7 @@ def setup_dependent_environment(self, module, spec, dep_spec): os.environ['MPICH_CC'] = os.environ['CC'] os.environ['MPICH_CXX'] = os.environ['CXX'] os.environ['MPICH_F77'] = os.environ['F77'] + os.environ['MPICH_F90'] = os.environ['FC'] os.environ['MPICH_FC'] = os.environ['FC'] From 90e1b1f0ea4320a5bdc0f54ec14a12fdda6584ff Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Fri, 12 Feb 2016 09:33:19 -0800 Subject: [PATCH 429/910] Use hash to make shorter module and dotkit names, fixes #433 --- lib/spack/spack/modules.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 7036626e29e..c8347635646 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -194,12 +194,14 @@ class Dotkit(EnvModule): @property def file_name(self): return join_path(Dotkit.path, self.spec.architecture, - self.spec.format('$_$@$%@$+$#.dk')) + '%s.dk' % self.use_name) @property def use_name(self): - return self.spec.format('$_$@$%@$+$#') - + return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version, + self.spec.compiler.name, + self.spec.compiler.version, + self.spec.dag_hash()) def _write(self, dk_file): # Category @@ -235,7 +237,10 @@ def file_name(self): @property def use_name(self): - return self.spec.format('$_$@$%@$+$#') + return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version, + self.spec.compiler.name, + self.spec.compiler.version, + self.spec.dag_hash()) def _write(self, m_file): From 77c17e1d92ac1c98a5c43247d6582876bc7d14d4 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 12 Feb 2016 12:51:43 -0500 Subject: [PATCH 430/910] Since my last patch didn't get traction, here is a new approach to building gcc on Darwin: - Add a variant specifying whether to build with binutils, defaulting to true - Auto-detect whether this is Darwin; if so, set binutils and gold defaults to false, as they don't work on Darwin - Disable Go, which doesn't build on Darwin - Clean up handling configure options --- .../repos/builtin/packages/gcc/package.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 3e5895cfb83..eb2129c4da2 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -26,6 +26,7 @@ from contextlib import closing from glob import glob +import sys class Gcc(Package): """The GNU Compiler Collection includes front ends for C, C++, @@ -49,13 +50,14 @@ class Gcc(Package): version('4.6.4', 'b407a3d1480c11667f293bfb1f17d1a4') version('4.5.4', '27e459c2566b8209ab064570e1b378f7') - variant('gold', default=True, description="Build the gold linker plugin for ld-based LTO") + variant('binutils', default=sys.platform != 'darwin', description="Build via binutils") + variant('gold', default=sys.platform != 'darwin', description="Build the gold linker plugin for ld-based LTO") depends_on("mpfr") depends_on("gmp") depends_on("mpc") # when @4.5: - depends_on("binutils~libiberty", when='~gold') - depends_on("binutils~libiberty+gold", when='+gold') + depends_on("binutils~libiberty", when='+binutils ~gold') + depends_on("binutils~libiberty+gold", when='+binutils +gold') # Save these until we can do optional deps. depends_on("isl", when=DEPENDS_ON_ISL_PREDICATE) @@ -67,7 +69,7 @@ def install(self, spec, prefix): filter_file(r"'@.*@'", "'@[[:alnum:]]*@'", 'libjava/configure', string=True) enabled_languages = set(('c', 'c++', 'fortran', 'java', 'objc')) - if spec.satisfies("@4.7.1:"): + if spec.satisfies("@4.7.1:") and sys.platform != 'darwin': enabled_languages.add('go') # Generic options to compile GCC @@ -79,17 +81,18 @@ def install(self, spec, prefix): "--with-mpfr=%s" % spec['mpfr'].prefix, "--with-gmp=%s" % spec['gmp'].prefix, "--enable-lto", - "--with-gnu-ld", - "--with-gnu-as", "--with-quad"] # Binutils - static_bootstrap_flags = "-static-libstdc++ -static-libgcc" - binutils_options = ["--with-sysroot=/", - "--with-stage1-ldflags=%s %s" % (self.rpath_args, static_bootstrap_flags), - "--with-boot-ldflags=%s %s" % (self.rpath_args, static_bootstrap_flags), - "--with-ld=%s/bin/ld" % spec['binutils'].prefix, - "--with-as=%s/bin/as" % spec['binutils'].prefix] - options.extend(binutils_options) + if spec.satisfies('+binutils'): + static_bootstrap_flags = "-static-libstdc++ -static-libgcc" + binutils_options = ["--with-sysroot=/", + "--with-stage1-ldflags=%s %s" % (self.rpath_args, static_bootstrap_flags), + "--with-boot-ldflags=%s %s" % (self.rpath_args, static_bootstrap_flags), + "--with-gnu-ld", + "--with-ld=%s/bin/ld" % spec['binutils'].prefix, + "--with-gnu-as", + "--with-as=%s/bin/as" % spec['binutils'].prefix] + options.extend(binutils_options) # Isl if spec.satisfies(Gcc.DEPENDS_ON_ISL_PREDICATE): isl_options = ["--with-isl=%s" % spec['isl'].prefix] From ff81aff2540f9eadd8d0ce123ae3143ebcb68789 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 12 Feb 2016 13:05:42 -0500 Subject: [PATCH 431/910] Remove source-code optimization and cleanup --- lib/spack/env/cc | 106 ++++++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 62 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index b83731404b4..ac8f717cc7f 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -65,7 +65,7 @@ function die { } for param in $parameters; do - if [[ -z ${!param} ]]; then + if [ -z "${!param}" ]; then die "Spack compiler must be run from spack! Input $param was missing!" fi done @@ -114,39 +114,31 @@ case "$command" in esac # Finish setting up the mode. -if [[ -z $mode ]]; then +if [ -z "$mode" ]; then mode=ccld for arg in "$@"; do - case "$arg" in - -v|-V|--version|-dumpversion) - mode=vcheck - break - ;; - -E) - mode=cpp - break - ;; - -c) - mode=cc - break - ;; - -S) - mode=as - break - ;; - esac + if [ "$arg" = -v -o "$arg" = -V -o "$arg" = --version -o "$arg" = -dumpversion ]; then + mode=vcheck + break + elif [ "$arg" = -E ]; then + mode=cpp + break + elif [ "$arg" = -c ]; then + mode=cc + break + fi done fi # Dump the version and exist if we're in testing mode. -if [[ $SPACK_TEST_COMMAND = dump-mode ]]; then +if [ "$SPACK_TEST_COMMAND" = "dump-mode" ]; then echo "$mode" exit fi # Check that at least one of the real commands was actually selected, # otherwise we don't know what to execute. -if [[ -z $command ]]; then +if [ -z "$command" ]; then die "ERROR: Compiler '$SPACK_COMPILER_SPEC' does not support compiling $language programs." fi @@ -159,8 +151,8 @@ if [[ -n $SPACK_TEST_COMMAND ]]; then # # Now do real parsing of the command line args, trying hard to keep - # non-rpath linker arguments in the proper order w.r.t. other command line - # arguments. This is important for things like groups. + # non-rpath linker arguments in the proper order w.r.t. other command + # line arguments. This is important for things like groups. # includes=() libraries=() @@ -168,33 +160,31 @@ if [[ -n $SPACK_TEST_COMMAND ]]; then rpaths=() other_args=() - while [[ -n $1 ]]; do + while [ -n "$1" ]; do case "$1" in -I*) arg="${1#-I}" - if [[ -z $arg ]]; then shift; arg="$1"; fi + if [ -z "$arg" ]; then shift; arg="$1"; fi includes+=("$arg") ;; -L*) arg="${1#-L}" - if [[ -z $arg ]]; then shift; arg="$1"; fi + if [ -z "$arg" ]; then shift; arg="$1"; fi libraries+=("$arg") ;; -l*) arg="${1#-l}" - if [[ -z $arg ]]; then shift; arg="$1"; fi + if [ -z "$arg" ]; then shift; arg="$1"; fi libs+=("$arg") ;; -Wl,*) arg="${1#-Wl,}" - if [[ -z $arg ]]; then shift; arg="$1"; fi - if [[ $arg = -rpath=* ]]; then + if [ -z "$arg" ]; then shift; arg="$1"; fi + if [[ "$arg" = -rpath=* ]]; then rpaths+=("${arg#-rpath=}") - elif [[ $arg = -rpath,* ]]; then - rpaths+=("${arg#-rpath,}") - elif [[ $arg = -rpath ]]; then + elif [[ "$arg" = -rpath ]]; then shift; arg="$1" - if [[ $arg != -Wl,* ]]; then + if [[ "$arg" != -Wl,* ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi rpaths+=("${arg#-Wl,}") @@ -204,14 +194,12 @@ if [[ -n $SPACK_TEST_COMMAND ]]; then ;; -Xlinker,*) arg="${1#-Xlinker,}" - if [[ -z $arg ]]; then shift; arg="$1"; fi - if [[ $arg = -rpath=* ]]; then + if [ -z "$arg" ]; then shift; arg="$1"; fi + if [[ "$arg" = -rpath=* ]]; then rpaths+=("${arg#-rpath=}") - elif [[ $arg = -rpath,* ]]; then - rpaths+=("${arg#-rpath,}") - elif [[ $arg = -rpath ]]; then + elif [[ "$arg" = -rpath ]]; then shift; arg="$1" - if [[ $arg != -Xlinker,* ]]; then + if [[ "$arg" != -Xlinker,* ]]; then die "-Xlinker,-rpath was not followed by -Xlinker,*" fi rpaths+=("${arg#-Xlinker,}") @@ -250,8 +238,8 @@ if [[ -n $SPACK_TEST_COMMAND ]]; then echo "${other_args[*]}" ;; *) - die "ERROR: Unknown test command" - ;; + echo "ERROR: Unknown test command" + exit 1 ;; esac exit fi @@ -259,12 +247,11 @@ fi # Read spack dependencies from the path environment variable IFS=':' read -ra deps <<< "$SPACK_DEPENDENCIES" for dep in "${deps[@]}"; do - if [[ -d $dep/include ]]; then + if [ -d "$dep/include" ]; then args=("-I$dep/include" "${args[@]}") fi - if [[ -d $dep/lib ]]; then - # libraries+=("$dep/lib") + if [ -d "$dep/lib" ]; then if [[ $mode = ccld ]]; then args=("-L$dep/lib" "-Wl,-rpath,$dep/lib" "${args[@]}") elif [[ $mode = ld ]]; then @@ -272,7 +259,7 @@ for dep in "${deps[@]}"; do fi fi - if [[ -d $dep/lib64 ]]; then + if [ -d "$dep/lib64" ]; then # libraries+=("$dep/lib64") if [[ $mode = ccld ]]; then args=("-L$dep/lib" "-Wl,-rpath,$dep/lib" "${args[@]}") @@ -284,18 +271,8 @@ done # Include all -L's and prefix/whatever dirs in rpath if [[ $mode = ccld ]]; then - # for dir in "${libraries[@]}"; do - # if [[ dir = $SPACK_INSTALL* ]]; then - # args=("-Wl,-rpath,$dir" "${args[@]}") - # fi - # done args=("-Wl,-rpath,$SPACK_PREFIX/lib" "-Wl,-rpath,$SPACK_PREFIX/lib64" "${args[@]}") elif [[ $mode = ld ]]; then - # for dir in "${libraries[@]}"; do - # if [[ dir = $SPACK_INSTALL* ]]; then - # args=("-rpath" "$dir" "${args[@]}") - # fi - # done args=("-rpath" "$SPACK_PREFIX/lib" "-rpath" "$SPACK_PREFIX/lib64" "${args[@]}") fi @@ -317,23 +294,28 @@ PATH="" for dir in "${env_path[@]}"; do remove="" for rm_dir in "${spack_env_dirs[@]}"; do - if [[ $dir = $rm_dir ]]; then remove=True; fi + if [ "$dir" = "$rm_dir" ]; then remove=True; fi done - if [[ -z $remove ]]; then - PATH="${PATH:+$PATH:}$dir" + if [ -z "$remove" ]; then + if [ -z "$PATH" ]; then + PATH="$dir" + else + PATH="$PATH:$dir" + fi fi done export PATH -full_command=("$command" "${args[@]}") +full_command=("$command") +full_command+=("${args[@]}") # # Write the input and output commands to debug logs if it's asked for. # -if [[ $SPACK_DEBUG = TRUE ]]; then +if [ "$SPACK_DEBUG" = "TRUE" ]; then input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.in.log" output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.out.log" - echo "$input_command" >> $input_log + echo "$input_command" >> $input_log echo "$mode ${full_command[@]}" >> $output_log fi From 8e33cc1ae118b138aadc3c354b622c045dd81009 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 12 Feb 2016 13:11:45 -0500 Subject: [PATCH 432/910] Properly wrap all long lines --- .../repos/builtin/packages/gcc/package.py | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index eb2129c4da2..8292670de00 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -50,8 +50,10 @@ class Gcc(Package): version('4.6.4', 'b407a3d1480c11667f293bfb1f17d1a4') version('4.5.4', '27e459c2566b8209ab064570e1b378f7') - variant('binutils', default=sys.platform != 'darwin', description="Build via binutils") - variant('gold', default=sys.platform != 'darwin', description="Build the gold linker plugin for ld-based LTO") + variant('binutils', default=sys.platform != 'darwin', + description="Build via binutils") + variant('gold', default=sys.platform != 'darwin', + description="Build the gold linker plugin for ld-based LTO") depends_on("mpfr") depends_on("gmp") @@ -66,7 +68,8 @@ class Gcc(Package): def install(self, spec, prefix): # libjava/configure needs a minor fix to install into spack paths. - filter_file(r"'@.*@'", "'@[[:alnum:]]*@'", 'libjava/configure', string=True) + filter_file(r"'@.*@'", "'@[[:alnum:]]*@'", 'libjava/configure', + string=True) enabled_languages = set(('c', 'c++', 'fortran', 'java', 'objc')) if spec.satisfies("@4.7.1:") and sys.platform != 'darwin': @@ -77,17 +80,19 @@ def install(self, spec, prefix): "--libdir=%s/lib64" % prefix, "--disable-multilib", "--enable-languages=" + ','.join(enabled_languages), - "--with-mpc=%s" % spec['mpc'].prefix, - "--with-mpfr=%s" % spec['mpfr'].prefix, - "--with-gmp=%s" % spec['gmp'].prefix, + "--with-mpc=%s" % spec['mpc'].prefix, + "--with-mpfr=%s" % spec['mpfr'].prefix, + "--with-gmp=%s" % spec['gmp'].prefix, "--enable-lto", "--with-quad"] # Binutils if spec.satisfies('+binutils'): static_bootstrap_flags = "-static-libstdc++ -static-libgcc" binutils_options = ["--with-sysroot=/", - "--with-stage1-ldflags=%s %s" % (self.rpath_args, static_bootstrap_flags), - "--with-boot-ldflags=%s %s" % (self.rpath_args, static_bootstrap_flags), + "--with-stage1-ldflags=%s %s" % + (self.rpath_args, static_bootstrap_flags), + "--with-boot-ldflags=%s %s" % + (self.rpath_args, static_bootstrap_flags), "--with-gnu-ld", "--with-ld=%s/bin/ld" % spec['binutils'].prefix, "--with-gnu-as", @@ -120,7 +125,8 @@ def write_rpath_specs(self): """Generate a spec file so the linker adds a rpath to the libs the compiler used to build the executable.""" if not self.spec_dir: - tty.warn("Could not install specs for %s." % self.spec.format('$_$@')) + tty.warn("Could not install specs for %s." % + self.spec.format('$_$@')) return gcc = Executable(join_path(self.prefix.bin, 'gcc')) @@ -130,5 +136,6 @@ def write_rpath_specs(self): for line in lines: out.write(line + "\n") if line.startswith("*link:"): - out.write("-rpath %s/lib:%s/lib64 \\\n"% (self.prefix, self.prefix)) + out.write("-rpath %s/lib:%s/lib64 \\\n" % + (self.prefix, self.prefix)) set_install_permissions(specs_file) From 2cd9ad8ce63ea29093fcc2d8e1bd749d5cbccf0b Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 17 Jan 2016 10:46:21 -0500 Subject: [PATCH 433/910] Use "-Wl,-rpath," instead of "-Wl,-rpath=" The former translates to a linker argument "-rpath DIR", whereas the latter translates to "-rpath=DIR". The latter is not support on OS X. --- lib/spack/docs/packaging_guide.rst | 6 +++--- lib/spack/spack/package.py | 4 ++-- lib/spack/spack/test/cc.py | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 59ba63fa355..bb8a26ad02f 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1711,15 +1711,15 @@ Compile-time library search paths * ``-L$dep_prefix/lib`` * ``-L$dep_prefix/lib64`` Runtime library search paths (RPATHs) - * ``-Wl,-rpath=$dep_prefix/lib`` - * ``-Wl,-rpath=$dep_prefix/lib64`` + * ``-Wl,-rpath,$dep_prefix/lib`` + * ``-Wl,-rpath,$dep_prefix/lib64`` Include search paths * ``-I$dep_prefix/include`` An example of this would be the ``libdwarf`` build, which has one dependency: ``libelf``. Every call to ``cc`` in the ``libdwarf`` build will have ``-I$LIBELF_PREFIX/include``, -``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath=$LIBELF_PREFIX/lib`` +``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath,$LIBELF_PREFIX/lib`` inserted on the command line. This is done transparently to the project's build system, which will just think it's using a system where ``libelf`` is readily available. Because of this, you **do diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8cb947c276c..49d4fb6b237 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1220,8 +1220,8 @@ def rpath(self): @property def rpath_args(self): - """Get the rpath args as a string, with -Wl,-rpath= for each element.""" - return " ".join("-Wl,-rpath=%s" % p for p in self.rpath) + """Get the rpath args as a string, with -Wl,-rpath, for each element.""" + return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) def validate_package_url(url_string): diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 905af28a06d..54d56383941 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -39,11 +39,11 @@ 'arg1', '-Wl,--start-group', 'arg2', - '-Wl,-rpath=/first/rpath', 'arg3', '-Wl,-rpath', '-Wl,/second/rpath', + '-Wl,-rpath,/first/rpath', 'arg3', '-Wl,-rpath', '-Wl,/second/rpath', '-llib1', '-llib2', 'arg4', '-Wl,--end-group', - '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath=/fourth/rpath', + '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath,/fourth/rpath', '-llib3', '-llib4', 'arg5', 'arg6'] @@ -95,13 +95,13 @@ def test_cpp_mode(self): def test_ccld_mode(self): self.check_cc('dump-mode', [], "ccld") self.check_cc('dump-mode', ['foo.c', '-o', 'foo'], "ccld") - self.check_cc('dump-mode', ['foo.c', '-o', 'foo', '-Wl,-rpath=foo'], "ccld") - self.check_cc('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath=foo'], "ccld") + self.check_cc('dump-mode', ['foo.c', '-o', 'foo', '-Wl,-rpath,foo'], "ccld") + self.check_cc('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ccld") def test_ld_mode(self): self.check_ld('dump-mode', [], "ld") - self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath=foo'], "ld") + self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ld") def test_includes(self): From 5038a38e296940f463c13d6fdda728d8673bc95a Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 18 Jan 2016 18:13:18 -0500 Subject: [PATCH 434/910] Correct -Xlinker arguments --- lib/spack/spack/test/cc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 54d56383941..11420ec44a1 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -43,7 +43,7 @@ '-llib1', '-llib2', 'arg4', '-Wl,--end-group', - '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath,/fourth/rpath', + '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath', '-Xlinker,/fourth/rpath', '-llib3', '-llib4', 'arg5', 'arg6'] From dc6a33b716dd6712da2e65a78a6a3ed98ca72d4d Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 12 Feb 2016 12:09:29 -0500 Subject: [PATCH 435/910] Handle multiple -Wl,-rpath,... paths --- lib/spack/env/cc | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index aacba996b39..c156b7b6073 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -176,14 +176,21 @@ while [ -n "$1" ]; do -Wl,*) arg="${1#-Wl,}" if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ "$arg" = -rpath=* ]]; then - rpaths+=("${arg#-rpath=}") - elif [[ "$arg" = -rpath ]]; then + if [[ $arg = -rpath=* ]]; then + arg="${arg#-rpath=}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done + elif [[ $arg = -rpath ]]; then shift; arg="$1" - if [[ "$arg" != -Wl,* ]]; then + if [[ $arg != -Wl,* ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi - rpaths+=("${arg#-Wl,}") + # TODO: Handle multiple -Wl, continuations of -Wl,-rpath + arg="${arg#-Wl,}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done else other_args+=("-Wl,$arg") fi @@ -191,11 +198,11 @@ while [ -n "$1" ]; do -Xlinker,*) arg="${1#-Xlinker,}" if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ "$arg" = -rpath=* ]]; then + if [[ $arg = -rpath=* ]]; then rpaths+=("${arg#-rpath=}") - elif [[ "$arg" = -rpath ]]; then + elif [[ $arg = -rpath ]]; then shift; arg="$1" - if [[ "$arg" != -Xlinker,* ]]; then + if [[ $arg != -Xlinker,* ]]; then die "-Xlinker,-rpath was not followed by -Xlinker,*" fi rpaths+=("${arg#-Xlinker,}") From 9a2c1090a6f40db762ac155d7d25063c2965c841 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 12 Feb 2016 12:37:03 -0500 Subject: [PATCH 436/910] Handle -Wl,-rpath,... syntax --- lib/spack/env/cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index c156b7b6073..41933f5e1fe 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -181,6 +181,11 @@ while [ -n "$1" ]; do for rpath in ${arg//,/ }; do rpaths+=("$rpath") done + elif [[ $arg = -rpath,* ]]; then + arg="${arg#-rpath,}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done elif [[ $arg = -rpath ]]; then shift; arg="$1" if [[ $arg != -Wl,* ]]; then From 69064395eb61db3b03d1ed14f16bef7ec2c94ee3 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 13 Feb 2016 15:12:09 -0500 Subject: [PATCH 437/910] Add debug output --- lib/spack/env/cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 41933f5e1fe..a431cffacf4 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -174,19 +174,26 @@ while [ -n "$1" ]; do libs+=("$arg") ;; -Wl,*) + echo "FOUND arg=[$arg]" >&2 arg="${1#-Wl,}" if [ -z "$arg" ]; then shift; arg="$1"; fi + echo "SHIFTED, arg=[$arg]" >&2 if [[ $arg = -rpath=* ]]; then + echo "CASE 1" >&2 arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do + echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done elif [[ $arg = -rpath,* ]]; then + echo "CASE 2" >&2 arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do + echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done elif [[ $arg = -rpath ]]; then + echo "CASE 3" >&2 shift; arg="$1" if [[ $arg != -Wl,* ]]; then die "-Wl,-rpath was not followed by -Wl,*" @@ -194,9 +201,11 @@ while [ -n "$1" ]; do # TODO: Handle multiple -Wl, continuations of -Wl,-rpath arg="${arg#-Wl,}" for rpath in ${arg//,/ }; do + echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done else + echo "OTHER" >&2 other_args+=("-Wl,$arg") fi ;; From 52647b9a5d8ab4b2fff7387f65347164576be088 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 13 Feb 2016 17:55:14 -0500 Subject: [PATCH 438/910] Using regexes instead of globbing to match path names --- lib/spack/env/cc | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index a431cffacf4..c46986e19cd 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -174,26 +174,19 @@ while [ -n "$1" ]; do libs+=("$arg") ;; -Wl,*) - echo "FOUND arg=[$arg]" >&2 arg="${1#-Wl,}" if [ -z "$arg" ]; then shift; arg="$1"; fi - echo "SHIFTED, arg=[$arg]" >&2 - if [[ $arg = -rpath=* ]]; then - echo "CASE 1" >&2 + if [[ $arg =~ -rpath=.* ]]; then arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do - echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done - elif [[ $arg = -rpath,* ]]; then - echo "CASE 2" >&2 + elif [[ $arg =~ -rpath,.* ]]; then arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do - echo " RPATH=[$rpath]" >&2 - rpaths+=("$rpath") + rpaths+=("$rpath") done elif [[ $arg = -rpath ]]; then - echo "CASE 3" >&2 shift; arg="$1" if [[ $arg != -Wl,* ]]; then die "-Wl,-rpath was not followed by -Wl,*" @@ -201,11 +194,9 @@ while [ -n "$1" ]; do # TODO: Handle multiple -Wl, continuations of -Wl,-rpath arg="${arg#-Wl,}" for rpath in ${arg//,/ }; do - echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done else - echo "OTHER" >&2 other_args+=("-Wl,$arg") fi ;; From 43670cbbd01562f6748f3ee6e2505be52e24bccb Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 13 Feb 2016 22:01:36 -0500 Subject: [PATCH 439/910] More games with quoting --- lib/spack/env/cc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index c46986e19cd..37483e00733 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -175,20 +175,19 @@ while [ -n "$1" ]; do ;; -Wl,*) arg="${1#-Wl,}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ $arg =~ -rpath=.* ]]; then + if [[ $arg = "-rpath=*" ]]; then arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg =~ -rpath,.* ]]; then + elif [[ $arg = "-rpath,*" ]]; then arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do - rpaths+=("$rpath") + rpaths+=("$rpath") done - elif [[ $arg = -rpath ]]; then + elif [[ $arg = "-rpath" ]]; then shift; arg="$1" - if [[ $arg != -Wl,* ]]; then + if [[ $arg != "-Wl,*" ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi # TODO: Handle multiple -Wl, continuations of -Wl,-rpath From a06e29fecbaf100f089c1eab7c6447cd469b95f6 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 14 Feb 2016 10:43:55 -0500 Subject: [PATCH 440/910] More quoting experiments --- lib/spack/env/cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 37483e00733..853a19dfddd 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -175,22 +175,22 @@ while [ -n "$1" ]; do ;; -Wl,*) arg="${1#-Wl,}" - if [[ $arg = "-rpath=*" ]]; then + # TODO: Handle multiple -Wl, continuations of -Wl,-rpath + if [[ $arg == '-rpath='* ]]; then arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg = "-rpath,*" ]]; then + elif [[ $arg == '-rpath,'* ]]; then arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg = "-rpath" ]]; then + elif [[ $arg == '-rpath' ]]; then shift; arg="$1" if [[ $arg != "-Wl,*" ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi - # TODO: Handle multiple -Wl, continuations of -Wl,-rpath arg="${arg#-Wl,}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") From 9868333e8e2586ffc55ee996a48ef62601a9c874 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 14 Feb 2016 11:34:08 -0500 Subject: [PATCH 441/910] Shell quoting is difficult --- lib/spack/env/cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 853a19dfddd..c3d11357224 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -188,7 +188,7 @@ while [ -n "$1" ]; do done elif [[ $arg == '-rpath' ]]; then shift; arg="$1" - if [[ $arg != "-Wl,*" ]]; then + if [[ $arg != '-Wl,'* ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi arg="${arg#-Wl,}" From d8c0edcc818b8f0fb2220993d469381ae072ebe4 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 14 Feb 2016 14:40:25 -0800 Subject: [PATCH 442/910] Minor code cleanup for gcc. --- var/spack/repos/builtin/packages/gcc/package.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 3e5895cfb83..f8958ee2907 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -36,8 +36,6 @@ class Gcc(Package): list_url = 'http://open-source-box.org/gcc/' list_depth = 2 - DEPENDS_ON_ISL_PREDICATE = '@5.0:' - version('5.3.0', 'c9616fd448f980259c31de613e575719') version('5.2.0', 'a51bcfeb3da7dd4c623e27207ed43467') version('4.9.3', '6f831b4d251872736e8e9cc09746f327') @@ -53,12 +51,11 @@ class Gcc(Package): depends_on("mpfr") depends_on("gmp") - depends_on("mpc") # when @4.5: + depends_on("mpc", when='@4.5:') + depends_on("isl", when='@5.0:') depends_on("binutils~libiberty", when='~gold') depends_on("binutils~libiberty+gold", when='+gold') - # Save these until we can do optional deps. - depends_on("isl", when=DEPENDS_ON_ISL_PREDICATE) #depends_on("ppl") #depends_on("cloog") @@ -91,7 +88,7 @@ def install(self, spec, prefix): "--with-as=%s/bin/as" % spec['binutils'].prefix] options.extend(binutils_options) # Isl - if spec.satisfies(Gcc.DEPENDS_ON_ISL_PREDICATE): + if 'isl' in spec: isl_options = ["--with-isl=%s" % spec['isl'].prefix] options.extend(isl_options) From bf162e60f17f8aebcd62184d72a194eb91e4ef4f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 15 Feb 2016 10:53:50 -0600 Subject: [PATCH 443/910] Add latest version --- var/spack/repos/builtin/packages/py-mpi4py/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/py-mpi4py/package.py b/var/spack/repos/builtin/packages/py-mpi4py/package.py index 8001689a18d..f599205644e 100644 --- a/var/spack/repos/builtin/packages/py-mpi4py/package.py +++ b/var/spack/repos/builtin/packages/py-mpi4py/package.py @@ -5,7 +5,9 @@ class PyMpi4py(Package): homepage = "https://pypi.python.org/pypi/mpi4py" url = "https://pypi.python.org/packages/source/m/mpi4py/mpi4py-1.3.1.tar.gz" + version('2.0.0', '4f7d8126d7367c239fd67615680990e3') version('1.3.1', 'dbe9d22bdc8ed965c23a7ceb6f32fc3c') + extends('python') depends_on('py-setuptools') depends_on('mpi') From db50f52bbc47ef6dbe2e26c54ce828c0e6076e96 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 16 Feb 2016 11:40:00 -0800 Subject: [PATCH 444/910] Adding the "Crypto++" package installation files. --- .../builtin/packages/cryptopp/package.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 var/spack/repos/builtin/packages/cryptopp/package.py diff --git a/var/spack/repos/builtin/packages/cryptopp/package.py b/var/spack/repos/builtin/packages/cryptopp/package.py new file mode 100644 index 00000000000..1693c4b160f --- /dev/null +++ b/var/spack/repos/builtin/packages/cryptopp/package.py @@ -0,0 +1,31 @@ +import glob +from spack import * + +class Cryptopp(Package): + """Crypto++ is an open-source C++ library of cryptographic schemes. The + library supports a number of different cryptography algorithms, including + authenticated encryption schemes (GCM, CCM), hash functions (SHA-1, SHA2), + public-key encryption (RSA, DSA), and a few obsolete/historical encryption + algorithms (MD5, Panama).""" + + homepage = "http://www.cryptopp.com/" + url = "http://www.cryptopp.com/cryptopp563.zip" + + version('5.6.3', '3c5b70e2ec98b7a24988734446242d07') + version('5.6.2', '7ed022585698df48e65ce9218f6c6a67') + + def install(self, spec, prefix): + make() + + mkdirp(prefix.include) + for hfile in glob.glob('*.h*'): + install(hfile, prefix.include) + + mkdirp(prefix.lib) + install('libcryptopp.a', prefix.lib) + + def url_for_version(self, version): + version_tuple = tuple(v for v in iter(version)) + version_string = reduce(lambda vs, nv: vs + str(nv), version_tuple, "") + + return "%scryptopp%s.zip" % (Cryptopp.homepage, version_string) From 9a6221ea40e8cda3bdfa495b7c4a93284e8f4895 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 16 Feb 2016 12:47:14 -0800 Subject: [PATCH 445/910] Added the installation files for the "ndiff" package. --- .../repos/builtin/packages/ndiff/package.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 var/spack/repos/builtin/packages/ndiff/package.py diff --git a/var/spack/repos/builtin/packages/ndiff/package.py b/var/spack/repos/builtin/packages/ndiff/package.py new file mode 100644 index 00000000000..10e445c81e9 --- /dev/null +++ b/var/spack/repos/builtin/packages/ndiff/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Ndiff(Package): + """The ndiff tool is a binary utility that compares putatively similar files + while ignoring small numeric differernces. This utility is most often used + to compare files containing a lot of floating-point numeric data that + may be slightly different due to numeric error.""" + + homepage = "http://ftp.math.utah.edu/pub/ndiff/" + url = "http://ftp.math.utah.edu/pub/ndiff/ndiff-2.00.tar.gz" + + version('2.00', '885548b4dc26e72c5455bebb5ba6c16d') + version('1.00', 'f41ffe5d12f36cd36b6311acf46eccdc') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + mkdirp(prefix.bin) + mkdirp('%s/lib' % prefix.share) + + make('install-exe', 'install-shrlib') From 3c8bbeafc78c00cd93fa4526a0e55bf16d36b454 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 16 Feb 2016 13:01:18 -0800 Subject: [PATCH 446/910] Added the installation files for the "Triangle" package. --- .../builtin/packages/Triangle/package.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 var/spack/repos/builtin/packages/Triangle/package.py diff --git a/var/spack/repos/builtin/packages/Triangle/package.py b/var/spack/repos/builtin/packages/Triangle/package.py new file mode 100644 index 00000000000..f65d93776df --- /dev/null +++ b/var/spack/repos/builtin/packages/Triangle/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Triangle(Package): + """Triangle is a two-dimensional mesh generator and Delaunay + triangulator. Triangle generates exact Delaunay triangulations, + constrained Delaunay triangulations, conforming Delaunay + triangulations, Voronoi diagrams, and high-quality triangular + meshes.""" + + homepage = "http://www.cs.cmu.edu/~quake/triangle.html" + url = "http://www.netlib.org/voronoi/triangle.zip" + + version('1.6', '10aff8d7950f5e0e2fb6dd2e340be2c9') + + def install(self, spec, prefix): + make() + mkdirp(prefix.bin) + + install('triangle', prefix.bin) + install('showme', prefix.bin) From 09254014b182ccf4cbc4ce291141b7ab39b9171d Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 16 Feb 2016 13:24:01 -0800 Subject: [PATCH 447/910] Added the installation files for the "TetGen" package. --- .../repos/builtin/packages/tetgen/package.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 var/spack/repos/builtin/packages/tetgen/package.py diff --git a/var/spack/repos/builtin/packages/tetgen/package.py b/var/spack/repos/builtin/packages/tetgen/package.py new file mode 100644 index 00000000000..30c2b766558 --- /dev/null +++ b/var/spack/repos/builtin/packages/tetgen/package.py @@ -0,0 +1,28 @@ +from spack import * + +class Tetgen(Package): + """TetGen is a program and library that can be used to generate tetrahedral + meshes for given 3D polyhedral domains. TetGen generates exact constrained + Delaunay tetrahedralizations, boundary conforming Delaunay meshes, and + Voronoi paritions.""" + + homepage = "http://www.tetgen.org" + url = "http://www.tetgen.org/files/tetgen1.4.3.tar.gz" + + version('1.4.3', 'd6a4bcdde2ac804f7ec66c29dcb63c18') + + # TODO: Make this a build dependency once build dependencies are supported + # (see: https://github.com/LLNL/spack/pull/378). + depends_on('cmake@2.8.7:', when='@1.5.0:') + + def install(self, spec, prefix): + make('tetgen', 'tetlib') + + mkdirp(prefix.bin) + install('tetgen', prefix.bin) + + mkdirp(prefix.include) + install('tetgen.h', prefix.include) + + mkdirp(prefix.lib) + install('libtet.a', prefix.lib) From 8cab10214e63344b77ffc34df3609dbc9f1690a5 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 16 Feb 2016 16:43:24 -0800 Subject: [PATCH 448/910] Added the installation files for the "ExodusII" package. Added the "+static" variant to "hdf5" to enable "ExodusII" support. --- .../packages/exodusii/exodus-cmake.patch | 12 +++++ .../builtin/packages/exodusii/package.py | 49 +++++++++++++++++++ .../repos/builtin/packages/hdf5/package.py | 4 ++ 3 files changed, 65 insertions(+) create mode 100644 var/spack/repos/builtin/packages/exodusii/exodus-cmake.patch create mode 100644 var/spack/repos/builtin/packages/exodusii/package.py diff --git a/var/spack/repos/builtin/packages/exodusii/exodus-cmake.patch b/var/spack/repos/builtin/packages/exodusii/exodus-cmake.patch new file mode 100644 index 00000000000..25355269cae --- /dev/null +++ b/var/spack/repos/builtin/packages/exodusii/exodus-cmake.patch @@ -0,0 +1,12 @@ +diff --git a/cmake-exodus b/cmake-exodus +index 787fd9d..ed073a2 100755 +--- a/cmake-exodus ++++ b/cmake-exodus +@@ -1,4 +1,6 @@ +-EXTRA_ARGS=$@ ++#!/bin/bash ++ ++EXTRA_ARGS=-DSEACASProj_ENABLE_CXX11=OFF + + ### Change this to point to the compilers you want to use + CC=gcc diff --git a/var/spack/repos/builtin/packages/exodusii/package.py b/var/spack/repos/builtin/packages/exodusii/package.py new file mode 100644 index 00000000000..89c04bf79cf --- /dev/null +++ b/var/spack/repos/builtin/packages/exodusii/package.py @@ -0,0 +1,49 @@ +from spack import * + +# TODO: Add support for a C++11 enabled installation that filters out the +# TODO: "C++11-Disabled" flag (but only if the spec compiler supports C++11). + +# TODO: Add support for parallel installation that uses MPI. + +# TODO: Create installation options for NetCDF that support larger page size +# TODO: suggested by Exodus (see the repository "README" file). + +class Exodusii(Package): + """Exodus II is a C++/Fortran library developed to store and retrieve data for + finite element analyses. It's used for preprocessing (problem definition), + postprocessing (results visualization), and data transfer between codes. + An Exodus II data file is a random access, machine independent, binary + file that is written and read via C, C++, or Fortran API routines.""" + + homepage = "https://github.com/gsjaardema/seacas" + url = "https://github.com/gsjaardema/seacas/archive/master.zip" + + version('2016-02-08', git='https://github.com/gsjaardema/seacas.git', commit='dcf3529') + + # TODO: Make this a build dependency once build dependencies are supported + # (see: https://github.com/LLNL/spack/pull/378). + depends_on('cmake@2.8.7:') + depends_on('hdf5+static~mpi') + depends_on('netcdf~mpi') + + patch('exodus-cmake.patch') + + def patch(self): + ff = FileFilter('cmake-exodus') + + ff.filter('CMAKE_INSTALL_PREFIX:PATH=${ACCESS}', + 'CMAKE_INSTALL_PREFIX:PATH=%s' % self.spec.prefix, string=True) + ff.filter('NetCDF_DIR:PATH=${TPL}', + 'NetCDF_DIR:PATH=%s' % self.spec['netcdf'].prefix, string=True) + ff.filter('HDF5_ROOT:PATH=${TPL}', + 'HDF5_ROOT:PATH=%s' % self.spec['hdf5'].prefix, string=True) + + def install(self, spec, prefix): + mkdirp('build') + cd('build') + + cmake_exodus = Executable('../cmake-exodus') + cmake_exodus() + + make() + make('install') diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 80f79539c09..7db4aff6315 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -42,6 +42,7 @@ class Hdf5(Package): version('1.8.13', 'c03426e9e77d7766944654280b467289') variant('debug', default=False, description='Builds a debug version of the library') + variant('static', default=False, description='Builds a static executable version of the library') variant('cxx', default=True, description='Enable C++ support') variant('fortran', default=True, description='Enable Fortran support') @@ -78,6 +79,9 @@ def install(self, spec, prefix): else: extra_args.append('--enable-production') + if '+static' in spec: + extra_args.append('--enable-static-exec') + if '+unsupported' in spec: extra_args.append("--enable-unsupported") From fca7ef2f7b3d8091e935073ec24570ebb163fe6d Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 16 Feb 2016 17:23:37 -0800 Subject: [PATCH 449/910] Moved "Triangle" to "triangle" to be more in line with Spack package naming conventions. --- .../repos/builtin/packages/{Triangle => triangle}/package.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename var/spack/repos/builtin/packages/{Triangle => triangle}/package.py (100%) diff --git a/var/spack/repos/builtin/packages/Triangle/package.py b/var/spack/repos/builtin/packages/triangle/package.py similarity index 100% rename from var/spack/repos/builtin/packages/Triangle/package.py rename to var/spack/repos/builtin/packages/triangle/package.py From 0388093f7aed92bd1d3282fe4f53c27e66030fd4 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 16 Feb 2016 17:33:19 -0800 Subject: [PATCH 450/910] Changed the hdf5 "+static" variant to become the "+shared" variant. --- var/spack/repos/builtin/packages/exodusii/package.py | 2 +- var/spack/repos/builtin/packages/hdf5/package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/exodusii/package.py b/var/spack/repos/builtin/packages/exodusii/package.py index 89c04bf79cf..d68baaa6d9d 100644 --- a/var/spack/repos/builtin/packages/exodusii/package.py +++ b/var/spack/repos/builtin/packages/exodusii/package.py @@ -23,7 +23,7 @@ class Exodusii(Package): # TODO: Make this a build dependency once build dependencies are supported # (see: https://github.com/LLNL/spack/pull/378). depends_on('cmake@2.8.7:') - depends_on('hdf5+static~mpi') + depends_on('hdf5+shared~mpi') depends_on('netcdf~mpi') patch('exodus-cmake.patch') diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 7db4aff6315..f4de92aa837 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -42,7 +42,7 @@ class Hdf5(Package): version('1.8.13', 'c03426e9e77d7766944654280b467289') variant('debug', default=False, description='Builds a debug version of the library') - variant('static', default=False, description='Builds a static executable version of the library') + variant('shared', default=False, description='Builds a static executable version of the library') variant('cxx', default=True, description='Enable C++ support') variant('fortran', default=True, description='Enable Fortran support') From 6cd76d69c3fa4b9eb2d30660de28206697af722d Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Wed, 17 Feb 2016 11:51:24 -0800 Subject: [PATCH 451/910] Fixed the "+shared" variant in the "hdf5" package. --- var/spack/repos/builtin/packages/exodusii/package.py | 2 +- var/spack/repos/builtin/packages/hdf5/package.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/exodusii/package.py b/var/spack/repos/builtin/packages/exodusii/package.py index d68baaa6d9d..af258b7e6e9 100644 --- a/var/spack/repos/builtin/packages/exodusii/package.py +++ b/var/spack/repos/builtin/packages/exodusii/package.py @@ -23,7 +23,7 @@ class Exodusii(Package): # TODO: Make this a build dependency once build dependencies are supported # (see: https://github.com/LLNL/spack/pull/378). depends_on('cmake@2.8.7:') - depends_on('hdf5+shared~mpi') + depends_on('hdf5~shared~mpi') depends_on('netcdf~mpi') patch('exodus-cmake.patch') diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index f4de92aa837..ed4e7c35c99 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -42,7 +42,7 @@ class Hdf5(Package): version('1.8.13', 'c03426e9e77d7766944654280b467289') variant('debug', default=False, description='Builds a debug version of the library') - variant('shared', default=False, description='Builds a static executable version of the library') + variant('shared', default=True, description='Builds a shared version of the library') variant('cxx', default=True, description='Enable C++ support') variant('fortran', default=True, description='Enable Fortran support') @@ -79,7 +79,9 @@ def install(self, spec, prefix): else: extra_args.append('--enable-production') - if '+static' in spec: + if '+shared' in spec: + extra_args.append('--enable-shared') + else: extra_args.append('--enable-static-exec') if '+unsupported' in spec: @@ -123,7 +125,6 @@ def install(self, spec, prefix): configure( "--prefix=%s" % prefix, "--with-zlib=%s" % spec['zlib'].prefix, - "--enable-shared", # TODO : this should be enabled by default, remove it? *extra_args) make() make("install") From 00125e4688bb457937b4907f8e69afe2a06737fc Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 17 Feb 2016 16:43:36 -0800 Subject: [PATCH 452/910] remove the unprotected key name from create.py Small fix to use dicts correctly. --- lib/spack/spack/cmd/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index edcea0718cd..6809209046e 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -222,7 +222,7 @@ def fetch_tarballs(url, name, args): archives_to_fetch = 1 if not versions: # If the fetch failed for some reason, revert to what the user provided - versions = { version : url } + versions = { "version" : url } elif len(versions) > 1: tty.msg("Found %s versions of %s:" % (len(versions), name), *spack.cmd.elide_list( From d8a70166d3abd14b5a34025f735cf1825179f84f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 18 Feb 2016 00:56:29 -0800 Subject: [PATCH 453/910] Fixes #434 Compiler detection was not getting triggered properly with some of the new config logic. Adjust the conditions under which Spack will serach for compilers. --- lib/spack/spack/compilers/__init__.py | 32 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 6159ef576ca..3a04bc2ebc5 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -74,28 +74,36 @@ def _to_dict(compiler): def get_compiler_config(arch=None, scope=None): """Return the compiler configuration for the specified architecture. """ - # If any configuration file has compilers, just stick with the - # ones already configured. - config = spack.config.get_config('compilers', scope=scope) - + # Check whether we're on a front-end (native) architecture. my_arch = spack.architecture.sys_type() if arch is None: arch = my_arch - if arch in config: - return config[arch] - - # Only for the current arch in *highest* scope: automatically try to - # find compilers if none are configured yet. - if arch == my_arch and scope == 'user': + def init_compiler_config(): + """Compiler search used when Spack has no compilers.""" config[arch] = {} compilers = find_compilers(*get_path('PATH')) for compiler in compilers: config[arch].update(_to_dict(compiler)) spack.config.update_config('compilers', config, scope=scope) - return config[arch] - return {} + config = spack.config.get_config('compilers', scope=scope) + + # Update the configuration if there are currently no compilers + # configured. Avoid updating automatically if there ARE site + # compilers configured but no user ones. + if arch == my_arch and arch not in config: + if scope is None: + # We know no compilers were configured in any scope. + init_compiler_config() + elif scope == 'user': + # Check the site config and update the user config if + # nothing is configured at the site level. + site_config = spack.config.get_config('compilers', scope='site') + if not site_config: + init_compiler_config() + + return config[arch] if arch in config else {} def add_compilers_to_config(compilers, arch=None, scope=None): From 19d10291bfeae45315fcef852baddec63b69247a Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Thu, 18 Feb 2016 15:45:29 -0800 Subject: [PATCH 454/910] modify compiler commands in python config files, fix for #431 --- var/spack/repos/builtin/packages/python/package.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index a1ce06feb03..58d401244e3 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -55,6 +55,20 @@ def install(self, spec, prefix): make() make("install") + # Modify compiler paths in configuration files. This is necessary for + # building site packages outside of spack + filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)', + (r'\4\5'), + join_path(prefix.lib, 'python%d.%d' % self.version[:2], '_sysconfigdata.py')) + + python3_version = '' + if spec.satisfies('@3:'): + python3_version = '-%d.%dm' % self.version[:2] + makefile_filepath = join_path(prefix.lib, 'python%d.%d' % self.version[:2], 'config%s' % python3_version, 'Makefile') + filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)', + (r'\4\5'), + makefile_filepath) + # ======================================================================== # Set up environment to make install easy for python extensions. From 30c304748213bee3669c75d4384bad146f68dbd7 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 19 Feb 2016 12:39:38 +0100 Subject: [PATCH 455/910] gromacs : added package --- .../repos/builtin/packages/fftw/package.py | 2 + .../repos/builtin/packages/gromacs/package.py | 56 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 var/spack/repos/builtin/packages/gromacs/package.py diff --git a/var/spack/repos/builtin/packages/fftw/package.py b/var/spack/repos/builtin/packages/fftw/package.py index 4d2b9642426..bc129aaf1a1 100644 --- a/var/spack/repos/builtin/packages/fftw/package.py +++ b/var/spack/repos/builtin/packages/fftw/package.py @@ -47,6 +47,8 @@ class Fftw(Package): depends_on('mpi', when='+mpi') + # TODO : add support for architecture specific optimizations as soon as targets are supported + def install(self, spec, prefix): options = ['--prefix=%s' % prefix, '--enable-shared', diff --git a/var/spack/repos/builtin/packages/gromacs/package.py b/var/spack/repos/builtin/packages/gromacs/package.py new file mode 100644 index 00000000000..5fe8399308b --- /dev/null +++ b/var/spack/repos/builtin/packages/gromacs/package.py @@ -0,0 +1,56 @@ +from spack import * + + +class Gromacs(Package): + """ + GROMACS (GROningen MAchine for Chemical Simulations) is a molecular dynamics package primarily designed for + simulations of proteins, lipids and nucleic acids. It was originally developed in the Biophysical Chemistry + department of University of Groningen, and is now maintained by contributors in universities and research centers + across the world. + + GROMACS is one of the fastest and most popular software packages available and can run on CPUs as well as GPUs. + It is free, open source released under the GNU General Public License. Starting from version 4.6, GROMACS is + released under the GNU Lesser General Public License. + """ + + homepage = 'http://www.gromacs.org' + url = 'ftp://ftp.gromacs.org/pub/gromacs/gromacs-5.1.2.tar.gz' + + version('5.1.2', '614d0be372f1a6f1f36382b7a6fcab98') + + variant('mpi', default=True, description='Activate MPI support') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Enables debug mode') + variant('double', default=False, description='Produces a double precision version of the executables') + + depends_on('mpi', when='+mpi') + + depends_on('fftw') + + # TODO : add GPU support + + def install(self, spec, prefix): + + options = [] + + if '+mpi' in spec: + options.append('-DGMX_MPI:BOOL=ON') + + if '+double' in spec: + options.append('-DGMX_DOUBLE:BOOL=ON') + + if '~shared' in spec: + options.append('-DBUILD_SHARED_LIBS:BOOL=OFF') + + if '+debug' in spec: + options.append('-DCMAKE_BUILD_TYPE:STRING=Debug') + else: + options.append('-DCMAKE_BUILD_TYPE:STRING=Release') + + options.extend(std_cmake_args) + + with working_dir('spack-build', create=True): + + cmake('..', *options) + make() + make('install') From 5c8dd6c3c86400ac82b061ce45f58b543526bfff Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Sat, 20 Feb 2016 17:18:49 -0800 Subject: [PATCH 456/910] llvm trunk version Adding a trunk version to the llvm package. This has all the features and requirements of the others, with the additional caveat that the llvm project makes no guarantee that trunk on all repositories together will necessarily make a working compiler. It has been tested, and worked with a version today, but not yesterday, so if you test keep that in mind. --- .../repos/builtin/packages/llvm/package.py | 41 +++++++++++++++---- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index 1805d3ded80..a8f19f90711 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -117,6 +117,21 @@ class Llvm(Package): }, } releases = [ + { + 'version' : 'trunk', + 'repo' : 'http://llvm.org/svn/llvm-project/llvm/trunk', + 'resources' : { + 'compiler-rt' : 'http://llvm.org/svn/llvm-project/compiler-rt/trunk', + 'openmp' : 'http://llvm.org/svn/llvm-project/openmp/trunk', + 'polly' : 'http://llvm.org/svn/llvm-project/polly/trunk', + 'libcxx' : 'http://llvm.org/svn/llvm-project/libcxx/trunk', + 'libcxxabi' : 'http://llvm.org/svn/llvm-project/libcxxabi/trunk', + 'clang' : 'http://llvm.org/svn/llvm-project/cfe/trunk', + 'clang-tools-extra' : 'http://llvm.org/svn/llvm-project/clang-tools-extra/trunk', + 'lldb' : 'http://llvm.org/svn/llvm-project/lldb/trunk', + 'llvm-libunwind' : 'http://llvm.org/svn/llvm-project/libunwind/trunk', + } + }, { 'version' : '3.7.0', 'md5':'b98b9495e5655a672d6cb83e1a180f8e', @@ -161,15 +176,25 @@ class Llvm(Package): ] for release in releases: - version(release['version'], release['md5'], url=llvm_url % release) + if release['version'] == 'trunk' : + version(release['version'], svn=release['repo']) - for name, md5 in release['resources'].items(): - resource(name=name, - url=resources[name]['url'] % release, - md5=md5, - destination=resources[name]['destination'], - when='@%(version)s' % release, - placement=resources[name].get('placement', None)) + for name, repo in release['resources'].items(): + resource(name=name, + svn=repo, + destination=resources[name]['destination'], + when='@%(version)s' % release, + placement=resources[name].get('placement', None)) + else: + version(release['version'], release['md5'], url=llvm_url % release) + + for name, md5 in release['resources'].items(): + resource(name=name, + url=resources[name]['url'] % release, + md5=md5, + destination=resources[name]['destination'], + when='@%(version)s' % release, + placement=resources[name].get('placement', None)) # SVN - current develop version('develop', svn='http://llvm.org/svn/llvm-project/llvm/trunk') From 4316f1cd31afd18e35b5ebd9852a156d2c68c8df Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 23 Feb 2016 16:57:19 +0100 Subject: [PATCH 457/910] Modified wrapper to have a different behavior when modeis vcheck --- lib/spack/env/cc | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index aacba996b39..dfb0801c569 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -113,14 +113,22 @@ case "$command" in ;; esac -# Finish setting up the mode. +# If any of the arguments below is present then the mode is vcheck. In vcheck mode nothing is added in terms of extra search paths or libraries if [ -z "$mode" ]; then - mode=ccld for arg in "$@"; do if [ "$arg" = -v -o "$arg" = -V -o "$arg" = --version -o "$arg" = -dumpversion ]; then mode=vcheck break - elif [ "$arg" = -E ]; then + fi + done +fi + +# Finish setting up the mode. + +if [ -z "$mode" ]; then + mode=ccld + for arg in "$@"; do + if [ "$arg" = -E ]; then mode=cpp break elif [ "$arg" = -c ]; then @@ -145,6 +153,11 @@ fi # Save original command for debug logging input_command="$@" +if [ "$mode" == vcheck ] ; then + exec "${input_command}" + exit +fi + # # Now do real parsing of the command line args, trying hard to keep # non-rpath linker arguments in the proper order w.r.t. other command @@ -330,3 +343,5 @@ if [ "$SPACK_DEBUG" = "TRUE" ]; then fi exec "${full_command[@]}" + + From 5ce97eeda52944128dc32c0d7728550d95520b0d Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 24 Feb 2016 09:17:04 +0100 Subject: [PATCH 458/910] intel : polished code --- lib/spack/env/cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index dfb0801c569..644d2be1d66 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -155,7 +155,6 @@ input_command="$@" if [ "$mode" == vcheck ] ; then exec "${input_command}" - exit fi # @@ -343,5 +342,3 @@ if [ "$SPACK_DEBUG" = "TRUE" ]; then fi exec "${full_command[@]}" - - From 976ae91dccd36d1feb6d0d20cd1ef5b3470c4c39 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 24 Feb 2016 14:11:57 +0100 Subject: [PATCH 459/910] llvm : removed duplicate version --- .../repos/builtin/packages/llvm/package.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index a8f19f90711..934d994bd3e 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -196,25 +196,6 @@ class Llvm(Package): when='@%(version)s' % release, placement=resources[name].get('placement', None)) - # SVN - current develop - version('develop', svn='http://llvm.org/svn/llvm-project/llvm/trunk') - resource(name='clang', svn='http://llvm.org/svn/llvm-project/cfe/trunk', - destination='tools', when='@develop', placement='clang') - resource(name='compiler-rt', svn='http://llvm.org/svn/llvm-project/compiler-rt/trunk', - destination='projects', when='@develop', placement='compiler-rt') - resource(name='openmp', svn='http://llvm.org/svn/llvm-project/openmp/trunk', - destination='projects', when='@develop', placement='openmp') - resource(name='libcxx', svn='http://llvm.org/svn/llvm-project/libcxx/trunk', - destination='projects', when='@develop', placement='libcxx') - resource(name='libcxxabi', svn='http://llvm.org/svn/llvm-project/libcxxabi/trunk', - destination='projects', when='@develop', placement='libcxxabi') - resource(name='polly', svn='http://llvm.org/svn/llvm-project/polly/trunk', - destination='tools', when='@develop', placement='polly') - resource(name='lldb', svn='http://llvm.org/svn/llvm-project/lldb/trunk', - destination='tools', when='@develop', placement='lldb') - - - def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag cmake_args = [ arg for arg in std_cmake_args if 'BUILD_TYPE' not in arg ] From 6f42dd556d1ead8e7cad9788004dc33c11240564 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 24 Feb 2016 17:37:58 +0100 Subject: [PATCH 460/910] stage : on-going refactoring --- lib/spack/spack/package.py | 149 +++++++++++++++++----------------- lib/spack/spack/stage.py | 68 ++++++++++------ lib/spack/spack/test/stage.py | 122 +++++++++++----------------- 3 files changed, 167 insertions(+), 172 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8019b29cba5..4fdc582479d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -847,93 +847,94 @@ def do_install(self, make_jobs=make_jobs) start_time = time.time() - if not fake: - if not skip_patch: - self.do_patch() - else: - self.do_stage() - - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) - - def cleanup(): - if not keep_prefix: - # If anything goes wrong, remove the install prefix - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + - "Manually remove this directory to fix:", - self.prefix, wrap=True) - - - def real_work(): - try: - tty.msg("Building %s." % self.name) - - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) - - # Set up process's build environment before running install. - if fake: - self.do_fake_install() + with self.stage: + if not fake: + if not skip_patch: + self.do_patch() else: - # Do the real install in the source directory. - self.stage.chdir_to_source() + self.do_stage() - # This redirects I/O to a build log (and optionally to the terminal) - log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), True): - self.install(self.spec, self.prefix) + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) - # Ensure that something was actually installed. - self._sanity_check_install() + def cleanup(): + if not keep_prefix: + # If anything goes wrong, remove the install prefix + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed." + + "Manually remove this directory to fix:", + self.prefix, wrap=True) - # Move build log into install directory on success - if not fake: - log_install_path = spack.install_layout.build_log_path(self.spec) - install(log_path, log_install_path) - # On successful install, remove the stage. - if not keep_stage: - self.stage.destroy() + def real_work(): + try: + tty.msg("Building %s." % self.name) - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) - tty.msg("Successfully installed %s." % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) + # Set up process's build environment before running install. + if fake: + self.do_fake_install() + else: + # Do the real install in the source directory. + self.stage.chdir_to_source() - except ProcessError, e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e + # This redirects I/O to a build log (and optionally to the terminal) + log_path = join_path(os.getcwd(), 'spack-build.out') + log_file = open(log_path, 'w') + with log_output(log_file, verbose, sys.stdout.isatty(), True): + self.install(self.spec, self.prefix) - except: - # other exceptions just clean up and raise. - cleanup() - raise + # Ensure that something was actually installed. + self._sanity_check_install() - # Set parallelism before starting build. - self.make_jobs = make_jobs + # Move build log into install directory on success + if not fake: + log_install_path = spack.install_layout.build_log_path(self.spec) + install(log_path, log_install_path) - # Do the build. - spack.build_environment.fork(self, real_work) + # On successful install, remove the stage. + if not keep_stage: + self.stage.destroy() - # note: PARENT of the build process adds the new package to - # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, self.prefix) + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - # Once everything else is done, run post install hooks - spack.hooks.post_install(self) + tty.msg("Successfully installed %s." % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) + + except ProcessError, e: + # Annotate with location of build log. + e.build_log = log_path + cleanup() + raise e + + except: + # other exceptions just clean up and raise. + cleanup() + raise + + # Set parallelism before starting build. + self.make_jobs = make_jobs + + # Do the build. + spack.build_environment.fork(self, real_work) + + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, self.prefix) + + # Once everything else is done, run post install hooks + spack.hooks.post_install(self) def _sanity_check_install(self): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index f217450d42d..96b1eaf3f2c 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -96,12 +96,44 @@ def __init__(self, url_or_fetch_strategy, **kwargs): self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. - self.name = kwargs.get('name') + # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name + # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root + self.name = kwargs.get('name') if 'name' in kwargs else STAGE_PREFIX + next(tempfile._get_candidate_names()) self.mirror_path = kwargs.get('mirror_path') self.tmp_root = find_tmp_root() - self.path = None + # Try to construct here a temporary name for the stage directory + # If this is a named stage, then construct a named path. + self.path = join_path(spack.stage_path, self.name) + + self.delete_on_exit = True + + def __enter__(self): + """ + Entering a stage context will create the stage directory + """ + # FIXME : if _setup is used only here, then it makes no sense to retain the function self._setup() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Exiting from a stage context will delete the stage directory unless: + - it was explicitly requested not to do so + - an exception has been raised + + Args: + exc_type: exception type + exc_val: exception value + exc_tb: exception traceback + + Returns: + Boolean + """ + self.delete_on_exit = False if exc_type is not None else self.delete_on_exit + + if self.delete_on_exit: + self.destroy() def _cleanup_dead_links(self): """Remove any dead links in the stage directory.""" @@ -163,35 +195,17 @@ def _setup(self): mkdirp(spack.stage_path) self._cleanup_dead_links() - # If this is a named stage, then construct a named path. - if self.name is not None: - self.path = join_path(spack.stage_path, self.name) - # If this is a temporary stage, them make the temp directory - tmp_dir = None if self.tmp_root: - if self.name is None: - # Unnamed tmp root. Link the path in + if self._need_to_create_path(): tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - self.name = os.path.basename(tmp_dir) - self.path = join_path(spack.stage_path, self.name) - if self._need_to_create_path(): - os.symlink(tmp_dir, self.path) - - else: - if self._need_to_create_path(): - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) + os.symlink(tmp_dir, self.path) # if we're not using a tmp dir, create the stage directly in the # stage dir, rather than linking to it. else: - if self.name is None: - self.path = tempfile.mkdtemp('', STAGE_PREFIX, spack.stage_path) - self.name = os.path.basename(self.path) - else: - if self._need_to_create_path(): - mkdirp(self.path) + if self._need_to_create_path(): + mkdirp(self.path) # Make sure we can actually do something with the stage we made. ensure_access(self.path) @@ -389,6 +403,12 @@ def source_path(self): def path(self): return self[0].path + def __enter__(self): + return self[0].__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + return self[0].__exit__(exc_type, exc_val, exc_tb) + def chdir_to_source(self): return self[0].chdir_to_source() diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index c1b2a2a5730..a7314eba4c5 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -192,116 +192,90 @@ def check_destroy(self, stage, stage_name): def test_setup_and_destroy_name_with_tmp(self): with use_tmp(True): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + self.check_setup(stage, stage_name) self.check_destroy(stage, stage_name) def test_setup_and_destroy_name_without_tmp(self): with use_tmp(False): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + self.check_setup(stage, stage_name) self.check_destroy(stage, stage_name) def test_setup_and_destroy_no_name_with_tmp(self): with use_tmp(True): - stage = Stage(archive_url) - self.check_setup(stage, None) - - stage.destroy() + with Stage(archive_url) as stage: + self.check_setup(stage, None) self.check_destroy(stage, None) def test_setup_and_destroy_no_name_without_tmp(self): with use_tmp(False): - stage = Stage(archive_url) - self.check_setup(stage, None) - - stage.destroy() + with Stage(archive_url) as stage: + self.check_setup(stage, None) self.check_destroy(stage, None) def test_chdir(self): - stage = Stage(archive_url, name=stage_name) - - stage.chdir() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.chdir() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) self.check_destroy(stage, stage_name) def test_fetch(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) self.check_destroy(stage, stage_name) def test_expand_archive(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.expand_archive() - self.check_expand_archive(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.expand_archive() + self.check_expand_archive(stage, stage_name) self.check_destroy(stage, stage_name) def test_expand_archive(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) self.check_destroy(stage, stage_name) def test_restage(self): - stage = Stage(archive_url, name=stage_name) + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) - stage.fetch() - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) + # Try to make a file in the old archive dir + with open('foobar', 'w') as file: + file.write("this file is to be destroyed.") - # Try to make a file in the old archive dir - with open('foobar', 'w') as file: - file.write("this file is to be destroyed.") + self.assertTrue('foobar' in os.listdir(stage.source_path)) - self.assertTrue('foobar' in os.listdir(stage.source_path)) - - # Make sure the file is not there after restage. - stage.restage() - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.chdir_to_source() - self.check_chdir_to_source(stage, stage_name) - self.assertFalse('foobar' in os.listdir(stage.source_path)) - - stage.destroy() + # Make sure the file is not there after restage. + stage.restage() + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.chdir_to_source() + self.check_chdir_to_source(stage, stage_name) + self.assertFalse('foobar' in os.listdir(stage.source_path)) self.check_destroy(stage, stage_name) From 20845a739f15190cf9610da4375d3e3c9fc61b6b Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 24 Feb 2016 22:18:51 -0500 Subject: [PATCH 461/910] Libevent depends on OpenSSL --- var/spack/repos/builtin/packages/libevent/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/libevent/package.py b/var/spack/repos/builtin/packages/libevent/package.py index 11b1083d672..2a44c49325d 100644 --- a/var/spack/repos/builtin/packages/libevent/package.py +++ b/var/spack/repos/builtin/packages/libevent/package.py @@ -23,6 +23,9 @@ class Libevent(Package): version('2.0.12', '42986228baf95e325778ed328a93e070') + depends_on('openssl') + + def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 30d9ca2bde033da28b6f6105f93ee05f4fd0acfa Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Fri, 26 Feb 2016 16:06:17 -0800 Subject: [PATCH 462/910] Updated the silo package and added the '+fortran' variant. --- .../repos/builtin/packages/silo/package.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/silo/package.py b/var/spack/repos/builtin/packages/silo/package.py index 9eda11df15f..d1aed78e0e8 100644 --- a/var/spack/repos/builtin/packages/silo/package.py +++ b/var/spack/repos/builtin/packages/silo/package.py @@ -1,19 +1,28 @@ from spack import * class Silo(Package): - """Silo is a library for reading and writing a wide variety of scientific data to binary, disk files.""" + """Silo is a library for reading and writing a wide variety of scientific + data to binary, disk files.""" homepage = "http://wci.llnl.gov/simulation/computer-codes/silo" url = "https://wci.llnl.gov/content/assets/docs/simulation/computer-codes/silo/silo-4.8/silo-4.8.tar.gz" - #version('4.9', 'a83eda4f06761a86726e918fc55e782a') version('4.8', 'b1cbc0e7ec435eb656dc4b53a23663c9') - depends_on("hdf5@:1.8.12") + variant('fortran', default=True, description='Enable Fortran support') + + depends_on("hdf5") def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-hdf5=%s" %spec['hdf5'].prefix) + config_args = [ + '--enable-fortran' if '+fortran' in spec else '--disable-fortran', + ] + + configure( + "--prefix=%s" % prefix, + "--with-hdf5=%s,%s" % (spec['hdf5'].prefix.include, spec['hdf5'].prefix.lib), + "--with-zlib=%s,%s" % (spec['zlib'].prefix.include, spec['zlib'].prefix.lib), + *config_args) make() make("install") From 8f3ac9ac8b7b1d16672e7a52f691966de1f8483f Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Sat, 27 Feb 2016 01:40:32 +0100 Subject: [PATCH 463/910] adding new version to praview, compiled with spack -d install -j 8 --keep-stage paraview@5.0.0+qt+python+tcl+opengl2%gcc@4.8.2 ^netcdf -mpi --- .../repos/builtin/packages/paraview/package.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index e43bdd44930..ccf2d14c06e 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -2,9 +2,11 @@ class Paraview(Package): homepage = 'http://www.paraview.org' - url = 'http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz' + url = 'http://www.paraview.org/files/v5.0/ParaView-v' + _url_str = 'http://www.paraview.org/files/v%s/ParaView-v%s-source.tar.gz' - version('4.4.0', 'fa1569857dd680ebb4d7ff89c2227378', url='http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz') + version('4.4.0', 'fa1569857dd680ebb4d7ff89c2227378') + version('5.0.0', '4598f0b421460c8bbc635c9a1c3bdbee') variant('python', default=False, description='Enable Python support') @@ -25,8 +27,8 @@ class Paraview(Package): depends_on('bzip2') depends_on('freetype') - depends_on('hdf5') depends_on('hdf5+mpi', when='+mpi') + depends_on('hdf5~mpi', when='~mpi') depends_on('jpeg') depends_on('libpng') depends_on('libtiff') @@ -35,6 +37,11 @@ class Paraview(Package): #depends_on('protobuf') # version mismatches? #depends_on('sqlite') # external version not supported depends_on('zlib') + + def url_for_version(self, version): + """Handle ParaView version-based custom URLs.""" + return self._url_str % (version.up_to(2), version) + def install(self, spec, prefix): with working_dir('spack-build', create=True): From 67f327f805868d369eec0392631392281b68c39d Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Sun, 28 Feb 2016 19:47:19 -0800 Subject: [PATCH 464/910] Updated links to use new llnl.gov address --- README.md | 4 ++-- lib/spack/docs/getting_started.rst | 2 +- lib/spack/spack/cmd/repo.py | 2 +- lib/spack/spack/repository.py | 2 +- lib/spack/spack/resource.py | 2 +- lib/spack/spack/test/namespace_trie.py | 2 +- lib/spack/spack/test/tally_plugin.py | 8 ++++---- var/spack/repos/builtin/packages/gdb/package.py | 2 +- var/spack/repos/builtin/packages/texinfo/package.py | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index bdce345764f..8664953c0cc 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ written in pure Python, and specs allow package authors to write a single build script for many different builds of the same package. See the -[Feature Overview](http://llnl.github.io/spack/features.html) +[Feature Overview](http://software.llnl.gov/spack/features.html) for examples and highlights. To install spack and install your first package: @@ -31,7 +31,7 @@ To install spack and install your first package: Documentation ---------------- -[**Full documentation**](http://llnl.github.io/spack) for Spack is +[**Full documentation**](http://software.llnl.gov/spack) for Spack is the first place to look. See also: diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index 67ca18e71ae..2c5b68ea650 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -22,7 +22,7 @@ go: $ spack install libelf For a richer experience, use Spack's `shell support -`_: +`_: .. code-block:: sh diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 34c755fb67f..908f5400ab2 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index f58cd52125b..e8d0cc09ec2 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py index 2bf92947fd3..ddfaaf4cb09 100644 --- a/lib/spack/spack/resource.py +++ b/lib/spack/spack/resource.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index d0d809004d5..647976df210 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index 9ca898c47ce..e0b9618e0cb 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://scalability-software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -34,7 +34,7 @@ def __init__(self): self.successCount = 0 self.failCount = 0 self.errorCount = 0 - + @property def numberOfTestsRun(self): """Excludes skipped tests""" @@ -48,10 +48,10 @@ def configure(self, options, conf): def addSuccess(self, test): self.successCount += 1 - + def addError(self, test, err): self.errorCount += 1 - + def addFailure(self, test, err): self.failCount += 1 diff --git a/var/spack/repos/builtin/packages/gdb/package.py b/var/spack/repos/builtin/packages/gdb/package.py index dd02b426b94..b346fe80c28 100644 --- a/var/spack/repos/builtin/packages/gdb/package.py +++ b/var/spack/repos/builtin/packages/gdb/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/repos/builtin/packages/texinfo/package.py b/var/spack/repos/builtin/packages/texinfo/package.py index a83c10c0c19..6cf8d790722 100644 --- a/var/spack/repos/builtin/packages/texinfo/package.py +++ b/var/spack/repos/builtin/packages/texinfo/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify From a339ac0a727458ab55b2e0d27ff17dcc139c0a32 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Mon, 29 Feb 2016 22:29:30 -0500 Subject: [PATCH 465/910] Bug Fix: When Spack create roots around for other versions, it sometimes finds files it thinks are tarballs, but are not. Previously, it would crash if any such files are found. This change allows it to simply skip them and move on, processing the rest of the files it finds correctly. --- lib/spack/spack/cmd/checksum.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index b1ad89dbb8a..c4519932335 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -58,24 +58,29 @@ def get_checksums(versions, urls, **kwargs): tty.msg("Downloading...") hashes = [] - for i, (url, version) in enumerate(zip(urls, versions)): + i = 0 + for url, version in zip(urls, versions): stage = Stage(url) try: stage.fetch() if i == 0 and first_stage_function: first_stage_function(stage) - hashes.append( - spack.util.crypto.checksum(hashlib.md5, stage.archive_file)) + hashes.append((version, + spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) except FailedDownloadError, e: tty.msg("Failed to fetch %s" % url) continue + except Exception, e: + tty.msg('Something failed on %s, skipping.\n (%s)' % (url, e)) + continue finally: if not keep_stage: stage.destroy() + i += 1 - return zip(versions, hashes) + return hashes From e414c5fdfbc91e5934ee59233475f8f862e2e0ce Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Mon, 29 Feb 2016 22:31:18 -0500 Subject: [PATCH 466/910] Added missing cmake dependencies --- var/spack/repos/builtin/packages/cgal/package.py | 1 + var/spack/repos/builtin/packages/curl/package.py | 6 +++--- var/spack/repos/builtin/packages/expat/package.py | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/cgal/package.py b/var/spack/repos/builtin/packages/cgal/package.py index 97356433be1..ef4a2736dbb 100644 --- a/var/spack/repos/builtin/packages/cgal/package.py +++ b/var/spack/repos/builtin/packages/cgal/package.py @@ -46,6 +46,7 @@ class Cgal(Package): depends_on('mpfr') depends_on('gmp') depends_on('zlib') + depends_on('cmake') # FIXME : Qt5 dependency missing (needs Qt5 and OpenGL) # FIXME : Optional third party libraries missing diff --git a/var/spack/repos/builtin/packages/curl/package.py b/var/spack/repos/builtin/packages/curl/package.py index 9e684445c71..6c302281a77 100644 --- a/var/spack/repos/builtin/packages/curl/package.py +++ b/var/spack/repos/builtin/packages/curl/package.py @@ -13,13 +13,13 @@ class Curl(Package): version('7.43.0', '11bddbb452a8b766b932f859aaeeed39') version('7.42.1', '296945012ce647b94083ed427c1877a8') - depends_on("openssl") +# depends_on("openssl") depends_on("zlib") def install(self, spec, prefix): configure('--prefix=%s' % prefix, - '--with-zlib=%s' % spec['zlib'].prefix, - '--with-ssl=%s' % spec['openssl'].prefix) + '--with-zlib=%s' % spec['zlib'].prefix) +# '--with-ssl=%s' % spec['openssl'].prefix) make() make("install") diff --git a/var/spack/repos/builtin/packages/expat/package.py b/var/spack/repos/builtin/packages/expat/package.py index 082da5bf0be..3f925c65461 100644 --- a/var/spack/repos/builtin/packages/expat/package.py +++ b/var/spack/repos/builtin/packages/expat/package.py @@ -7,6 +7,7 @@ class Expat(Package): version('2.1.0', 'dd7dab7a5fea97d2a6a43f511449b7cd') + depends_on('cmake') def install(self, spec, prefix): From 15ae92aae9c8ac521bb5d7ee5f210d265494d87a Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Mon, 29 Feb 2016 22:59:28 -0500 Subject: [PATCH 467/910] New packages added: blitz, netcdf-cxx4, netcdf-fortran, proj, udunits2 --- .../repos/builtin/packages/blitz/package.py | 40 ++++++++++++++++++ .../builtin/packages/netcdf-cxx4/package.py | 41 +++++++++++++++++++ .../packages/netcdf-fortran/package.py | 21 ++++++++++ .../repos/builtin/packages/proj/package.py | 37 +++++++++++++++++ .../builtin/packages/udunits2/package.py | 16 ++++++++ 5 files changed, 155 insertions(+) create mode 100644 var/spack/repos/builtin/packages/blitz/package.py create mode 100644 var/spack/repos/builtin/packages/netcdf-cxx4/package.py create mode 100644 var/spack/repos/builtin/packages/netcdf-fortran/package.py create mode 100644 var/spack/repos/builtin/packages/proj/package.py create mode 100644 var/spack/repos/builtin/packages/udunits2/package.py diff --git a/var/spack/repos/builtin/packages/blitz/package.py b/var/spack/repos/builtin/packages/blitz/package.py new file mode 100644 index 00000000000..82ff6349250 --- /dev/null +++ b/var/spack/repos/builtin/packages/blitz/package.py @@ -0,0 +1,40 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install blitz +# +# You can always get back here to change things with: +# +# spack edit blitz +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class Blitz(Package): + """N-dimensional arrays for C++""" + homepage = "http://github.com/blitzpp/blitz" + +# This version doesn't have the configure script generated yet. + url = "https://github.com/blitzpp/blitz/tarball/1.0.0" +#http://prdownloads.sourceforge.net/%(namelower)s + + version('1.0.0', '9f040b9827fe22228a892603671a77af') + + # FIXME: Add dependencies if this package requires them. + # depends_on("foo") + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + # FIXME: Spack couldn't guess one, so here are some options: + configure('--prefix=%s' % prefix) + # cmake('.', *std_cmake_args) + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py new file mode 100644 index 00000000000..9d70eab05f5 --- /dev/null +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -0,0 +1,41 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install netcdf-cxx4 +# +# You can always get back here to change things with: +# +# spack edit netcdf-cxx4 +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class NetcdfCxx4(Package): + """C++ interface for NetCDF4""" + homepage = "http://www.unidata.ucar.edu/downloads/netcdf/netcdf-cxx/index.jsp" + url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx4-4.2.tar.gz" + + version('4.2', 'd019853802092cf686254aaba165fc81') + + + variant('mpi', default=True, description='Enables MPI parallelism') +# variant('hdf4', default=False, description="Enable HDF4 support") + + # NetCDF-CXX4 doesn't really depend (directly) on MPI. However... this + # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) + depends_on('mpi', when='+mpi') + depends_on('netcdf') + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + configure('--prefix=%s' % prefix) + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py new file mode 100644 index 00000000000..8e5c8ecc3a0 --- /dev/null +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -0,0 +1,21 @@ +from spack import * + +class NetcdfFortran(Package): + """Fortran interface for NetCDF4""" + + homepage = "http://www.unidata.ucar.edu/downloads/netcdf/netcdf-cxx/index.jsp" + url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-fortran-4.4.3.tar.gz" + + version('4.4.3', 'bfd4ae23a34635b273d3eb0d91cbde9e') + + variant('mpi', default=True, description='Enables MPI parallelism') + + # NetCDF-CXX4 doesn't really depend (directly) on MPI. However... this + # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) + depends_on('mpi', when='+mpi') + depends_on('netcdf') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/proj/package.py b/var/spack/repos/builtin/packages/proj/package.py new file mode 100644 index 00000000000..4a0d3feac75 --- /dev/null +++ b/var/spack/repos/builtin/packages/proj/package.py @@ -0,0 +1,37 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install proj +# +# You can always get back here to change things with: +# +# spack edit proj +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class Proj(Package): + """Cartographic Projections""" + homepage = "https://github.com/OSGeo/proj.4/wiki" + url = "http://download.osgeo.org/proj/proj-4.9.2.tar.gz" + + version('4.9.2', '9843131676e31bbd903d60ae7dc76cf9') + version('4.9.1', '3cbb2a964fd19a496f5f4265a717d31c') + version('4.8.0', 'd815838c92a29179298c126effbb1537') + version('4.7.0', '927d34623b52e0209ba2bfcca18fe8cd') + version('4.6.1', '7dbaab8431ad50c25669fd3fb28dc493') + + # FIXME: Add dependencies if this package requires them. + # depends_on("foo") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/udunits2/package.py b/var/spack/repos/builtin/packages/udunits2/package.py new file mode 100644 index 00000000000..9954a733bb5 --- /dev/null +++ b/var/spack/repos/builtin/packages/udunits2/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Udunits2(Package): + """Automated units conversion""" + + homepage = "http://www.unidata.ucar.edu/software/udunits" + url = "ftp://ftp.unidata.ucar.edu/pub/udunits/udunits-2.2.20.tar.gz" + + version('2.2.20', '1586b70a49dfe05da5fcc29ef239dce0') + + depends_on('expat') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") From eb1d90a8cfadeb556754b57e5d21e5605b9a0e9d Mon Sep 17 00:00:00 2001 From: citibeth Date: Mon, 29 Feb 2016 23:31:54 -0500 Subject: [PATCH 468/910] Undid accidental change on this branch. --- var/spack/repos/builtin/packages/curl/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/curl/package.py b/var/spack/repos/builtin/packages/curl/package.py index 6c302281a77..9e684445c71 100644 --- a/var/spack/repos/builtin/packages/curl/package.py +++ b/var/spack/repos/builtin/packages/curl/package.py @@ -13,13 +13,13 @@ class Curl(Package): version('7.43.0', '11bddbb452a8b766b932f859aaeeed39') version('7.42.1', '296945012ce647b94083ed427c1877a8') -# depends_on("openssl") + depends_on("openssl") depends_on("zlib") def install(self, spec, prefix): configure('--prefix=%s' % prefix, - '--with-zlib=%s' % spec['zlib'].prefix) -# '--with-ssl=%s' % spec['openssl'].prefix) + '--with-zlib=%s' % spec['zlib'].prefix, + '--with-ssl=%s' % spec['openssl'].prefix) make() make("install") From a9f0b98d4d30e44032f5b9c86c5b6e27425098f5 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 1 Mar 2016 15:53:53 -0600 Subject: [PATCH 469/910] Typo fix --- lib/spack/env/cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index aacba996b39..a323c481243 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -130,7 +130,7 @@ if [ -z "$mode" ]; then done fi -# Dump the version and exist if we're in testing mode. +# Dump the version and exit if we're in testing mode. if [ "$SPACK_TEST_COMMAND" = "dump-mode" ]; then echo "$mode" exit From 8174489787c56cee1726ca36799c236e4869f471 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Tue, 1 Mar 2016 15:25:57 -0700 Subject: [PATCH 470/910] + Provide two new variants for cmake: 1) +qt - build the cmake-gui Qt application. - adds a dependency on Qt. 2) +sphinxbuild - build the html CMake documentation. - adds a dependency on python and py-sphinx --- .../repos/builtin/packages/cmake/package.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index e20c1e4aeb0..f39a6812843 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -37,16 +37,34 @@ class Cmake(Package): version('2.8.10.2', '097278785da7182ec0aea8769d06860c') variant('ncurses', default=True, description='Enables the build of the ncurses gui') + variant('qt', default=False, description='Enables the build of cmake-gui') + variant('sphinxbuild', default=False, description='Enables the generation of html and man page documentation') + depends_on('ncurses', when='+ncurses') + depends_on('qt', when='+qt') + depends_on('python@2.7.11:', when='+sphinxbuild') + depends_on('py-sphinx', when='+sphinxbuild') def url_for_version(self, version): """Handle CMake's version-based custom URLs.""" return 'https://cmake.org/files/v%s/cmake-%s.tar.gz' % (version.up_to(2), version) - def install(self, spec, prefix): - configure('--prefix=' + prefix, - '--parallel=' + str(make_jobs), - '--', '-DCMAKE_USE_OPENSSL=ON') + + options = ['--prefix=%s' % prefix] + options.append('--parallel=%s' % str(make_jobs)) + + if '+qt' in spec: + options.append('--qt-gui') + + if '+sphinxbuild' in spec: + options.append('--sphinx-html') + options.append('--sphinx-man') + + options.append('--') + options.append('-DCMAKE_USE_OPENSSL=ON') + + configure(*options) + make() make('install') From c488f7c4d8e2ff240d561820df11a51518199a53 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 1 Mar 2016 23:57:34 -0800 Subject: [PATCH 471/910] Fix bug in install permission setting. --- lib/spack/llnl/util/filesystem.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 24cfbfde719..da3cf96050f 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -152,15 +152,20 @@ def set_install_permissions(path): def copy_mode(src, dest): src_mode = os.stat(src).st_mode dest_mode = os.stat(dest).st_mode - if src_mode | stat.S_IXUSR: dest_mode |= stat.S_IXUSR - if src_mode | stat.S_IXGRP: dest_mode |= stat.S_IXGRP - if src_mode | stat.S_IXOTH: dest_mode |= stat.S_IXOTH + if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR + if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP + if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH os.chmod(dest, dest_mode) def install(src, dest): """Manually install a file to a particular location.""" tty.debug("Installing %s to %s" % (src, dest)) + + # Expand dsst to its eventual full path if it is a directory. + if os.path.isdir(dest): + dest = join_path(dest, os.path.basename(src)) + shutil.copy(src, dest) set_install_permissions(dest) copy_mode(src, dest) From be306d09e99a69732afc3f44724222ab6c6d71cc Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 2 Mar 2016 00:04:46 -0800 Subject: [PATCH 472/910] Move repo creation code into repository.py --- lib/spack/spack/cmd/repo.py | 46 +-------------------------- lib/spack/spack/repository.py | 58 ++++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 46 deletions(-) diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 908f5400ab2..c2e352786d1 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -74,51 +74,7 @@ def setup_parser(subparser): def repo_create(args): """Create a new package repository.""" - root = canonicalize_path(args.directory) - namespace = args.namespace - - if not args.namespace: - namespace = os.path.basename(root) - - if not re.match(r'\w[\.\w-]*', namespace): - tty.die("'%s' is not a valid namespace." % namespace) - - existed = False - if os.path.exists(root): - if os.path.isfile(root): - tty.die('File %s already exists and is not a directory' % root) - elif os.path.isdir(root): - if not os.access(root, os.R_OK | os.W_OK): - tty.die('Cannot create new repo in %s: cannot access directory.' % root) - if os.listdir(root): - tty.die('Cannot create new repo in %s: directory is not empty.' % root) - existed = True - - full_path = os.path.realpath(root) - parent = os.path.dirname(full_path) - if not os.access(parent, os.R_OK | os.W_OK): - tty.die("Cannot create repository in %s: can't access parent!" % root) - - try: - config_path = os.path.join(root, repo_config_name) - packages_path = os.path.join(root, packages_dir_name) - - mkdirp(packages_path) - with open(config_path, 'w') as config: - config.write("repo:\n") - config.write(" namespace: '%s'\n" % namespace) - - except (IOError, OSError) as e: - tty.die('Failed to create new repository in %s.' % root, - "Caused by %s: %s" % (type(e), e)) - - # try to clean up. - if existed: - shutil.rmtree(config_path, ignore_errors=True) - shutil.rmtree(packages_path, ignore_errors=True) - else: - shutil.rmtree(root, ignore_errors=True) - + full_path, namespace = create_repo(args.directory, args.namespace) tty.msg("Created repo with namespace '%s'." % namespace) tty.msg("To register it with spack, run this command:", 'spack repo add %s' % full_path) diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index e8d0cc09ec2..6aa75cb43e5 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -33,7 +33,7 @@ from external import yaml import llnl.util.tty as tty -from llnl.util.filesystem import join_path +from llnl.util.filesystem import * import spack.error import spack.config @@ -705,6 +705,58 @@ def __contains__(self, pkg_name): return self.exists(pkg_name) +def create_repo(root, namespace=None): + """Create a new repository in root with the specified namespace. + + If the namespace is not provided, use basename of root. + Return the canonicalized path and the namespace of the created repository. + """ + root = canonicalize_path(root) + if not namespace: + namespace = os.path.basename(root) + + if not re.match(r'\w[\.\w-]*', namespace): + raise InvalidNamespaceError("'%s' is not a valid namespace." % namespace) + + existed = False + if os.path.exists(root): + if os.path.isfile(root): + raise BadRepoError('File %s already exists and is not a directory' % root) + elif os.path.isdir(root): + if not os.access(root, os.R_OK | os.W_OK): + raise BadRepoError('Cannot create new repo in %s: cannot access directory.' % root) + if os.listdir(root): + raise BadRepoError('Cannot create new repo in %s: directory is not empty.' % root) + existed = True + + full_path = os.path.realpath(root) + parent = os.path.dirname(full_path) + if not os.access(parent, os.R_OK | os.W_OK): + raise BadRepoError("Cannot create repository in %s: can't access parent!" % root) + + try: + config_path = os.path.join(root, repo_config_name) + packages_path = os.path.join(root, packages_dir_name) + + mkdirp(packages_path) + with open(config_path, 'w') as config: + config.write("repo:\n") + config.write(" namespace: '%s'\n" % namespace) + + except (IOError, OSError) as e: + raise BadRepoError('Failed to create new repository in %s.' % root, + "Caused by %s: %s" % (type(e), e)) + + # try to clean up. + if existed: + shutil.rmtree(config_path, ignore_errors=True) + shutil.rmtree(packages_path, ignore_errors=True) + else: + shutil.rmtree(root, ignore_errors=True) + + return full_path, namespace + + class RepoError(spack.error.SpackError): """Superclass for repository-related errors.""" @@ -713,6 +765,10 @@ class NoRepoConfiguredError(RepoError): """Raised when there are no repositories configured.""" +class InvalidNamespaceError(RepoError): + """Raised when an invalid namespace is encountered.""" + + class BadRepoError(RepoError): """Raised when repo layout is invalid.""" From 21d125c9147f5d3cfc0eb8a9516c8ce24df7279b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 2 Mar 2016 00:08:36 -0800 Subject: [PATCH 473/910] Fixes #175: Dump environment provenance as well as build log. --- lib/spack/spack/directory_layout.py | 14 +++++++++++++- lib/spack/spack/package.py | 7 +++++++ lib/spack/spack/util/environment.py | 7 +++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 3e416a6a1fc..29d87b65b33 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -173,7 +173,9 @@ def __init__(self, root, **kwargs): self.spec_file_name = 'spec.yaml' self.extension_file_name = 'extensions.yaml' - self.build_log_name = 'build.out' # TODO: use config file. + self.build_log_name = 'build.out' # build log. + self.build_env_name = 'build.env' # build environment + self.packages_dir = 'repos' # archive of package.py files # Cache of already written/read extension maps. self._extension_maps = {} @@ -231,6 +233,16 @@ def build_log_path(self, spec): self.build_log_name) + def build_env_path(self, spec): + return join_path(self.path_for_spec(spec), self.metadata_dir, + self.build_env_name) + + + def build_packages_path(self, spec): + return join_path(self.path_for_spec(spec), self.metadata_dir, + self.packages_dir) + + def create_install_directory(self, spec): _check_concrete(spec) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8019b29cba5..5b1927fe8f4 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -66,6 +66,7 @@ from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive, extension from spack.util.executable import ProcessError +from spack.util.environment import dump_environment """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] @@ -884,10 +885,14 @@ def real_work(): # Do the real install in the source directory. self.stage.chdir_to_source() + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') + # This redirects I/O to a build log (and optionally to the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') log_file = open(log_path, 'w') with log_output(log_file, verbose, sys.stdout.isatty(), True): + dump_environment(env_path) self.install(self.spec, self.prefix) # Ensure that something was actually installed. @@ -896,7 +901,9 @@ def real_work(): # Move build log into install directory on success if not fake: log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) install(log_path, log_install_path) + install(env_path, env_install_path) # On successful install, remove the stage. if not keep_stage: diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index cd413dcfbc8..ae8e5708bef 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -63,3 +63,10 @@ def pop_keys(dictionary, *keys): for key in keys: if key in dictionary: dictionary.pop(key) + + +def dump_environment(path): + """Dump the current environment out to a file.""" + with open(path, 'w') as env_file: + for key,val in sorted(os.environ.items()): + env_file.write("%s=%s\n" % (key, val)) From 52081c46d641fcf679e70a04e9937db67e8787e4 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 2 Mar 2016 00:09:24 -0800 Subject: [PATCH 474/910] Fixes #469: Store package.py files in the .spack directory. - Adds packages in spack repos inside the .spack directory, so that packages can be rebuilt automatically later. --- lib/spack/spack/package.py | 51 +++++++++++++++++++++++++++++++++++ lib/spack/spack/repository.py | 39 +++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 5b1927fe8f4..9f1825ca211 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -58,6 +58,7 @@ import spack.mirror import spack.hooks import spack.directives +import spack.repository import spack.build_environment import spack.url import spack.util.web @@ -502,6 +503,7 @@ def fetcher(self): self._fetcher = self._make_fetcher() return self._fetcher + @fetcher.setter def fetcher(self, f): self._fetcher = f @@ -905,6 +907,9 @@ def real_work(): install(log_path, log_install_path) install(env_path, env_install_path) + packages_dir = spack.install_layout.build_packages_path(self.spec) + dump_packages(self.spec, packages_dir) + # On successful install, remove the stage. if not keep_stage: self.stage.destroy() @@ -1219,6 +1224,52 @@ def validate_package_url(url_string): tty.die("Invalid file type in URL: '%s'" % url_string) +def dump_packages(spec, path): + """Dump all package information for a spec and its dependencies. + + This creates a package repository within path for every + namespace in the spec DAG, and fills the repos wtih package + files and patch files for every node in the DAG. + """ + mkdirp(path) + + # Copy in package.py files from any dependencies. + # Note that we copy them in as they are in the *install* directory + # NOT as they are in the repository, because we want a snapshot of + # how *this* particular build was done. + for node in spec.traverse(): + if node is not spec: + # Locate the dependency package in the install tree and find + # its provenance information. + source = spack.install_layout.build_packages_path(node) + source_repo_root = join_path(source, node.namespace) + + # There's no provenance installed for the source package. Skip it. + # User can always get something current from the builtin repo. + if not os.path.isdir(source_repo_root): + continue + + # Create a source repo and get the pkg directory out of it. + try: + source_repo = spack.repository.Repo(source_repo_root) + source_pkg_dir = source_repo.dirname_for_package_name(node.name) + except RepoError as e: + tty.warn("Warning: Couldn't copy in provenance for %s" % node.name) + + # Create a destination repository + dest_repo_root = join_path(path, node.namespace) + if not os.path.exists(dest_repo_root): + spack.repository.create_repo(dest_repo_root) + repo = spack.repository.Repo(dest_repo_root) + + # Get the location of the package in the dest repo. + dest_pkg_dir = repo.dirname_for_package_name(node.name) + if node is not spec: + install_tree(source_pkg_dir, dest_pkg_dir) + else: + spack.repo.dump_provenance(node, dest_pkg_dir) + + def print_pkg(message): """Outputs a message with a package icon.""" from llnl.util.tty.color import cwrite diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 6aa75cb43e5..8d06fefe7f0 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -316,6 +316,16 @@ def get(self, spec, new=False): return self.repo_for_pkg(spec).get(spec) + @_autospec + def dump_provenance(self, spec, path): + """Dump provenance information for a spec to a particular path. + + This dumps the package file and any associated patch files. + Raises UnknownPackageError if not found. + """ + return self.repo_for_pkg(spec).dump_provenance(spec, path) + + def dirname_for_package_name(self, pkg_name): return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) @@ -552,6 +562,35 @@ def get(self, spec, new=False): return self._instances[key] + @_autospec + def dump_provenance(self, spec, path): + """Dump provenance information for a spec to a particular path. + + This dumps the package file and any associated patch files. + Raises UnknownPackageError if not found. + """ + # Some preliminary checks. + if spec.virtual: + raise UnknownPackageError(spec.name) + + if spec.namespace and spec.namespace != self.namespace: + raise UnknownPackageError("Repository %s does not contain package %s." + % (self.namespace, spec.fullname)) + + # Install any patch files needed by packages. + mkdirp(path) + for spec, patches in spec.package.patches.items(): + for patch in patches: + if patch.path: + if os.path.exists(patch.path): + install(patch.path, path) + else: + tty.warn("Patch file did not exist: %s" % patch.path) + + # Install the package.py file itself. + install(self.filename_for_package_name(spec), path) + + def purge(self): """Clear entire package instance cache.""" self._instances.clear() From a0c6519de9d03744e33b5187d42ec0d966dbd3e0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 2 Mar 2016 01:41:43 -0800 Subject: [PATCH 475/910] Fixes #476: create was broken for FTP URLs. --- lib/spack/spack/util/web.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index e26daef2964..73f4858b023 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -86,12 +86,12 @@ def _spider(args): if not "Content-type" in resp.headers: tty.debug("ignoring page " + url) - return pages + return pages, links if not resp.headers["Content-type"].startswith('text/html'): tty.debug("ignoring page " + url + " with content type " + resp.headers["Content-type"]) - return pages + return pages, links # Do the real GET request when we know it's just HTML. req.get_method = lambda: "GET" @@ -173,7 +173,7 @@ def spider(root_url, **kwargs): performance over a sequential fetch. """ max_depth = kwargs.setdefault('depth', 1) - pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False)) + pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False)) return pages, links From 726b350689bf6da7b82eec170001adc8100beb71 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 12:52:38 +0100 Subject: [PATCH 476/910] test : fixed failing unit tests --- lib/spack/spack/mirror.py | 49 +++++++++---------- lib/spack/spack/stage.py | 7 ++- lib/spack/spack/test/configure_guess.py | 12 ++--- lib/spack/spack/test/git_fetch.py | 31 +++++++------ lib/spack/spack/test/hg_fetch.py | 31 +++++++------ lib/spack/spack/test/link_tree.py | 6 +-- lib/spack/spack/test/mirror.py | 62 ++++++++++++------------- lib/spack/spack/test/svn_fetch.py | 31 +++++++------ 8 files changed, 116 insertions(+), 113 deletions(-) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index fa29e208036..bc8870926f9 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -168,32 +168,33 @@ def create(path, specs, **kwargs): pkg = spec.package tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) try: - for ii, stage in enumerate(pkg.stage): - fetcher = stage.fetcher - if ii == 0: - # create a subdirectory for the current package@version - archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) - name = spec.format("$_$@") - else: - resource = stage.resource - archive_path = join_path(subdir, suggest_archive_basename(resource)) - name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) - subdir = os.path.dirname(archive_path) - mkdirp(subdir) + with pkg.stage: + for ii, stage in enumerate(pkg.stage): + fetcher = stage.fetcher + if ii == 0: + # create a subdirectory for the current package@version + archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) + name = spec.format("$_$@") + else: + resource = stage.resource + archive_path = join_path(subdir, suggest_archive_basename(resource)) + name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) + subdir = os.path.dirname(archive_path) + mkdirp(subdir) - if os.path.exists(archive_path): - tty.msg("{name} : already added".format(name=name)) - else: - everything_already_exists = False - fetcher.fetch() - if not kwargs.get('no_checksum', False): - fetcher.check() - tty.msg("{name} : checksum passed".format(name=name)) + if os.path.exists(archive_path): + tty.msg("{name} : already added".format(name=name)) + else: + everything_already_exists = False + fetcher.fetch() + if not kwargs.get('no_checksum', False): + fetcher.check() + tty.msg("{name} : checksum passed".format(name=name)) - # Fetchers have to know how to archive their files. Use - # that to move/copy/create an archive in the mirror. - fetcher.archive(archive_path) - tty.msg("{name} : added".format(name=name)) + # Fetchers have to know how to archive their files. Use + # that to move/copy/create an archive in the mirror. + fetcher.archive(archive_path) + tty.msg("{name} : added".format(name=name)) if everything_already_exists: present.append(spec) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 96b1eaf3f2c..956d1c87069 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -404,10 +404,13 @@ def path(self): return self[0].path def __enter__(self): - return self[0].__enter__() + for item in self: + item.__enter__() + return self def __exit__(self, exc_type, exc_val, exc_tb): - return self[0].__exit__(exc_type, exc_val, exc_tb) + for item in reversed(self): + item.__exit__(exc_type, exc_val, exc_tb) def chdir_to_source(self): return self[0].chdir_to_source() diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index a4e8565b62b..bc2332acc23 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -52,8 +52,6 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) - if self.stage: - self.stage.destroy() os.chdir(self.orig_dir) @@ -64,12 +62,12 @@ def check_archive(self, filename, system): url = 'file://' + join_path(os.getcwd(), 'archive.tar.gz') print url - self.stage = Stage(url) - self.stage.fetch() + with Stage(url) as stage: + stage.fetch() - guesser = ConfigureGuesser() - guesser(self.stage) - self.assertEqual(system, guesser.build_system) + guesser = ConfigureGuesser() + guesser(stage) + self.assertEqual(system, guesser.build_system) def test_python(self): diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index d84433176a8..caa076823e4 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -76,26 +76,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('git')] = args - self.pkg.do_stage() - self.assert_rev(rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked_file = 'foobarbaz' - touch(untracked_file) - self.assertTrue(os.path.isfile(untracked_file)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked_file)) + untracked_file = 'foobarbaz' + touch(untracked_file) + self.assertTrue(os.path.isfile(untracked_file)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked_file)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_master(self): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index bbcb64e4c1a..75aa7ab17e7 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -68,26 +68,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('hg')] = args - self.pkg.do_stage() - self.assertEqual(self.repo.get_rev(), rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assertEqual(self.repo.get_rev(), rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assertEqual(self.repo.get_rev(), rev) + self.assertEqual(self.repo.get_rev(), rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index 886b7ef4c5d..7b67e873dde 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -38,6 +38,8 @@ class LinkTreeTest(unittest.TestCase): def setUp(self): self.stage = Stage('link-tree-test') + # FIXME : possibly this test needs to be refactored to avoid the explicit call to __enter__ and __exit__ + self.stage.__enter__() with working_dir(self.stage.path): touchp('source/1') @@ -51,10 +53,8 @@ def setUp(self): source_path = os.path.join(self.stage.path, 'source') self.link_tree = LinkTree(source_path) - def tearDown(self): - if self.stage: - self.stage.destroy() + self.stage.__exit__(None, None, None) def check_file_link(self, filename): diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index f83cc8090c9..9e2c6319782 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -74,14 +74,14 @@ def set_up_package(self, name, MockRepoClass, url_attr): def check_mirror(self): - stage = Stage('spack-mirror-test') - mirror_root = join_path(stage.path, 'test-mirror') + with Stage('spack-mirror-test') as stage: + mirror_root = join_path(stage.path, 'test-mirror') + + # register mirror with spack config + mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } + spack.config.update_config('mirrors', mirrors) - # register mirror with spack config - mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } - spack.config.update_config('mirrors', mirrors) - try: os.chdir(stage.path) spack.mirror.create( mirror_root, self.repos, no_checksum=True) @@ -97,38 +97,36 @@ def check_mirror(self): files = os.listdir(subdir) self.assertEqual(len(files), 1) - # Now try to fetch each package. - for name, mock_repo in self.repos.items(): - spec = Spec(name).concretized() - pkg = spec.package + # Now try to fetch each package. + for name, mock_repo in self.repos.items(): + spec = Spec(name).concretized() + pkg = spec.package - pkg._stage = None - saved_checksum_setting = spack.do_checksum - try: - # Stage the archive from the mirror and cd to it. - spack.do_checksum = False - pkg.do_stage(mirror_only=True) + saved_checksum_setting = spack.do_checksum + with pkg.stage: + try: + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) - # Compare the original repo with the expanded archive - original_path = mock_repo.path - if 'svn' in name: - # have to check out the svn repo to compare. - original_path = join_path(mock_repo.path, 'checked_out') - svn('checkout', mock_repo.url, original_path) + # Compare the original repo with the expanded archive + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) - dcmp = dircmp(original_path, pkg.stage.source_path) + dcmp = dircmp(original_path, pkg.stage.source_path) - # make sure there are no new files in the expanded tarball - self.assertFalse(dcmp.right_only) + # make sure there are no new files in the expanded tarball + self.assertFalse(dcmp.right_only) - # and that all original files are present. - self.assertTrue(all(l in exclude for l in dcmp.left_only)) + # and that all original files are present. + self.assertTrue(all(l in exclude for l in dcmp.left_only)) - finally: - spack.do_checksum = saved_checksum_setting - pkg.do_clean() - finally: - stage.destroy() + finally: + spack.do_checksum = saved_checksum_setting + pkg.do_clean() def test_git_mirror(self): diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 454a7f1d1ff..6ac9e2f3431 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -82,26 +82,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('svn')] = args - self.pkg.do_stage() - self.assert_rev(rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_default(self): From 65b2a24f7c12380d0815e5705d1c1e66e5fd22a9 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 13:03:40 +0100 Subject: [PATCH 477/910] stage : removed _setup method --- lib/spack/spack/stage.py | 57 ++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 956d1c87069..48770fb4079 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -111,9 +111,32 @@ def __init__(self, url_or_fetch_strategy, **kwargs): def __enter__(self): """ Entering a stage context will create the stage directory + + If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise + this will attempt to create a stage in a temporary directory and link it into spack.stage_path. + + Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location + in tmp_dirs, fall back to making the stage inside spack.stage_path. """ - # FIXME : if _setup is used only here, then it makes no sense to retain the function - self._setup() + # Create the top-level stage directory + mkdirp(spack.stage_path) + self._cleanup_dead_links() + + # If this is a temporary stage, them make the temp directory + if self.tmp_root: + if self._need_to_create_path(): + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + os.symlink(tmp_dir, self.path) + + # if we're not using a tmp dir, create the stage directly in the + # stage dir, rather than linking to it. + else: + if self._need_to_create_path(): + mkdirp(self.path) + + # Make sure we can actually do something with the stage we made. + ensure_access(self.path) + return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -180,36 +203,6 @@ def _need_to_create_path(self): return False - def _setup(self): - """Creates the stage directory. - If spack.use_tmp_stage is False, the stage directory is created - directly under spack.stage_path. - - If spack.use_tmp_stage is True, this will attempt to create a - stage in a temporary directory and link it into spack.stage_path. - Spack will use the first writable location in spack.tmp_dirs to - create a stage. If there is no valid location in tmp_dirs, fall - back to making the stage inside spack.stage_path. - """ - # Create the top-level stage directory - mkdirp(spack.stage_path) - self._cleanup_dead_links() - - # If this is a temporary stage, them make the temp directory - if self.tmp_root: - if self._need_to_create_path(): - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) - - # if we're not using a tmp dir, create the stage directly in the - # stage dir, rather than linking to it. - else: - if self._need_to_create_path(): - mkdirp(self.path) - - # Make sure we can actually do something with the stage we made. - ensure_access(self.path) - @property def archive_file(self): """Path to the source archive within this stage directory.""" From d649b715edae0871254695e2deacf078554a1475 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 13:16:04 +0100 Subject: [PATCH 478/910] stage : updated functions doc _cleanup_dead_links : fixed minor bug --- lib/spack/spack/stage.py | 41 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 48770fb4079..900acd664da 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -42,33 +42,26 @@ class Stage(object): - """A Stage object manages a directory where some source code is - downloaded and built before being installed. It handles - fetching the source code, either as an archive to be expanded - or by checking it out of a repository. A stage's lifecycle - looks like this: + """ + A Stage object is a context manager that handles a directory where some source code is downloaded and built + before being installed. It handles fetching the source code, either as an archive to be expanded or by checking + it out of a repository. A stage's lifecycle looks like this: - Stage() - Constructor creates the stage directory. - fetch() - Fetch a source archive into the stage. - expand_archive() - Expand the source archive. - - Build and install the archive. This is handled by the Package class. - destroy() - Remove the stage once the package has been installed. + ``` + with Stage() as stage: # Context manager creates and destroys the stage directory + fetch() # Fetch a source archive into the stage. + expand_archive() # Expand the source archive. + # Build and install the archive. This is handled by the Package class. + ``` - If spack.use_tmp_stage is True, spack will attempt to create stages - in a tmp directory. Otherwise, stages are created directly in - spack.stage_path. + If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory. + Otherwise, stages are created directly in spack.stage_path. - There are two kinds of stages: named and unnamed. Named stages can - persist between runs of spack, e.g. if you fetched a tarball but - didn't finish building it, you won't have to fetch it again. + There are two kinds of stages: named and unnamed. Named stages can persist between runs of spack, e.g. if you + fetched a tarball but didn't finish building it, you won't have to fetch it again. - Unnamed stages are created using standard mkdtemp mechanisms or - similar, and are intended to persist for only one run of spack. + Unnamed stages are created using standard mkdtemp mechanisms or similar, and are intended to persist for + only one run of spack. """ def __init__(self, url_or_fetch_strategy, **kwargs): @@ -164,7 +157,7 @@ def _cleanup_dead_links(self): path = join_path(spack.stage_path, file) if os.path.islink(path): real_path = os.path.realpath(path) - if not os.path.exists(path): + if not os.path.exists(real_path): os.unlink(path) def _need_to_create_path(self): From 9001b9ed3c01f8ccaceaca60d6a34c3551f77240 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 15:56:09 +0100 Subject: [PATCH 479/910] package : minor syntax fixes mirror : extracted add_single_spec from create --- lib/spack/spack/mirror.py | 98 ++++++++++++++++++++------------------ lib/spack/spack/package.py | 4 +- 2 files changed, 54 insertions(+), 48 deletions(-) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index bc8870926f9..5ed7aff176b 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -110,7 +110,6 @@ def suggest_archive_basename(resource): return basename - def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with package archives. @@ -158,58 +157,65 @@ def create(path, specs, **kwargs): "Cannot create directory '%s':" % mirror_root, str(e)) # Things to keep track of while parsing specs. - present = [] - mirrored = [] - error = [] + categories = { + 'present': [], + 'mirrored': [], + 'error': [] + } # Iterate through packages and download all the safe tarballs for each of them - everything_already_exists = True for spec in version_specs: - pkg = spec.package - tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) - try: - with pkg.stage: - for ii, stage in enumerate(pkg.stage): - fetcher = stage.fetcher - if ii == 0: - # create a subdirectory for the current package@version - archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) - name = spec.format("$_$@") - else: - resource = stage.resource - archive_path = join_path(subdir, suggest_archive_basename(resource)) - name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) - subdir = os.path.dirname(archive_path) - mkdirp(subdir) + add_single_spec(spec, mirror_root, categories, **kwargs) - if os.path.exists(archive_path): - tty.msg("{name} : already added".format(name=name)) - else: - everything_already_exists = False - fetcher.fetch() - if not kwargs.get('no_checksum', False): - fetcher.check() - tty.msg("{name} : checksum passed".format(name=name)) + return categories['present'], categories['mirrored'], categories['error'] - # Fetchers have to know how to archive their files. Use - # that to move/copy/create an archive in the mirror. - fetcher.archive(archive_path) - tty.msg("{name} : added".format(name=name)) - if everything_already_exists: - present.append(spec) - else: - mirrored.append(spec) - except Exception, e: - if spack.debug: - sys.excepthook(*sys.exc_info()) - else: - tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) - error.append(spec) - finally: - pkg.stage.destroy() +def add_single_spec(spec, mirror_root, categories, **kwargs): + tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) + spec_exists_in_mirror = True + try: + with spec.package.stage: + # fetcher = stage.fetcher + # fetcher.fetch() + # ... + # fetcher.archive(archive_path) + for ii, stage in enumerate(spec.package.stage): + fetcher = stage.fetcher + if ii == 0: + # create a subdirectory for the current package@version + archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) + name = spec.format("$_$@") + else: + resource = stage.resource + archive_path = join_path(subdir, suggest_archive_basename(resource)) + name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) + subdir = os.path.dirname(archive_path) + mkdirp(subdir) - return (present, mirrored, error) + if os.path.exists(archive_path): + tty.msg("{name} : already added".format(name=name)) + else: + spec_exists_in_mirror = False + fetcher.fetch() + if not kwargs.get('no_checksum', False): + fetcher.check() + tty.msg("{name} : checksum passed".format(name=name)) + + # Fetchers have to know how to archive their files. Use + # that to move/copy/create an archive in the mirror. + fetcher.archive(archive_path) + tty.msg("{name} : added".format(name=name)) + + if spec_exists_in_mirror: + categories['present'].append(spec) + else: + categories['mirrored'].append(spec) + except Exception as e: + if spack.debug: + sys.excepthook(*sys.exc_info()) + else: + tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) + categories['error'].append(spec) class MirrorError(spack.error.SpackError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 4fdc582479d..29b781a7499 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -732,7 +732,7 @@ def do_patch(self): # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. if os.path.isfile(bad_file): - tty.msg("Patching failed last time. Restaging.") + tty.msg("Patching failed last time. Restaging.") self.stage.restage() self.stage.chdir_to_source() @@ -912,7 +912,7 @@ def real_work(): % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) print_pkg(self.prefix) - except ProcessError, e: + except ProcessError as e: # Annotate with location of build log. e.build_log = log_path cleanup() From 901e4851b993f73c4bdd92b85b2469d6673b7617 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 16:19:32 +0100 Subject: [PATCH 480/910] _cleanup_dead_links : factored method into a function and put it in llnl.filesystem --- lib/spack/llnl/util/filesystem.py | 19 +++++++++++++++++-- lib/spack/spack/stage.py | 20 ++++---------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 24cfbfde719..4a708b639a3 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links'] import os import sys @@ -235,7 +235,7 @@ def touchp(path): def force_symlink(src, dest): try: os.symlink(src, dest) - except OSError, e: + except OSError as e: os.remove(dest) os.symlink(src, dest) @@ -339,3 +339,18 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): if order == 'post': yield (source_path, dest_path) + +def remove_dead_links(root): + """ + Removes any dead link that is present in root + + Args: + root: path where to search for dead links + + """ + for file in os.listdir(root): + path = join_path(root, file) + if os.path.islink(path): + real_path = os.path.realpath(path) + if not os.path.exists(real_path): + os.unlink(path) \ No newline at end of file diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 900acd664da..e9106431920 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -98,7 +98,7 @@ def __init__(self, url_or_fetch_strategy, **kwargs): # Try to construct here a temporary name for the stage directory # If this is a named stage, then construct a named path. self.path = join_path(spack.stage_path, self.name) - + # Flag to decide whether to delete the stage folder on exit or not self.delete_on_exit = True def __enter__(self): @@ -113,20 +113,17 @@ def __enter__(self): """ # Create the top-level stage directory mkdirp(spack.stage_path) - self._cleanup_dead_links() + remove_dead_links(spack.stage_path) - # If this is a temporary stage, them make the temp directory + # If a tmp_root exists then create a directory there and then link it in the stage area, + # otherwise create the stage directory in self.path if self.tmp_root: if self._need_to_create_path(): tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) os.symlink(tmp_dir, self.path) - - # if we're not using a tmp dir, create the stage directly in the - # stage dir, rather than linking to it. else: if self._need_to_create_path(): mkdirp(self.path) - # Make sure we can actually do something with the stage we made. ensure_access(self.path) @@ -151,15 +148,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): if self.delete_on_exit: self.destroy() - def _cleanup_dead_links(self): - """Remove any dead links in the stage directory.""" - for file in os.listdir(spack.stage_path): - path = join_path(spack.stage_path, file) - if os.path.islink(path): - real_path = os.path.realpath(path) - if not os.path.exists(real_path): - os.unlink(path) - def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we looked at path. Returns True if path already exists and is ok. From 4d63544fe918393de9265f8879bb0199a542cc3b Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 16:55:57 +0100 Subject: [PATCH 481/910] remove_link_tree : moved to llnl.util.filesystem --- lib/spack/llnl/util/filesystem.py | 20 ++++++++++++++++++-- lib/spack/spack/stage.py | 21 +++------------------ 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 4a708b639a3..015eeb9aa1c 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links', 'remove_linked_tree'] import os import sys @@ -353,4 +353,20 @@ def remove_dead_links(root): if os.path.islink(path): real_path = os.path.realpath(path) if not os.path.exists(real_path): - os.unlink(path) \ No newline at end of file + os.unlink(path) + +def remove_linked_tree(path): + """ + Removes a directory and its contents. If the directory is a symlink, follows the link and removes the real + directory before removing the link. + + Args: + path: directory to be removed + + """ + if os.path.exists(path): + if os.path.islink(path): + shutil.rmtree(os.path.realpath(path), True) + os.unlink(path) + else: + shutil.rmtree(path, True) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index e9106431920..e87b822a8f4 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -114,15 +114,13 @@ def __enter__(self): # Create the top-level stage directory mkdirp(spack.stage_path) remove_dead_links(spack.stage_path) - # If a tmp_root exists then create a directory there and then link it in the stage area, # otherwise create the stage directory in self.path - if self.tmp_root: - if self._need_to_create_path(): + if self._need_to_create_path(): + if self.tmp_root: tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) os.symlink(tmp_dir, self.path) - else: - if self._need_to_create_path(): + else: mkdirp(self.path) # Make sure we can actually do something with the stage we made. ensure_access(self.path) @@ -436,19 +434,6 @@ def ensure_access(file=spack.stage_path): tty.die("Insufficient permissions for %s" % file) -def remove_linked_tree(path): - """Removes a directory and its contents. If the directory is a symlink, - follows the link and reamoves the real directory before removing the - link. - """ - if os.path.exists(path): - if os.path.islink(path): - shutil.rmtree(os.path.realpath(path), True) - os.unlink(path) - else: - shutil.rmtree(path, True) - - def purge(): """Remove all build directories in the top-level stage path.""" if os.path.isdir(spack.stage_path): From ca41909ec52ab2b8f82ca91421d3882300531ac2 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 17:19:27 +0100 Subject: [PATCH 482/910] package : removed do_clean() --- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/package.py | 9 +------- lib/spack/spack/test/git_fetch.py | 4 ---- lib/spack/spack/test/hg_fetch.py | 3 --- lib/spack/spack/test/mirror.py | 38 ++++++++++++------------------- lib/spack/spack/test/svn_fetch.py | 4 ---- 6 files changed, 17 insertions(+), 43 deletions(-) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 6e7179122c1..0c8bd1d5281 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -43,4 +43,4 @@ def clean(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.do_clean() + package.stage.destroy() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 29b781a7499..fa64b2c0472 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1138,13 +1138,6 @@ def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() - - def do_clean(self): - """Removes the package's build stage and source tarball.""" - if os.path.exists(self.stage.path): - self.stage.destroy() - - def format_doc(self, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get('indent', 0) @@ -1181,7 +1174,7 @@ def fetch_remote_versions(self): try: return spack.util.web.find_versions_of_archive( *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) - except spack.error.NoNetworkConnectionError, e: + except spack.error.NoNetworkConnectionError as e: tty.die("Package.fetch_versions couldn't connect to:", e.url, e.message) diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index caa076823e4..6d6a67a1d38 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -52,19 +52,15 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def assert_rev(self, rev): """Check that the current git revision is equal to the supplied rev.""" self.assertEqual(self.repo.rev_hash('HEAD'), self.repo.rev_hash(rev)) - def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 75aa7ab17e7..d884ed78a0f 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -49,13 +49,10 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(HgFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def try_fetch(self, rev, test_file, args): """Tries to: diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 9e2c6319782..f117e042424 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -104,29 +104,21 @@ def check_mirror(self): saved_checksum_setting = spack.do_checksum with pkg.stage: - try: - # Stage the archive from the mirror and cd to it. - spack.do_checksum = False - pkg.do_stage(mirror_only=True) - - # Compare the original repo with the expanded archive - original_path = mock_repo.path - if 'svn' in name: - # have to check out the svn repo to compare. - original_path = join_path(mock_repo.path, 'checked_out') - svn('checkout', mock_repo.url, original_path) - - dcmp = dircmp(original_path, pkg.stage.source_path) - - # make sure there are no new files in the expanded tarball - self.assertFalse(dcmp.right_only) - - # and that all original files are present. - self.assertTrue(all(l in exclude for l in dcmp.left_only)) - - finally: - spack.do_checksum = saved_checksum_setting - pkg.do_clean() + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) + # Compare the original repo with the expanded archive + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) + dcmp = dircmp(original_path, pkg.stage.source_path) + # make sure there are no new files in the expanded tarball + self.assertFalse(dcmp.right_only) + # and that all original files are present. + self.assertTrue(all(l in exclude for l in dcmp.left_only)) + spack.do_checksum = saved_checksum_setting def test_git_mirror(self): diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 6ac9e2f3431..13a00f5df79 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -51,13 +51,10 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(SvnFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def assert_rev(self, rev): """Check that the current revision is equal to the supplied rev.""" @@ -70,7 +67,6 @@ def get_rev(): return match.group(1) self.assertEqual(get_rev(), rev) - def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with From 21cd05aad5e8f45a7ebac7b213855a16685d229f Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 17:28:21 +0100 Subject: [PATCH 483/910] package : updated doc --- lib/spack/spack/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index fa64b2c0472..0214dcd7716 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -291,7 +291,6 @@ class SomePackage(Package): .. code-block:: python - p.do_clean() # removes the stage directory entirely p.do_restage() # removes the build directory and # re-expands the archive. From 6bbadbfaac406f9abe81e64f174d8a2b4c9c255e Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 2 Mar 2016 13:18:43 -0600 Subject: [PATCH 484/910] Add CPATH pointing to include directories. --- lib/spack/docs/basic_usage.rst | 2 +- lib/spack/spack/modules.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 0578f0c8db6..f94ac3d2ba8 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -896,7 +896,7 @@ Or, similarly with modules, you could type: $ spack load mpich %gcc@4.4.7 These commands will add appropriate directories to your ``PATH``, -``MANPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use +``MANPATH``, ``CPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use a package, you can type unload or unuse similarly: .. code-block:: sh diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index c8347635646..c27043db8cd 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -33,6 +33,7 @@ * /bin directories to be appended to PATH * /lib* directories for LD_LIBRARY_PATH + * /include directories for CPATH * /man* and /share/man* directories for MANPATH * the package prefix for CMAKE_PREFIX_PATH @@ -121,6 +122,7 @@ def add_path(path_name, directory): ('LIBRARY_PATH', self.spec.prefix.lib64), ('LD_LIBRARY_PATH', self.spec.prefix.lib), ('LD_LIBRARY_PATH', self.spec.prefix.lib64), + ('CPATH', self.spec.prefix.include), ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib, 'pkgconfig')), ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib64, 'pkgconfig'))]: From 976d0240c467b9ff74333fafcd2ac1e896819a80 Mon Sep 17 00:00:00 2001 From: citibeth Date: Wed, 2 Mar 2016 18:58:47 -0500 Subject: [PATCH 485/910] Removed FIXMEs. --- .../repos/builtin/packages/blitz/package.py | 27 +------------------ .../builtin/packages/netcdf-cxx4/package.py | 21 +-------------- .../packages/netcdf-fortran/package.py | 2 +- .../repos/builtin/packages/proj/package.py | 19 +------------ 4 files changed, 4 insertions(+), 65 deletions(-) diff --git a/var/spack/repos/builtin/packages/blitz/package.py b/var/spack/repos/builtin/packages/blitz/package.py index 82ff6349250..9413b276fe9 100644 --- a/var/spack/repos/builtin/packages/blitz/package.py +++ b/var/spack/repos/builtin/packages/blitz/package.py @@ -1,40 +1,15 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install blitz -# -# You can always get back here to change things with: -# -# spack edit blitz -# -# See the spack documentation for more information on building -# packages. -# from spack import * class Blitz(Package): """N-dimensional arrays for C++""" homepage = "http://github.com/blitzpp/blitz" - -# This version doesn't have the configure script generated yet. url = "https://github.com/blitzpp/blitz/tarball/1.0.0" -#http://prdownloads.sourceforge.net/%(namelower)s version('1.0.0', '9f040b9827fe22228a892603671a77af') - # FIXME: Add dependencies if this package requires them. - # depends_on("foo") + # No dependencies def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. - # FIXME: Spack couldn't guess one, so here are some options: configure('--prefix=%s' % prefix) - # cmake('.', *std_cmake_args) - - # FIXME: Add logic to build and install here make() make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py index 9d70eab05f5..8d51a106794 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -1,19 +1,3 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install netcdf-cxx4 -# -# You can always get back here to change things with: -# -# spack edit netcdf-cxx4 -# -# See the spack documentation for more information on building -# packages. -# from spack import * class NetcdfCxx4(Package): @@ -25,17 +9,14 @@ class NetcdfCxx4(Package): variant('mpi', default=True, description='Enables MPI parallelism') -# variant('hdf4', default=False, description="Enable HDF4 support") - # NetCDF-CXX4 doesn't really depend (directly) on MPI. However... this + # netcdf-cxx4 doesn't really depend (directly) on MPI. However... this # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) depends_on('mpi', when='+mpi') depends_on('netcdf') def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. configure('--prefix=%s' % prefix) - # FIXME: Add logic to build and install here make() make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py index 8e5c8ecc3a0..9e4aee95fb8 100644 --- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -10,7 +10,7 @@ class NetcdfFortran(Package): variant('mpi', default=True, description='Enables MPI parallelism') - # NetCDF-CXX4 doesn't really depend (directly) on MPI. However... this + # netcdf-fortran doesn't really depend (directly) on MPI. However... this # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) depends_on('mpi', when='+mpi') depends_on('netcdf') diff --git a/var/spack/repos/builtin/packages/proj/package.py b/var/spack/repos/builtin/packages/proj/package.py index 4a0d3feac75..797772f4f6f 100644 --- a/var/spack/repos/builtin/packages/proj/package.py +++ b/var/spack/repos/builtin/packages/proj/package.py @@ -1,19 +1,3 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install proj -# -# You can always get back here to change things with: -# -# spack edit proj -# -# See the spack documentation for more information on building -# packages. -# from spack import * class Proj(Package): @@ -27,8 +11,7 @@ class Proj(Package): version('4.7.0', '927d34623b52e0209ba2bfcca18fe8cd') version('4.6.1', '7dbaab8431ad50c25669fd3fb28dc493') - # FIXME: Add dependencies if this package requires them. - # depends_on("foo") + # No dependencies def install(self, spec, prefix): configure('--prefix=%s' % prefix) From 717bcb862428a1b1a3b9c70f925c3fe6c325f622 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 3 Mar 2016 09:17:49 +0100 Subject: [PATCH 486/910] test : optimized import statements --- lib/spack/spack/test/concretize.py | 2 -- lib/spack/spack/test/config.py | 6 ++--- lib/spack/spack/test/configure_guess.py | 9 ++------ lib/spack/spack/test/database.py | 11 +++++---- lib/spack/spack/test/directory_layout.py | 13 +++++------ lib/spack/spack/test/git_fetch.py | 11 ++------- lib/spack/spack/test/hg_fetch.py | 10 +++------ lib/spack/spack/test/install.py | 9 ++------ lib/spack/spack/test/link_tree.py | 7 ++---- lib/spack/spack/test/lock.py | 8 +++---- lib/spack/spack/test/make_executable.py | 6 ++--- lib/spack/spack/test/mirror.py | 5 ++--- lib/spack/spack/test/mock_packages_test.py | 8 +++---- lib/spack/spack/test/mock_repo.py | 4 ---- lib/spack/spack/test/multimethod.py | 5 +---- lib/spack/spack/test/namespace_trie.py | 1 + lib/spack/spack/test/optional_deps.py | 4 +--- lib/spack/spack/test/packages.py | 26 ++++++---------------- lib/spack/spack/test/python_version.py | 3 +-- lib/spack/spack/test/spec_dag.py | 2 -- lib/spack/spack/test/spec_semantics.py | 1 - lib/spack/spack/test/spec_syntax.py | 3 ++- lib/spack/spack/test/stage.py | 8 +++---- lib/spack/spack/test/svn_fetch.py | 14 ++++-------- lib/spack/spack/test/tally_plugin.py | 4 ++-- lib/spack/spack/test/unit_install.py | 3 ++- lib/spack/spack/test/url_extrapolate.py | 3 --- lib/spack/spack/test/url_parse.py | 2 +- lib/spack/spack/test/url_substitution.py | 1 - lib/spack/spack/test/versions.py | 1 + lib/spack/spack/test/yaml.py | 1 + 31 files changed, 62 insertions(+), 129 deletions(-) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 7f2938aec5a..794344fb6ae 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -22,8 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest - import spack from spack.spec import Spec, CompilerSpec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index d8be5a855b8..0562d2d6209 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -22,13 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest -import shutil import os +import shutil from tempfile import mkdtemp -from ordereddict_backport import OrderedDict + import spack import spack.config +from ordereddict_backport import OrderedDict from spack.test.mock_packages_test import * # Some sample compiler config data diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index bc2332acc23..2440d120e5d 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -23,20 +23,15 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest import shutil import tempfile +import unittest from llnl.util.filesystem import * - from spack.cmd.create import ConfigureGuesser from spack.stage import Stage - -from spack.fetch_strategy import URLFetchStrategy -from spack.directory_layout import YamlDirectoryLayout -from spack.util.executable import which from spack.test.mock_packages_test import * -from spack.test.mock_repo import MockArchive +from spack.util.executable import which class InstallTest(unittest.TestCase): diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 0205f4b8ce2..9a57e1f03ea 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -26,19 +26,18 @@ These tests check the database is functioning properly, both in memory and in its file """ -import tempfile -import shutil import multiprocessing - -from llnl.util.lock import * -from llnl.util.filesystem import join_path +import shutil +import tempfile import spack +from llnl.util.filesystem import join_path +from llnl.util.lock import * +from llnl.util.tty.colify import colify from spack.database import Database from spack.directory_layout import YamlDirectoryLayout from spack.test.mock_packages_test import * -from llnl.util.tty.colify import colify def _print_ref_counts(): """Print out all ref counts for the graph used here, for debugging""" diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 925cb648edd..d814572d4a0 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -25,20 +25,17 @@ """\ This test verifies that the Spack directory layout works properly. """ -import unittest -import tempfile -import shutil import os - -from llnl.util.filesystem import * +import shutil +import tempfile import spack -from spack.spec import Spec -from spack.repository import RepoPath +from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout +from spack.repository import RepoPath +from spack.spec import Spec from spack.test.mock_packages_test import * - # number of packages to test (to reduce test time) max_packages = 10 diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 6d6a67a1d38..35780441166 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -23,19 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest -import shutil -import tempfile - -from llnl.util.filesystem import * import spack -from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which - +from llnl.util.filesystem import * from spack.test.mock_packages_test import * from spack.test.mock_repo import MockGitRepo +from spack.version import ver class GitFetchTest(MockPackagesTest): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index d884ed78a0f..b8a0c1ec460 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -23,16 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest - -from llnl.util.filesystem import * - import spack + from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which -from spack.test.mock_packages_test import * from spack.test.mock_repo import MockHgRepo +from llnl.util.filesystem import * +from spack.test.mock_packages_test import * class HgFetchTest(MockPackagesTest): diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 8863d13c426..8297893f012 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -22,18 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os -import unittest import shutil import tempfile -from llnl.util.filesystem import * - import spack -from spack.stage import Stage -from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite +from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout -from spack.util.executable import which +from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite from spack.test.mock_packages_test import * from spack.test.mock_repo import MockArchive diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index 7b67e873dde..ee37e765c7b 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -24,8 +24,6 @@ ############################################################################## import os import unittest -import shutil -import tempfile from llnl.util.filesystem import * from llnl.util.link_tree import LinkTree @@ -38,8 +36,7 @@ class LinkTreeTest(unittest.TestCase): def setUp(self): self.stage = Stage('link-tree-test') - # FIXME : possibly this test needs to be refactored to avoid the explicit call to __enter__ and __exit__ - self.stage.__enter__() + self.stage.create() with working_dir(self.stage.path): touchp('source/1') @@ -54,7 +51,7 @@ def setUp(self): self.link_tree = LinkTree(source_path) def tearDown(self): - self.stage.__exit__(None, None, None) + self.stage.destroy() def check_file_link(self, filename): diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py index bc68df01dbe..3b11d18da49 100644 --- a/lib/spack/spack/test/lock.py +++ b/lib/spack/spack/test/lock.py @@ -25,15 +25,13 @@ """ These tests ensure that our lock works correctly. """ -import unittest -import os -import tempfile import shutil +import tempfile +import unittest from multiprocessing import Process -from llnl.util.lock import * from llnl.util.filesystem import join_path, touch - +from llnl.util.lock import * from spack.util.multiproc import Barrier # This is the longest a failed test will take, as the barriers will diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index d568a28d443..a2606acf19b 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -28,13 +28,13 @@ This just tests whether the right args are getting passed to make. """ import os -import unittest -import tempfile import shutil +import tempfile +import unittest from llnl.util.filesystem import * -from spack.util.environment import path_put_first from spack.build_environment import MakeExecutable +from spack.util.environment import path_put_first class MakeExecutableTest(unittest.TestCase): diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index f117e042424..e707adfe9d5 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -23,11 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from filecmp import dircmp - import spack import spack.mirror -from spack.util.compression import decompressor_for + +from filecmp import dircmp from spack.test.mock_packages_test import * from spack.test.mock_repo import * diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index e9f1f95df55..0b8867b61e2 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -22,17 +22,15 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys import os import shutil -import unittest import tempfile -from ordereddict_backport import OrderedDict - -from llnl.util.filesystem import mkdirp +import unittest import spack import spack.config +from llnl.util.filesystem import mkdirp +from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index ed94023b0eb..a8bdfb55710 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -26,13 +26,9 @@ import shutil from llnl.util.filesystem import * - -import spack -from spack.version import ver from spack.stage import Stage from spack.util.executable import which - # # VCS Systems used by mock repo code. # diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 7bf4ff0a0a5..2d4b8cd584a 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -25,14 +25,11 @@ """ Test for multi_method dispatch. """ -import unittest import spack from spack.multimethod import * -from spack.version import * -from spack.spec import Spec -from spack.multimethod import when from spack.test.mock_packages_test import * +from spack.version import * class MultiMethodTest(MockPackagesTest): diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index d0d809004d5..83fb34bf765 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest + from spack.util.naming import NamespaceTrie diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index ebd72819992..55f35ea4c96 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -22,10 +22,8 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest -import spack -from spack.spec import Spec, CompilerSpec +from spack.spec import Spec from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 83984dc5f65..07ff0b21af0 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -22,14 +22,12 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest - -from llnl.util.filesystem import join_path import spack +from llnl.util.filesystem import join_path from spack.repository import Repo -from spack.util.naming import mod_to_class from spack.test.mock_packages_test import * +from spack.util.naming import mod_to_class class PackagesTest(MockPackagesTest): @@ -77,30 +75,20 @@ def test_package_class_names(self): # def test_import_package(self): - import spack.pkg.builtin.mock.mpich + pass def test_import_package_as(self): - import spack.pkg.builtin.mock.mpich as mp + pass def test_import_class_from_package(self): - from spack.pkg.builtin.mock.mpich import Mpich + pass def test_import_module_from_package(self): - from spack.pkg.builtin.mock import mpich + pass def test_import_namespace_container_modules(self): - import spack.pkg - import spack.pkg as p - from spack import pkg - - import spack.pkg.builtin - import spack.pkg.builtin as b - from spack.pkg import builtin - - import spack.pkg.builtin.mock - import spack.pkg.builtin.mock as m - from spack.pkg.builtin import mock + pass diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index d74d3b9b7d7..42949753044 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -28,12 +28,11 @@ Spack was originally 2.7, but enough systems in 2014 are still using 2.6 on their frontend nodes that we need 2.6 to get adopted. """ -import unittest import os import re +import unittest import llnl.util.tty as tty - import pyqver2 import spack diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 632f777cdeb..5e6162b6e66 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -31,8 +31,6 @@ import spack import spack.package -from llnl.util.lang import list_modules - from spack.spec import Spec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 44a09cbd7fb..8c33d1ff6e7 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -22,7 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest from spack.spec import * from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 1daaa4be8fc..6e08e30e135 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -23,9 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest + import spack.spec -from spack.spec import * from spack.parse import Token +from spack.spec import * # Sample output for a complex lexing. complex_lex = [Token(ID, 'mvapich_foo'), diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index a7314eba4c5..dbcf89d8645 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -25,15 +25,13 @@ """\ Test that the Stage class works correctly. """ -import unittest -import shutil import os -import getpass +import shutil +import unittest from contextlib import * -from llnl.util.filesystem import * - import spack +from llnl.util.filesystem import * from spack.stage import Stage from spack.util.executable import which diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 13a00f5df79..1ee4ee700e7 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -24,18 +24,12 @@ ############################################################################## import os import re -import unittest -import shutil -import tempfile - -from llnl.util.filesystem import * - import spack -from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which -from spack.test.mock_packages_test import * + from spack.test.mock_repo import svn, MockSvnRepo +from spack.version import ver +from spack.test.mock_packages_test import * +from llnl.util.filesystem import * class SvnFetchTest(MockPackagesTest): diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index 9ca898c47ce..eb1e4a3240e 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -22,10 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from nose.plugins import Plugin - import os +from nose.plugins import Plugin + class Tally(Plugin): name = 'tally' diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index ccc409dd602..18615b7efe8 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -22,10 +22,11 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest import itertools +import unittest import spack + test_install = __import__("spack.cmd.test-install", fromlist=["BuildId", "create_test_output", "TestResult"]) diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py index 87adf894017..068a335b491 100644 --- a/lib/spack/spack/test/url_extrapolate.py +++ b/lib/spack/spack/test/url_extrapolate.py @@ -25,10 +25,7 @@ """\ Tests ability of spack to extrapolate URL versions from existing versions. """ -import spack import spack.url as url -from spack.spec import Spec -from spack.version import ver from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index efde7c0c73b..561d4658a10 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -27,8 +27,8 @@ detection in Homebrew. """ import unittest + import spack.url as url -from pprint import pprint class UrlParseTest(unittest.TestCase): diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index aec8baf4eaa..2be38af0d34 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -27,7 +27,6 @@ """ import unittest -import spack import spack.url as url diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 108450e098e..2732006eb3d 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -28,6 +28,7 @@ where it makes sense. """ import unittest + from spack.version import * diff --git a/lib/spack/spack/test/yaml.py b/lib/spack/spack/test/yaml.py index 5a357b8e696..b930c022f26 100644 --- a/lib/spack/spack/test/yaml.py +++ b/lib/spack/spack/test/yaml.py @@ -26,6 +26,7 @@ Test Spack's custom YAML format. """ import unittest + import spack.util.spack_yaml as syaml test_file = """\ From 353d12d2e6e24c0f62795dd9e649a801b566c4d8 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 3 Mar 2016 09:18:47 +0100 Subject: [PATCH 487/910] stage : added create function as decided in #477 comments --- lib/spack/spack/stage.py | 50 ++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index e87b822a8f4..5d8b09fa9e6 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -105,26 +105,10 @@ def __enter__(self): """ Entering a stage context will create the stage directory - If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise - this will attempt to create a stage in a temporary directory and link it into spack.stage_path. - - Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location - in tmp_dirs, fall back to making the stage inside spack.stage_path. + Returns: + self """ - # Create the top-level stage directory - mkdirp(spack.stage_path) - remove_dead_links(spack.stage_path) - # If a tmp_root exists then create a directory there and then link it in the stage area, - # otherwise create the stage directory in self.path - if self._need_to_create_path(): - if self.tmp_root: - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) - else: - mkdirp(self.path) - # Make sure we can actually do something with the stage we made. - ensure_access(self.path) - + self.create() return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -314,8 +298,34 @@ def restage(self): """ self.fetcher.reset() + def create(self): + """ + Creates the stage directory + + If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise + this will attempt to create a stage in a temporary directory and link it into spack.stage_path. + + Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location + in tmp_dirs, fall back to making the stage inside spack.stage_path. + """ + # Create the top-level stage directory + mkdirp(spack.stage_path) + remove_dead_links(spack.stage_path) + # If a tmp_root exists then create a directory there and then link it in the stage area, + # otherwise create the stage directory in self.path + if self._need_to_create_path(): + if self.tmp_root: + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + os.symlink(tmp_dir, self.path) + else: + mkdirp(self.path) + # Make sure we can actually do something with the stage we made. + ensure_access(self.path) + def destroy(self): - """Remove this stage directory.""" + """ + Removes this stage directory + """ remove_linked_tree(self.path) # Make sure we don't end up in a removed directory From 5ef36215569e1151164ecb630011de425387732a Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 3 Mar 2016 10:10:30 +0100 Subject: [PATCH 488/910] stage : minor syntax fix --- lib/spack/spack/stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 5d8b09fa9e6..985043c1035 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -246,7 +246,7 @@ def fetch(self, mirror_only=False): self.fetcher = fetcher self.fetcher.fetch() break - except spack.error.SpackError, e: + except spack.error.SpackError as e: tty.msg("Fetching from %s failed." % fetcher) tty.debug(e) continue From db92699c02f9aaf435d6bc073c89cd2c75d8824c Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 3 Mar 2016 11:05:36 +0100 Subject: [PATCH 489/910] test : fixed tests that were broken by import optimization --- lib/spack/spack/test/packages.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 07ff0b21af0..f0b5e05f3b5 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -75,20 +75,30 @@ def test_package_class_names(self): # def test_import_package(self): - pass + import spack.pkg.builtin.mock.mpich def test_import_package_as(self): - pass + import spack.pkg.builtin.mock.mpich as mp def test_import_class_from_package(self): - pass + from spack.pkg.builtin.mock.mpich import Mpich def test_import_module_from_package(self): - pass + from spack.pkg.builtin.mock import mpich def test_import_namespace_container_modules(self): - pass + import spack.pkg + import spack.pkg as p + from spack import pkg + + import spack.pkg.builtin + import spack.pkg.builtin as b + from spack.pkg import builtin + + import spack.pkg.builtin.mock + import spack.pkg.builtin.mock as m + from spack.pkg.builtin import mock From 63ae40716908ee8c19d49de83accff482d921331 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 3 Mar 2016 16:39:21 +0100 Subject: [PATCH 490/910] netlib-lapack : unified build with netlib-blas openssl : updated version py-numpy : updated dependencies --- .../builtin/packages/netlib-blas/package.py | 46 --------------- .../builtin/packages/netlib-lapack/package.py | 57 +++++++------------ .../repos/builtin/packages/openssl/package.py | 1 + .../builtin/packages/py-numpy/package.py | 3 +- 4 files changed, 21 insertions(+), 86 deletions(-) delete mode 100644 var/spack/repos/builtin/packages/netlib-blas/package.py diff --git a/var/spack/repos/builtin/packages/netlib-blas/package.py b/var/spack/repos/builtin/packages/netlib-blas/package.py deleted file mode 100644 index 85e97323d35..00000000000 --- a/var/spack/repos/builtin/packages/netlib-blas/package.py +++ /dev/null @@ -1,46 +0,0 @@ -from spack import * -import os - - -class NetlibBlas(Package): - """Netlib reference BLAS""" - homepage = "http://www.netlib.org/lapack/" - url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" - - version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') - - variant('fpic', default=False, description="Build with -fpic compiler option") - - # virtual dependency - provides('blas') - - # Doesn't always build correctly in parallel - parallel = False - - def patch(self): - os.symlink('make.inc.example', 'make.inc') - - mf = FileFilter('make.inc') - mf.filter('^FORTRAN.*', 'FORTRAN = f90') - mf.filter('^LOADER.*', 'LOADER = f90') - mf.filter('^CC =.*', 'CC = cc') - - if '+fpic' in self.spec: - mf.filter('^OPTS.*=.*', 'OPTS = -O2 -frecursive -fpic') - mf.filter('^CFLAGS =.*', 'CFLAGS = -O3 -fpic') - - - def install(self, spec, prefix): - make('blaslib') - - # Tests that blas builds correctly - make('blas_testing') - - # No install provided - mkdirp(prefix.lib) - install('librefblas.a', prefix.lib) - - # Blas virtual package should provide blas.a and libblas.a - with working_dir(prefix.lib): - symlink('librefblas.a', 'blas.a') - symlink('librefblas.a', 'libblas.a') diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py index fb6b99e27c3..1bb77919d50 100644 --- a/var/spack/repos/builtin/packages/netlib-lapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -1,16 +1,15 @@ from spack import * + class NetlibLapack(Package): """ - LAPACK version 3.X is a comprehensive FORTRAN library that does - linear algebra operations including matrix inversions, least - squared solutions to linear sets of equations, eigenvector - analysis, singular value decomposition, etc. It is a very - comprehensive and reputable package that has found extensive - use in the scientific community. + LAPACK version 3.X is a comprehensive FORTRAN library that does linear algebra operations including matrix + inversions, least squared solutions to linear sets of equations, eigenvector analysis, singular value + decomposition, etc. It is a very comprehensive and reputable package that has found extensive use in the + scientific community. """ homepage = "http://www.netlib.org/lapack/" - url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" + url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') version('3.4.2', '61bf1a8a4469d4bdb7604f5897179478') @@ -18,42 +17,24 @@ class NetlibLapack(Package): version('3.4.0', '02d5706ec03ba885fc246e5fa10d8c70') version('3.3.1', 'd0d533ec9a5b74933c2a1e84eedc58b4') - variant('shared', default=False, description="Build shared library version") + variant('debug', default=False, description='Activates the Debug build type') + variant('shared', default=True, description="Build shared library version") + + variant('lapacke', default=True, description='Activates the build of the LAPACKE C interface') # virtual dependency + provides('blas') provides('lapack') - # blas is a virtual dependency. - depends_on('blas') - depends_on('cmake') - # Doesn't always build correctly in parallel - parallel = False - - @when('^netlib-blas') - def get_blas_libs(self): - blas = self.spec['netlib-blas'] - return [join_path(blas.prefix.lib, 'blas.a')] - - - @when('^atlas') - def get_blas_libs(self): - blas = self.spec['atlas'] - return [join_path(blas.prefix.lib, l) - for l in ('libf77blas.a', 'libatlas.a')] - - def install(self, spec, prefix): - blas_libs = ";".join(self.get_blas_libs()) - cmake_args = [".", '-DBLAS_LIBRARIES=' + blas_libs] - - if '+shared' in spec: - cmake_args.append('-DBUILD_SHARED_LIBS=ON') - - cmake_args += std_cmake_args - - cmake(*cmake_args) - make() - make("install") + cmake_args = ['-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), + '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), + '-DLAPACKE:BOOL=%s' % ('ON' if '+lapacke' in spec else 'OFF')] + cmake_args.extend(std_cmake_args) + with working_dir('spack-build', create=True): + cmake('..', *cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index c73102f05d3..70afaf4038a 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -17,6 +17,7 @@ class Openssl(Package): version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') version('1.0.2f', 'b3bf73f507172be9292ea2a8c28b659d') + version('1.0.2g', 'f3c710c045cdee5fd114feb69feba7aa') depends_on("zlib") parallel = False diff --git a/var/spack/repos/builtin/packages/py-numpy/package.py b/var/spack/repos/builtin/packages/py-numpy/package.py index 03548111863..0844ff5caa6 100644 --- a/var/spack/repos/builtin/packages/py-numpy/package.py +++ b/var/spack/repos/builtin/packages/py-numpy/package.py @@ -12,8 +12,7 @@ class PyNumpy(Package): extends('python') depends_on('py-nose') - depends_on('netlib-blas+fpic', when='+blas') - depends_on('netlib-lapack+shared', when='+blas') + depends_on('lapack+shared', when='+blas') def install(self, spec, prefix): if '+blas' in spec: From a531a4f5e306388fcb6aa3c2f9db7bb93b530242 Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Thu, 3 Mar 2016 18:00:50 +0100 Subject: [PATCH 491/910] help spack versions paraview to get versions without v --- lib/spack/spack/url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 02c0b83e260..ad551a6dedb 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -225,7 +225,7 @@ def parse_version_offset(path): (r'_((\d+\.)+\d+[a-z]?)[.]orig$', stem), # e.g. http://www.openssl.org/source/openssl-0.9.8s.tar.gz - (r'-([^-]+(-alpha|-beta)?)', stem), + (r'-v?([^-]+(-alpha|-beta)?)', stem), # e.g. astyle_1.23_macosx.tar.gz (r'_([^_]+(_alpha|_beta)?)', stem), From 7183db1b7df3bce7e5f174ce38a50476c2e3b218 Mon Sep 17 00:00:00 2001 From: Scott Pakin Date: Thu, 3 Mar 2016 11:30:57 -0700 Subject: [PATCH 492/910] Added LLVM 3.7.1 support --- .../repos/builtin/packages/llvm/package.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index 934d994bd3e..280e400f696 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. @@ -34,7 +34,7 @@ class Llvm(Package): it is the full name of the project. """ homepage = 'http://llvm.org/' - url = 'http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz' + url = 'http://llvm.org/releases/3.7.1/llvm-3.7.1.src.tar.xz' version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package @@ -132,6 +132,21 @@ class Llvm(Package): 'llvm-libunwind' : 'http://llvm.org/svn/llvm-project/libunwind/trunk', } }, + { + 'version' : '3.7.1', + 'md5':'bf8b3a2c79e61212c5409041dfdbd319', + 'resources' : { + 'compiler-rt' : '1c6975daf30bb3b0473b53c3a1a6ff01', + 'openmp' : 'b4ad08cda4e5c22e42b66062b140438e', + 'polly' : '3a2a7367002740881637f4d47bca4dc3', + 'libcxx' : 'f9c43fa552a10e14ff53b94d04bea140', + 'libcxxabi' : '52d925afac9f97e9dcac90745255c169', + 'clang' : '0acd026b5529164197563d135a8fd83e', + 'clang-tools-extra' : '5d49ff745037f061a7c86aeb6a24c3d2', + 'lldb' : 'a106d8a0d21fc84d76953822fbaf3398', + 'llvm-libunwind' : '814bd52c9247c5d04629658fbcb3ab8c', + } + }, { 'version' : '3.7.0', 'md5':'b98b9495e5655a672d6cb83e1a180f8e', From c5d9ee89246b3d2aeddb756a04588424051d3295 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 3 Mar 2016 14:41:20 -0800 Subject: [PATCH 493/910] Revert "refactoring proposal : turn Stage into a context manager" --- lib/spack/llnl/util/filesystem.py | 35 +--- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/mirror.py | 55 +++--- lib/spack/spack/package.py | 176 +++++++++---------- lib/spack/spack/stage.py | 186 +++++++++++---------- lib/spack/spack/test/concretize.py | 2 + lib/spack/spack/test/config.py | 6 +- lib/spack/spack/test/configure_guess.py | 21 ++- lib/spack/spack/test/database.py | 11 +- lib/spack/spack/test/directory_layout.py | 13 +- lib/spack/spack/test/git_fetch.py | 46 +++-- lib/spack/spack/test/hg_fetch.py | 44 ++--- lib/spack/spack/test/install.py | 9 +- lib/spack/spack/test/link_tree.py | 7 +- lib/spack/spack/test/lock.py | 8 +- lib/spack/spack/test/make_executable.py | 6 +- lib/spack/spack/test/mirror.py | 69 ++++---- lib/spack/spack/test/mock_packages_test.py | 8 +- lib/spack/spack/test/mock_repo.py | 4 + lib/spack/spack/test/multimethod.py | 5 +- lib/spack/spack/test/namespace_trie.py | 1 - lib/spack/spack/test/optional_deps.py | 4 +- lib/spack/spack/test/packages.py | 6 +- lib/spack/spack/test/python_version.py | 3 +- lib/spack/spack/test/spec_dag.py | 2 + lib/spack/spack/test/spec_semantics.py | 1 + lib/spack/spack/test/spec_syntax.py | 3 +- lib/spack/spack/test/stage.py | 130 ++++++++------ lib/spack/spack/test/svn_fetch.py | 49 +++--- lib/spack/spack/test/tally_plugin.py | 4 +- lib/spack/spack/test/unit_install.py | 3 +- lib/spack/spack/test/url_extrapolate.py | 3 + lib/spack/spack/test/url_parse.py | 2 +- lib/spack/spack/test/url_substitution.py | 1 + lib/spack/spack/test/versions.py | 1 - lib/spack/spack/test/yaml.py | 1 - 36 files changed, 504 insertions(+), 423 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 366237ef8f3..da3cf96050f 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links', 'remove_linked_tree'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink'] import os import sys @@ -240,7 +240,7 @@ def touchp(path): def force_symlink(src, dest): try: os.symlink(src, dest) - except OSError as e: + except OSError, e: os.remove(dest) os.symlink(src, dest) @@ -344,34 +344,3 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): if order == 'post': yield (source_path, dest_path) - -def remove_dead_links(root): - """ - Removes any dead link that is present in root - - Args: - root: path where to search for dead links - - """ - for file in os.listdir(root): - path = join_path(root, file) - if os.path.islink(path): - real_path = os.path.realpath(path) - if not os.path.exists(real_path): - os.unlink(path) - -def remove_linked_tree(path): - """ - Removes a directory and its contents. If the directory is a symlink, follows the link and removes the real - directory before removing the link. - - Args: - path: directory to be removed - - """ - if os.path.exists(path): - if os.path.islink(path): - shutil.rmtree(os.path.realpath(path), True) - os.unlink(path) - else: - shutil.rmtree(path, True) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 0c8bd1d5281..6e7179122c1 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -43,4 +43,4 @@ def clean(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.stage.destroy() + package.do_clean() diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 5ed7aff176b..fa29e208036 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -110,6 +110,7 @@ def suggest_archive_basename(resource): return basename + def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with package archives. @@ -157,29 +158,17 @@ def create(path, specs, **kwargs): "Cannot create directory '%s':" % mirror_root, str(e)) # Things to keep track of while parsing specs. - categories = { - 'present': [], - 'mirrored': [], - 'error': [] - } + present = [] + mirrored = [] + error = [] # Iterate through packages and download all the safe tarballs for each of them + everything_already_exists = True for spec in version_specs: - add_single_spec(spec, mirror_root, categories, **kwargs) - - return categories['present'], categories['mirrored'], categories['error'] - - -def add_single_spec(spec, mirror_root, categories, **kwargs): - tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) - spec_exists_in_mirror = True - try: - with spec.package.stage: - # fetcher = stage.fetcher - # fetcher.fetch() - # ... - # fetcher.archive(archive_path) - for ii, stage in enumerate(spec.package.stage): + pkg = spec.package + tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) + try: + for ii, stage in enumerate(pkg.stage): fetcher = stage.fetcher if ii == 0: # create a subdirectory for the current package@version @@ -195,7 +184,7 @@ def add_single_spec(spec, mirror_root, categories, **kwargs): if os.path.exists(archive_path): tty.msg("{name} : already added".format(name=name)) else: - spec_exists_in_mirror = False + everything_already_exists = False fetcher.fetch() if not kwargs.get('no_checksum', False): fetcher.check() @@ -206,16 +195,20 @@ def add_single_spec(spec, mirror_root, categories, **kwargs): fetcher.archive(archive_path) tty.msg("{name} : added".format(name=name)) - if spec_exists_in_mirror: - categories['present'].append(spec) - else: - categories['mirrored'].append(spec) - except Exception as e: - if spack.debug: - sys.excepthook(*sys.exc_info()) - else: - tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) - categories['error'].append(spec) + if everything_already_exists: + present.append(spec) + else: + mirrored.append(spec) + except Exception, e: + if spack.debug: + sys.excepthook(*sys.exc_info()) + else: + tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) + error.append(spec) + finally: + pkg.stage.destroy() + + return (present, mirrored, error) class MirrorError(spack.error.SpackError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 106c546d5ce..9f1825ca211 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -293,6 +293,7 @@ class SomePackage(Package): .. code-block:: python + p.do_clean() # removes the stage directory entirely p.do_restage() # removes the build directory and # re-expands the archive. @@ -502,6 +503,7 @@ def fetcher(self): self._fetcher = self._make_fetcher() return self._fetcher + @fetcher.setter def fetcher(self, f): self._fetcher = f @@ -733,7 +735,7 @@ def do_patch(self): # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. if os.path.isfile(bad_file): - tty.msg("Patching failed last time. Restaging.") + tty.msg("Patching failed last time. Restaging.") self.stage.restage() self.stage.chdir_to_source() @@ -848,103 +850,102 @@ def do_install(self, make_jobs=make_jobs) start_time = time.time() - with self.stage: - if not fake: - if not skip_patch: - self.do_patch() + if not fake: + if not skip_patch: + self.do_patch() + else: + self.do_stage() + + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) + + def cleanup(): + if not keep_prefix: + # If anything goes wrong, remove the install prefix + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed." + + "Manually remove this directory to fix:", + self.prefix, wrap=True) + + + def real_work(): + try: + tty.msg("Building %s." % self.name) + + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) + + # Set up process's build environment before running install. + if fake: + self.do_fake_install() else: - self.do_stage() + # Do the real install in the source directory. + self.stage.chdir_to_source() - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') - def cleanup(): - if not keep_prefix: - # If anything goes wrong, remove the install prefix - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + - "Manually remove this directory to fix:", - self.prefix, wrap=True) + # This redirects I/O to a build log (and optionally to the terminal) + log_path = join_path(os.getcwd(), 'spack-build.out') + log_file = open(log_path, 'w') + with log_output(log_file, verbose, sys.stdout.isatty(), True): + dump_environment(env_path) + self.install(self.spec, self.prefix) + # Ensure that something was actually installed. + self._sanity_check_install() - def real_work(): - try: - tty.msg("Building %s." % self.name) + # Move build log into install directory on success + if not fake: + log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) + install(log_path, log_install_path) + install(env_path, env_install_path) - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) + packages_dir = spack.install_layout.build_packages_path(self.spec) + dump_packages(self.spec, packages_dir) - # Set up process's build environment before running install. - if fake: - self.do_fake_install() - else: - # Do the real install in the source directory. - self.stage.chdir_to_source() + # On successful install, remove the stage. + if not keep_stage: + self.stage.destroy() - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - # This redirects I/O to a build log (and optionally to the terminal) - log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), True): - dump_environment(env_path) - self.install(self.spec, self.prefix) + tty.msg("Successfully installed %s." % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) - # Ensure that something was actually installed. - self._sanity_check_install() + except ProcessError, e: + # Annotate with location of build log. + e.build_log = log_path + cleanup() + raise e - # Move build log into install directory on success - if not fake: - log_install_path = spack.install_layout.build_log_path(self.spec) - env_install_path = spack.install_layout.build_env_path(self.spec) - install(log_path, log_install_path) - install(env_path, env_install_path) + except: + # other exceptions just clean up and raise. + cleanup() + raise - packages_dir = spack.install_layout.build_packages_path(self.spec) - dump_packages(self.spec, packages_dir) + # Set parallelism before starting build. + self.make_jobs = make_jobs - # On successful install, remove the stage. - if not keep_stage: - self.stage.destroy() + # Do the build. + spack.build_environment.fork(self, real_work) - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, self.prefix) - tty.msg("Successfully installed %s." % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) - - except ProcessError as e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e - - except: - # other exceptions just clean up and raise. - cleanup() - raise - - # Set parallelism before starting build. - self.make_jobs = make_jobs - - # Do the build. - spack.build_environment.fork(self, real_work) - - # note: PARENT of the build process adds the new package to - # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, self.prefix) - - # Once everything else is done, run post install hooks - spack.hooks.post_install(self) + # Once everything else is done, run post install hooks + spack.hooks.post_install(self) def _sanity_check_install(self): @@ -1148,6 +1149,13 @@ def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() + + def do_clean(self): + """Removes the package's build stage and source tarball.""" + if os.path.exists(self.stage.path): + self.stage.destroy() + + def format_doc(self, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get('indent', 0) @@ -1184,7 +1192,7 @@ def fetch_remote_versions(self): try: return spack.util.web.find_versions_of_archive( *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) - except spack.error.NoNetworkConnectionError as e: + except spack.error.NoNetworkConnectionError, e: tty.die("Package.fetch_versions couldn't connect to:", e.url, e.message) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 985043c1035..f217450d42d 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -42,26 +42,33 @@ class Stage(object): - """ - A Stage object is a context manager that handles a directory where some source code is downloaded and built - before being installed. It handles fetching the source code, either as an archive to be expanded or by checking - it out of a repository. A stage's lifecycle looks like this: + """A Stage object manages a directory where some source code is + downloaded and built before being installed. It handles + fetching the source code, either as an archive to be expanded + or by checking it out of a repository. A stage's lifecycle + looks like this: - ``` - with Stage() as stage: # Context manager creates and destroys the stage directory - fetch() # Fetch a source archive into the stage. - expand_archive() # Expand the source archive. - # Build and install the archive. This is handled by the Package class. - ``` + Stage() + Constructor creates the stage directory. + fetch() + Fetch a source archive into the stage. + expand_archive() + Expand the source archive. + + Build and install the archive. This is handled by the Package class. + destroy() + Remove the stage once the package has been installed. - If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory. - Otherwise, stages are created directly in spack.stage_path. + If spack.use_tmp_stage is True, spack will attempt to create stages + in a tmp directory. Otherwise, stages are created directly in + spack.stage_path. - There are two kinds of stages: named and unnamed. Named stages can persist between runs of spack, e.g. if you - fetched a tarball but didn't finish building it, you won't have to fetch it again. + There are two kinds of stages: named and unnamed. Named stages can + persist between runs of spack, e.g. if you fetched a tarball but + didn't finish building it, you won't have to fetch it again. - Unnamed stages are created using standard mkdtemp mechanisms or similar, and are intended to persist for - only one run of spack. + Unnamed stages are created using standard mkdtemp mechanisms or + similar, and are intended to persist for only one run of spack. """ def __init__(self, url_or_fetch_strategy, **kwargs): @@ -89,46 +96,21 @@ def __init__(self, url_or_fetch_strategy, **kwargs): self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. - # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name - # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root - self.name = kwargs.get('name') if 'name' in kwargs else STAGE_PREFIX + next(tempfile._get_candidate_names()) + self.name = kwargs.get('name') self.mirror_path = kwargs.get('mirror_path') self.tmp_root = find_tmp_root() - # Try to construct here a temporary name for the stage directory - # If this is a named stage, then construct a named path. - self.path = join_path(spack.stage_path, self.name) - # Flag to decide whether to delete the stage folder on exit or not - self.delete_on_exit = True + self.path = None + self._setup() - def __enter__(self): - """ - Entering a stage context will create the stage directory - - Returns: - self - """ - self.create() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """ - Exiting from a stage context will delete the stage directory unless: - - it was explicitly requested not to do so - - an exception has been raised - - Args: - exc_type: exception type - exc_val: exception value - exc_tb: exception traceback - - Returns: - Boolean - """ - self.delete_on_exit = False if exc_type is not None else self.delete_on_exit - - if self.delete_on_exit: - self.destroy() + def _cleanup_dead_links(self): + """Remove any dead links in the stage directory.""" + for file in os.listdir(spack.stage_path): + path = join_path(spack.stage_path, file) + if os.path.islink(path): + real_path = os.path.realpath(path) + if not os.path.exists(path): + os.unlink(path) def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we @@ -166,6 +148,54 @@ def _need_to_create_path(self): return False + def _setup(self): + """Creates the stage directory. + If spack.use_tmp_stage is False, the stage directory is created + directly under spack.stage_path. + + If spack.use_tmp_stage is True, this will attempt to create a + stage in a temporary directory and link it into spack.stage_path. + Spack will use the first writable location in spack.tmp_dirs to + create a stage. If there is no valid location in tmp_dirs, fall + back to making the stage inside spack.stage_path. + """ + # Create the top-level stage directory + mkdirp(spack.stage_path) + self._cleanup_dead_links() + + # If this is a named stage, then construct a named path. + if self.name is not None: + self.path = join_path(spack.stage_path, self.name) + + # If this is a temporary stage, them make the temp directory + tmp_dir = None + if self.tmp_root: + if self.name is None: + # Unnamed tmp root. Link the path in + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + self.name = os.path.basename(tmp_dir) + self.path = join_path(spack.stage_path, self.name) + if self._need_to_create_path(): + os.symlink(tmp_dir, self.path) + + else: + if self._need_to_create_path(): + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + os.symlink(tmp_dir, self.path) + + # if we're not using a tmp dir, create the stage directly in the + # stage dir, rather than linking to it. + else: + if self.name is None: + self.path = tempfile.mkdtemp('', STAGE_PREFIX, spack.stage_path) + self.name = os.path.basename(self.path) + else: + if self._need_to_create_path(): + mkdirp(self.path) + + # Make sure we can actually do something with the stage we made. + ensure_access(self.path) + @property def archive_file(self): """Path to the source archive within this stage directory.""" @@ -246,7 +276,7 @@ def fetch(self, mirror_only=False): self.fetcher = fetcher self.fetcher.fetch() break - except spack.error.SpackError as e: + except spack.error.SpackError, e: tty.msg("Fetching from %s failed." % fetcher) tty.debug(e) continue @@ -298,34 +328,8 @@ def restage(self): """ self.fetcher.reset() - def create(self): - """ - Creates the stage directory - - If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise - this will attempt to create a stage in a temporary directory and link it into spack.stage_path. - - Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location - in tmp_dirs, fall back to making the stage inside spack.stage_path. - """ - # Create the top-level stage directory - mkdirp(spack.stage_path) - remove_dead_links(spack.stage_path) - # If a tmp_root exists then create a directory there and then link it in the stage area, - # otherwise create the stage directory in self.path - if self._need_to_create_path(): - if self.tmp_root: - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) - else: - mkdirp(self.path) - # Make sure we can actually do something with the stage we made. - ensure_access(self.path) - def destroy(self): - """ - Removes this stage directory - """ + """Remove this stage directory.""" remove_linked_tree(self.path) # Make sure we don't end up in a removed directory @@ -385,15 +389,6 @@ def source_path(self): def path(self): return self[0].path - def __enter__(self): - for item in self: - item.__enter__() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - for item in reversed(self): - item.__exit__(exc_type, exc_val, exc_tb) - def chdir_to_source(self): return self[0].chdir_to_source() @@ -444,6 +439,19 @@ def ensure_access(file=spack.stage_path): tty.die("Insufficient permissions for %s" % file) +def remove_linked_tree(path): + """Removes a directory and its contents. If the directory is a symlink, + follows the link and reamoves the real directory before removing the + link. + """ + if os.path.exists(path): + if os.path.islink(path): + shutil.rmtree(os.path.realpath(path), True) + os.unlink(path) + else: + shutil.rmtree(path, True) + + def purge(): """Remove all build directories in the top-level stage path.""" if os.path.isdir(spack.stage_path): diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 794344fb6ae..7f2938aec5a 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -22,6 +22,8 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import unittest + import spack from spack.spec import Spec, CompilerSpec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 0562d2d6209..d8be5a855b8 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -22,13 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os +import unittest import shutil +import os from tempfile import mkdtemp - +from ordereddict_backport import OrderedDict import spack import spack.config -from ordereddict_backport import OrderedDict from spack.test.mock_packages_test import * # Some sample compiler config data diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index 2440d120e5d..a4e8565b62b 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -23,15 +23,20 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import unittest import shutil import tempfile -import unittest from llnl.util.filesystem import * + from spack.cmd.create import ConfigureGuesser from spack.stage import Stage -from spack.test.mock_packages_test import * + +from spack.fetch_strategy import URLFetchStrategy +from spack.directory_layout import YamlDirectoryLayout from spack.util.executable import which +from spack.test.mock_packages_test import * +from spack.test.mock_repo import MockArchive class InstallTest(unittest.TestCase): @@ -47,6 +52,8 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) + if self.stage: + self.stage.destroy() os.chdir(self.orig_dir) @@ -57,12 +64,12 @@ def check_archive(self, filename, system): url = 'file://' + join_path(os.getcwd(), 'archive.tar.gz') print url - with Stage(url) as stage: - stage.fetch() + self.stage = Stage(url) + self.stage.fetch() - guesser = ConfigureGuesser() - guesser(stage) - self.assertEqual(system, guesser.build_system) + guesser = ConfigureGuesser() + guesser(self.stage) + self.assertEqual(system, guesser.build_system) def test_python(self): diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 9a57e1f03ea..0205f4b8ce2 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -26,18 +26,19 @@ These tests check the database is functioning properly, both in memory and in its file """ -import multiprocessing -import shutil import tempfile +import shutil +import multiprocessing + +from llnl.util.lock import * +from llnl.util.filesystem import join_path import spack -from llnl.util.filesystem import join_path -from llnl.util.lock import * -from llnl.util.tty.colify import colify from spack.database import Database from spack.directory_layout import YamlDirectoryLayout from spack.test.mock_packages_test import * +from llnl.util.tty.colify import colify def _print_ref_counts(): """Print out all ref counts for the graph used here, for debugging""" diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index d814572d4a0..925cb648edd 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -25,17 +25,20 @@ """\ This test verifies that the Spack directory layout works properly. """ -import os -import shutil +import unittest import tempfile +import shutil +import os + +from llnl.util.filesystem import * import spack -from llnl.util.filesystem import * -from spack.directory_layout import YamlDirectoryLayout -from spack.repository import RepoPath from spack.spec import Spec +from spack.repository import RepoPath +from spack.directory_layout import YamlDirectoryLayout from spack.test.mock_packages_test import * + # number of packages to test (to reduce test time) max_packages = 10 diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 35780441166..d84433176a8 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -23,12 +23,19 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import unittest +import shutil +import tempfile + +from llnl.util.filesystem import * import spack -from llnl.util.filesystem import * +from spack.version import ver +from spack.stage import Stage +from spack.util.executable import which + from spack.test.mock_packages_test import * from spack.test.mock_repo import MockGitRepo -from spack.version import ver class GitFetchTest(MockPackagesTest): @@ -45,15 +52,19 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) + def tearDown(self): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() self.repo.destroy() + self.pkg.do_clean() + def assert_rev(self, rev): """Check that the current git revision is equal to the supplied rev.""" self.assertEqual(self.repo.rev_hash('HEAD'), self.repo.rev_hash(rev)) + def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with @@ -65,27 +76,26 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('git')] = args - with self.pkg.stage: - self.pkg.do_stage() - self.assert_rev(rev) + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked_file = 'foobarbaz' - touch(untracked_file) - self.assertTrue(os.path.isfile(untracked_file)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked_file)) + untracked_file = 'foobarbaz' + touch(untracked_file) + self.assertTrue(os.path.isfile(untracked_file)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked_file)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_master(self): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index b8a0c1ec460..bbcb64e4c1a 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -23,12 +23,16 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import spack +import unittest -from spack.version import ver -from spack.test.mock_repo import MockHgRepo from llnl.util.filesystem import * + +import spack +from spack.version import ver +from spack.stage import Stage +from spack.util.executable import which from spack.test.mock_packages_test import * +from spack.test.mock_repo import MockHgRepo class HgFetchTest(MockPackagesTest): @@ -45,10 +49,13 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) + def tearDown(self): """Destroy the stage space used by this test.""" super(HgFetchTest, self).tearDown() self.repo.destroy() + self.pkg.do_clean() + def try_fetch(self, rev, test_file, args): """Tries to: @@ -61,27 +68,26 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('hg')] = args - with self.pkg.stage: - self.pkg.do_stage() - self.assertEqual(self.repo.get_rev(), rev) + self.pkg.do_stage() + self.assertEqual(self.repo.get_rev(), rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assertEqual(self.repo.get_rev(), rev) + self.assertEqual(self.repo.get_rev(), rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 8297893f012..8863d13c426 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -22,13 +22,18 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import os +import unittest import shutil import tempfile -import spack from llnl.util.filesystem import * -from spack.directory_layout import YamlDirectoryLayout + +import spack +from spack.stage import Stage from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite +from spack.directory_layout import YamlDirectoryLayout +from spack.util.executable import which from spack.test.mock_packages_test import * from spack.test.mock_repo import MockArchive diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index ee37e765c7b..886b7ef4c5d 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -24,6 +24,8 @@ ############################################################################## import os import unittest +import shutil +import tempfile from llnl.util.filesystem import * from llnl.util.link_tree import LinkTree @@ -36,7 +38,6 @@ class LinkTreeTest(unittest.TestCase): def setUp(self): self.stage = Stage('link-tree-test') - self.stage.create() with working_dir(self.stage.path): touchp('source/1') @@ -50,8 +51,10 @@ def setUp(self): source_path = os.path.join(self.stage.path, 'source') self.link_tree = LinkTree(source_path) + def tearDown(self): - self.stage.destroy() + if self.stage: + self.stage.destroy() def check_file_link(self, filename): diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py index 3b11d18da49..bc68df01dbe 100644 --- a/lib/spack/spack/test/lock.py +++ b/lib/spack/spack/test/lock.py @@ -25,13 +25,15 @@ """ These tests ensure that our lock works correctly. """ -import shutil -import tempfile import unittest +import os +import tempfile +import shutil from multiprocessing import Process -from llnl.util.filesystem import join_path, touch from llnl.util.lock import * +from llnl.util.filesystem import join_path, touch + from spack.util.multiproc import Barrier # This is the longest a failed test will take, as the barriers will diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index a2606acf19b..d568a28d443 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -28,13 +28,13 @@ This just tests whether the right args are getting passed to make. """ import os -import shutil -import tempfile import unittest +import tempfile +import shutil from llnl.util.filesystem import * -from spack.build_environment import MakeExecutable from spack.util.environment import path_put_first +from spack.build_environment import MakeExecutable class MakeExecutableTest(unittest.TestCase): diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index e707adfe9d5..f83cc8090c9 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -23,10 +23,11 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +from filecmp import dircmp + import spack import spack.mirror - -from filecmp import dircmp +from spack.util.compression import decompressor_for from spack.test.mock_packages_test import * from spack.test.mock_repo import * @@ -73,14 +74,14 @@ def set_up_package(self, name, MockRepoClass, url_attr): def check_mirror(self): - with Stage('spack-mirror-test') as stage: - mirror_root = join_path(stage.path, 'test-mirror') - - # register mirror with spack config - mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } - spack.config.update_config('mirrors', mirrors) + stage = Stage('spack-mirror-test') + mirror_root = join_path(stage.path, 'test-mirror') + # register mirror with spack config + mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } + spack.config.update_config('mirrors', mirrors) + try: os.chdir(stage.path) spack.mirror.create( mirror_root, self.repos, no_checksum=True) @@ -96,28 +97,38 @@ def check_mirror(self): files = os.listdir(subdir) self.assertEqual(len(files), 1) - # Now try to fetch each package. - for name, mock_repo in self.repos.items(): - spec = Spec(name).concretized() - pkg = spec.package + # Now try to fetch each package. + for name, mock_repo in self.repos.items(): + spec = Spec(name).concretized() + pkg = spec.package - saved_checksum_setting = spack.do_checksum - with pkg.stage: - # Stage the archive from the mirror and cd to it. - spack.do_checksum = False - pkg.do_stage(mirror_only=True) - # Compare the original repo with the expanded archive - original_path = mock_repo.path - if 'svn' in name: - # have to check out the svn repo to compare. - original_path = join_path(mock_repo.path, 'checked_out') - svn('checkout', mock_repo.url, original_path) - dcmp = dircmp(original_path, pkg.stage.source_path) - # make sure there are no new files in the expanded tarball - self.assertFalse(dcmp.right_only) - # and that all original files are present. - self.assertTrue(all(l in exclude for l in dcmp.left_only)) - spack.do_checksum = saved_checksum_setting + pkg._stage = None + saved_checksum_setting = spack.do_checksum + try: + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) + + # Compare the original repo with the expanded archive + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) + + dcmp = dircmp(original_path, pkg.stage.source_path) + + # make sure there are no new files in the expanded tarball + self.assertFalse(dcmp.right_only) + + # and that all original files are present. + self.assertTrue(all(l in exclude for l in dcmp.left_only)) + + finally: + spack.do_checksum = saved_checksum_setting + pkg.do_clean() + finally: + stage.destroy() def test_git_mirror(self): diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 0b8867b61e2..e9f1f95df55 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -22,15 +22,17 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import sys import os import shutil -import tempfile import unittest +import tempfile +from ordereddict_backport import OrderedDict + +from llnl.util.filesystem import mkdirp import spack import spack.config -from llnl.util.filesystem import mkdirp -from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index a8bdfb55710..ed94023b0eb 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -26,9 +26,13 @@ import shutil from llnl.util.filesystem import * + +import spack +from spack.version import ver from spack.stage import Stage from spack.util.executable import which + # # VCS Systems used by mock repo code. # diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 2d4b8cd584a..7bf4ff0a0a5 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -25,11 +25,14 @@ """ Test for multi_method dispatch. """ +import unittest import spack from spack.multimethod import * -from spack.test.mock_packages_test import * from spack.version import * +from spack.spec import Spec +from spack.multimethod import when +from spack.test.mock_packages_test import * class MultiMethodTest(MockPackagesTest): diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index 2023ba6d96f..647976df210 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -23,7 +23,6 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest - from spack.util.naming import NamespaceTrie diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index 55f35ea4c96..ebd72819992 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -22,8 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import unittest -from spack.spec import Spec +import spack +from spack.spec import Spec, CompilerSpec from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index f0b5e05f3b5..83984dc5f65 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -22,12 +22,14 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import unittest + +from llnl.util.filesystem import join_path import spack -from llnl.util.filesystem import join_path from spack.repository import Repo -from spack.test.mock_packages_test import * from spack.util.naming import mod_to_class +from spack.test.mock_packages_test import * class PackagesTest(MockPackagesTest): diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index 42949753044..d74d3b9b7d7 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -28,11 +28,12 @@ Spack was originally 2.7, but enough systems in 2014 are still using 2.6 on their frontend nodes that we need 2.6 to get adopted. """ +import unittest import os import re -import unittest import llnl.util.tty as tty + import pyqver2 import spack diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 5e6162b6e66..632f777cdeb 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -31,6 +31,8 @@ import spack import spack.package +from llnl.util.lang import list_modules + from spack.spec import Spec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 8c33d1ff6e7..44a09cbd7fb 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import unittest from spack.spec import * from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 6e08e30e135..1daaa4be8fc 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -23,10 +23,9 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest - import spack.spec -from spack.parse import Token from spack.spec import * +from spack.parse import Token # Sample output for a complex lexing. complex_lex = [Token(ID, 'mvapich_foo'), diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index dbcf89d8645..c1b2a2a5730 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -25,13 +25,15 @@ """\ Test that the Stage class works correctly. """ -import os -import shutil import unittest +import shutil +import os +import getpass from contextlib import * -import spack from llnl.util.filesystem import * + +import spack from spack.stage import Stage from spack.util.executable import which @@ -190,90 +192,116 @@ def check_destroy(self, stage, stage_name): def test_setup_and_destroy_name_with_tmp(self): with use_tmp(True): - with Stage(archive_url, name=stage_name) as stage: - self.check_setup(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + self.check_setup(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_setup_and_destroy_name_without_tmp(self): with use_tmp(False): - with Stage(archive_url, name=stage_name) as stage: - self.check_setup(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + self.check_setup(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_setup_and_destroy_no_name_with_tmp(self): with use_tmp(True): - with Stage(archive_url) as stage: - self.check_setup(stage, None) + stage = Stage(archive_url) + self.check_setup(stage, None) + + stage.destroy() self.check_destroy(stage, None) def test_setup_and_destroy_no_name_without_tmp(self): with use_tmp(False): - with Stage(archive_url) as stage: - self.check_setup(stage, None) + stage = Stage(archive_url) + self.check_setup(stage, None) + + stage.destroy() self.check_destroy(stage, None) def test_chdir(self): - with Stage(archive_url, name=stage_name) as stage: - stage.chdir() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + + stage.chdir() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_fetch(self): - with Stage(archive_url, name=stage_name) as stage: - stage.fetch() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + + stage.fetch() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_expand_archive(self): - with Stage(archive_url, name=stage_name) as stage: - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - stage.expand_archive() - self.check_expand_archive(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + + stage.expand_archive() + self.check_expand_archive(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_expand_archive(self): - with Stage(archive_url, name=stage_name) as stage: - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_restage(self): - with Stage(archive_url, name=stage_name) as stage: - stage.fetch() - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) + stage = Stage(archive_url, name=stage_name) - # Try to make a file in the old archive dir - with open('foobar', 'w') as file: - file.write("this file is to be destroyed.") + stage.fetch() + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) - self.assertTrue('foobar' in os.listdir(stage.source_path)) + # Try to make a file in the old archive dir + with open('foobar', 'w') as file: + file.write("this file is to be destroyed.") - # Make sure the file is not there after restage. - stage.restage() - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - stage.chdir_to_source() - self.check_chdir_to_source(stage, stage_name) - self.assertFalse('foobar' in os.listdir(stage.source_path)) + self.assertTrue('foobar' in os.listdir(stage.source_path)) + + # Make sure the file is not there after restage. + stage.restage() + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) + + stage.chdir_to_source() + self.check_chdir_to_source(stage, stage_name) + self.assertFalse('foobar' in os.listdir(stage.source_path)) + + stage.destroy() self.check_destroy(stage, stage_name) diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 1ee4ee700e7..454a7f1d1ff 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -24,13 +24,19 @@ ############################################################################## import os import re -import spack +import unittest +import shutil +import tempfile -from spack.test.mock_repo import svn, MockSvnRepo -from spack.version import ver -from spack.test.mock_packages_test import * from llnl.util.filesystem import * +import spack +from spack.version import ver +from spack.stage import Stage +from spack.util.executable import which +from spack.test.mock_packages_test import * +from spack.test.mock_repo import svn, MockSvnRepo + class SvnFetchTest(MockPackagesTest): """Tests fetching from a dummy git repository.""" @@ -45,10 +51,13 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) + def tearDown(self): """Destroy the stage space used by this test.""" super(SvnFetchTest, self).tearDown() self.repo.destroy() + self.pkg.do_clean() + def assert_rev(self, rev): """Check that the current revision is equal to the supplied rev.""" @@ -61,6 +70,7 @@ def get_rev(): return match.group(1) self.assertEqual(get_rev(), rev) + def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with @@ -72,27 +82,26 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('svn')] = args - with self.pkg.stage: - self.pkg.do_stage() - self.assert_rev(rev) + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index 4163ab95dd6..e0b9618e0cb 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -22,10 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os - from nose.plugins import Plugin +import os + class Tally(Plugin): name = 'tally' diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index 18615b7efe8..ccc409dd602 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -22,11 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import itertools import unittest +import itertools import spack - test_install = __import__("spack.cmd.test-install", fromlist=["BuildId", "create_test_output", "TestResult"]) diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py index 068a335b491..87adf894017 100644 --- a/lib/spack/spack/test/url_extrapolate.py +++ b/lib/spack/spack/test/url_extrapolate.py @@ -25,7 +25,10 @@ """\ Tests ability of spack to extrapolate URL versions from existing versions. """ +import spack import spack.url as url +from spack.spec import Spec +from spack.version import ver from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index 561d4658a10..efde7c0c73b 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -27,8 +27,8 @@ detection in Homebrew. """ import unittest - import spack.url as url +from pprint import pprint class UrlParseTest(unittest.TestCase): diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index 2be38af0d34..aec8baf4eaa 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -27,6 +27,7 @@ """ import unittest +import spack import spack.url as url diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 2732006eb3d..108450e098e 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -28,7 +28,6 @@ where it makes sense. """ import unittest - from spack.version import * diff --git a/lib/spack/spack/test/yaml.py b/lib/spack/spack/test/yaml.py index b930c022f26..5a357b8e696 100644 --- a/lib/spack/spack/test/yaml.py +++ b/lib/spack/spack/test/yaml.py @@ -26,7 +26,6 @@ Test Spack's custom YAML format. """ import unittest - import spack.util.spack_yaml as syaml test_file = """\ From 311d71ec86bc7e245520a816cee4053c31a8a086 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 3 Mar 2016 16:50:51 -0600 Subject: [PATCH 494/910] Modifications to get PGI working --- lib/spack/env/cc | 6 +++--- lib/spack/env/pgi/case-insensitive/pgCC | 1 - lib/spack/env/pgi/{pgf77 => pgc++} | 0 lib/spack/env/pgi/{pgf90 => pgfortran} | 0 lib/spack/spack/compilers/pgi.py | 18 +++++++++--------- 5 files changed, 12 insertions(+), 13 deletions(-) delete mode 120000 lib/spack/env/pgi/case-insensitive/pgCC rename lib/spack/env/pgi/{pgf77 => pgc++} (100%) rename lib/spack/env/pgi/{pgf90 => pgfortran} (100%) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index a323c481243..b8b6c86e010 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -90,15 +90,15 @@ case "$command" in command="$SPACK_CC" language="C" ;; - c++|CC|g++|clang++|icpc|pgCC|xlc++) + c++|CC|g++|clang++|icpc|pgc++|xlc++) command="$SPACK_CXX" language="C++" ;; - f90|fc|f95|gfortran|ifort|pgf90|xlf90|nagfor) + f90|fc|f95|gfortran|ifort|pgfortran|xlf90|nagfor) command="$SPACK_FC" language="Fortran 90" ;; - f77|gfortran|ifort|pgf77|xlf|nagfor) + f77|gfortran|ifort|pgfortran|xlf|nagfor) command="$SPACK_F77" language="Fortran 77" ;; diff --git a/lib/spack/env/pgi/case-insensitive/pgCC b/lib/spack/env/pgi/case-insensitive/pgCC deleted file mode 120000 index e2deb67f3b6..00000000000 --- a/lib/spack/env/pgi/case-insensitive/pgCC +++ /dev/null @@ -1 +0,0 @@ -../../cc \ No newline at end of file diff --git a/lib/spack/env/pgi/pgf77 b/lib/spack/env/pgi/pgc++ similarity index 100% rename from lib/spack/env/pgi/pgf77 rename to lib/spack/env/pgi/pgc++ diff --git a/lib/spack/env/pgi/pgf90 b/lib/spack/env/pgi/pgfortran similarity index 100% rename from lib/spack/env/pgi/pgf90 rename to lib/spack/env/pgi/pgfortran diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 9ac74cfbdb5..c6a1078bd9f 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -29,28 +29,28 @@ class Pgi(Compiler): cc_names = ['pgcc'] # Subclasses use possible names of C++ compiler - cxx_names = ['pgCC'] + cxx_names = ['pgc++', 'pgCC'] # Subclasses use possible names of Fortran 77 compiler - f77_names = ['pgf77'] + f77_names = ['pgfortran', 'pgf77'] # Subclasses use possible names of Fortran 90 compiler - fc_names = ['pgf95', 'pgf90'] + fc_names = ['pgfortran', 'pgf95', 'pgf90'] # Named wrapper links within spack.build_env_path link_paths = { 'cc' : 'pgi/pgcc', - 'cxx' : 'pgi/case-insensitive/pgCC', - 'f77' : 'pgi/pgf77', - 'fc' : 'pgi/pgf90' } + 'cxx' : 'pgi/pgc++', + 'f77' : 'pgi/pgfortran', + 'fc' : 'pgi/pgfortran' } @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. Output looks like this:: - pgf95 10.2-0 64-bit target on x86-64 Linux -tp nehalem-64 - Copyright 1989-2000, The Portland Group, Inc. All Rights Reserved. - Copyright 2000-2010, STMicroelectronics, Inc. All Rights Reserved. + pgcc 15.10-0 64-bit target on x86-64 Linux -tp sandybridge + The Portland Group - PGI Compilers and Tools + Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. """ return get_compiler_version( comp, '-V', r'pg[^ ]* ([^ ]+) \d\d\d?-bit target') From 202fbc681265bd26739e0af3c37bb09a80b7e564 Mon Sep 17 00:00:00 2001 From: citibeth Date: Thu, 3 Mar 2016 23:40:29 -0500 Subject: [PATCH 495/910] Rmoved periods. Should help use of cut'n'paste from console windows. --- lib/spack/spack/cmd/checksum.py | 6 +++--- lib/spack/spack/cmd/compiler.py | 6 +++--- lib/spack/spack/cmd/create.py | 10 +++++----- lib/spack/spack/cmd/mirror.py | 4 ++-- lib/spack/spack/cmd/module.py | 6 +++--- lib/spack/spack/cmd/repo.py | 12 ++++++------ lib/spack/spack/config.py | 6 +++--- lib/spack/spack/database.py | 2 +- lib/spack/spack/directives.py | 4 ++-- lib/spack/spack/directory_layout.py | 4 ++-- lib/spack/spack/fetch_strategy.py | 16 ++++++++-------- lib/spack/spack/mirror.py | 4 ++-- lib/spack/spack/package.py | 26 +++++++++++++------------- lib/spack/spack/repository.py | 6 +++--- lib/spack/spack/stage.py | 4 ++-- 15 files changed, 58 insertions(+), 58 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index c4519932335..966ff9a5e96 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -100,11 +100,11 @@ def checksum(parser, args): else: versions = pkg.fetch_remote_versions() if not versions: - tty.die("Could not fetch any versions for %s." % pkg.name) + tty.die("Could not fetch any versions for %s" % pkg.name) sorted_versions = sorted(versions, reverse=True) - tty.msg("Found %s versions of %s." % (len(versions), pkg.name), + tty.msg("Found %s versions of %s" % (len(versions), pkg.name), *spack.cmd.elide_list( ["%-10s%s" % (v, versions[v]) for v in sorted_versions])) print @@ -121,7 +121,7 @@ def checksum(parser, args): keep_stage=args.keep_stage) if not version_hashes: - tty.die("Could not fetch any versions for %s." % pkg.name) + tty.die("Could not fetch any versions for %s" % pkg.name) version_lines = [" version('%s', '%s')" % (v, h) for v, h in version_hashes] tty.msg("Checksummed new versions of %s:" % pkg.name, *version_lines) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 75b51f6b495..3e58e821847 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -96,7 +96,7 @@ def compiler_remove(args): compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) if not compilers: - tty.die("No compilers match spec %s." % cspec) + tty.die("No compilers match spec %s" % cspec) elif not args.all and len(compilers) > 1: tty.error("Multiple compilers match spec %s. Choose one:" % cspec) colify(reversed(sorted([c.spec for c in compilers])), indent=4) @@ -105,7 +105,7 @@ def compiler_remove(args): for compiler in compilers: spack.compilers.remove_compiler_from_config(compiler.spec, scope=args.scope) - tty.msg("Removed compiler %s." % compiler.spec) + tty.msg("Removed compiler %s" % compiler.spec) def compiler_info(args): @@ -114,7 +114,7 @@ def compiler_info(args): compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) if not compilers: - tty.error("No compilers match spec %s." % cspec) + tty.error("No compilers match spec %s" % cspec) else: for c in compilers: print str(c.spec) + ":" diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 6809209046e..4564143f834 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -156,7 +156,7 @@ def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL version = spack.url.parse_version(url) if not version: - tty.die("Couldn't guess a version string from %s." % url) + tty.die("Couldn't guess a version string from %s" % url) # Try to guess a name. If it doesn't work, allow the user to override. if args.alternate_name: @@ -189,7 +189,7 @@ def find_repository(spec, args): try: repo = Repo(repo_path) if spec.namespace and spec.namespace != repo.namespace: - tty.die("Can't create package with namespace %s in repo with namespace %s." + tty.die("Can't create package with namespace %s in repo with namespace %s" % (spec.namespace, repo.namespace)) except RepoError as e: tty.die(str(e)) @@ -252,7 +252,7 @@ def create(parser, args): name = spec.name # factors out namespace, if any repo = find_repository(spec, args) - tty.msg("This looks like a URL for %s version %s." % (name, version)) + tty.msg("This looks like a URL for %s version %s" % (name, version)) tty.msg("Creating template for package %s" % name) # Fetch tarballs (prompting user if necessary) @@ -266,7 +266,7 @@ def create(parser, args): keep_stage=args.keep_stage) if not ver_hash_tuples: - tty.die("Could not fetch any tarballs for %s." % name) + tty.die("Could not fetch any tarballs for %s" % name) # Prepend 'py-' to python package names, by convention. if guesser.build_system == 'python': @@ -291,4 +291,4 @@ def create(parser, args): # If everything checks out, go ahead and edit. spack.editor(pkg_path) - tty.msg("Created package %s." % pkg_path) + tty.msg("Created package %s" % pkg_path) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 8e9438c1a37..fcd15a6a904 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -126,7 +126,7 @@ def mirror_remove(args): old_value = mirrors.pop(name) spack.config.update_config('mirrors', mirrors, scope=args.scope) - tty.msg("Removed mirror %s with url %s." % (name, old_value)) + tty.msg("Removed mirror %s with url %s" % (name, old_value)) def mirror_list(args): @@ -203,7 +203,7 @@ def mirror_create(args): verb = "updated" if existed else "created" tty.msg( - "Successfully %s mirror in %s." % (verb, directory), + "Successfully %s mirror in %s" % (verb, directory), "Archive stats:", " %-4d already present" % p, " %-4d added" % m, diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index a5a9570eb52..1d6867c1d96 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -58,7 +58,7 @@ def module_find(mtype, spec_array): should type to use that package's module. """ if mtype not in module_types: - tty.die("Invalid module type: '%s'. Options are %s." % (mtype, comma_or(module_types))) + tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) specs = spack.cmd.parse_specs(spec_array) if len(specs) > 1: @@ -78,7 +78,7 @@ def module_find(mtype, spec_array): mt = module_types[mtype] mod = mt(specs[0]) if not os.path.isfile(mod.file_name): - tty.die("No %s module is installed for %s." % (mtype, spec)) + tty.die("No %s module is installed for %s" % (mtype, spec)) print mod.use_name @@ -94,7 +94,7 @@ def module_refresh(): shutil.rmtree(cls.path, ignore_errors=False) mkdirp(cls.path) for spec in specs: - tty.debug(" Writing file for %s." % spec) + tty.debug(" Writing file for %s" % spec) cls(spec).write() diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index c2e352786d1..87c782000f9 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -89,11 +89,11 @@ def repo_add(args): # check if the path exists if not os.path.exists(canon_path): - tty.die("No such file or directory: '%s'." % path) + tty.die("No such file or directory: %s" % path) # Make sure the path is a directory. if not os.path.isdir(canon_path): - tty.die("Not a Spack repository: '%s'." % path) + tty.die("Not a Spack repository: %s" % path) # Make sure it's actually a spack repository by constructing it. repo = Repo(canon_path) @@ -103,7 +103,7 @@ def repo_add(args): if not repos: repos = [] if repo.root in repos or path in repos: - tty.die("Repository is already registered with Spack: '%s'" % path) + tty.die("Repository is already registered with Spack: %s" % path) repos.insert(0, canon_path) spack.config.update_config('repos', repos, args.scope) @@ -122,7 +122,7 @@ def repo_remove(args): if canon_path == repo_canon_path: repos.remove(repo_path) spack.config.update_config('repos', repos, args.scope) - tty.msg("Removed repository '%s'." % repo_path) + tty.msg("Removed repository %s" % repo_path) return # If it is a namespace, remove corresponding repo @@ -132,13 +132,13 @@ def repo_remove(args): if repo.namespace == path_or_namespace: repos.remove(path) spack.config.update_config('repos', repos, args.scope) - tty.msg("Removed repository '%s' with namespace %s." + tty.msg("Removed repository %s with namespace '%s'." % (repo.root, repo.namespace)) return except RepoError as e: continue - tty.die("No repository with path or namespace: '%s'" + tty.die("No repository with path or namespace: %s" % path_or_namespace) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 6fecde99801..576a5afa2ee 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -205,7 +205,7 @@ def validate_section_name(section): """Raise a ValueError if the section is not a valid section.""" if section not in section_schemas: - raise ValueError("Invalid config section: '%s'. Options are %s." + raise ValueError("Invalid config section: '%s'. Options are %s" % (section, section_schemas)) @@ -335,7 +335,7 @@ def validate_scope(scope): return config_scopes[scope] else: - raise ValueError("Invalid config scope: '%s'. Must be one of %s." + raise ValueError("Invalid config scope: '%s'. Must be one of %s" % (scope, config_scopes.keys())) @@ -350,7 +350,7 @@ def _read_config_file(filename, schema): "Invlaid configuration. %s exists but is not a file." % filename) elif not os.access(filename, os.R_OK): - raise ConfigFileError("Config file is not readable: %s." % filename) + raise ConfigFileError("Config file is not readable: %s" % filename) try: tty.debug("Reading config file %s" % filename) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 9cbe7de44a3..089d29325e6 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -330,7 +330,7 @@ def _check_ref_counts(self): found = rec.ref_count if not expected == found: raise AssertionError( - "Invalid ref_count: %s: %d (expected %d), in DB %s." + "Invalid ref_count: %s: %d (expected %d), in DB %s" % (key, found, expected, self._index_path)) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index c8542f55f02..61cd3030124 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -125,7 +125,7 @@ def __init__(self, dicts=None): dicts = (dicts,) elif type(dicts) not in (list, tuple): raise TypeError( - "dicts arg must be list, tuple, or string. Found %s." + "dicts arg must be list, tuple, or string. Found %s" % type(dicts)) self.dicts = dicts @@ -317,5 +317,5 @@ class CircularReferenceError(DirectiveError): def __init__(self, directive, package): super(CircularReferenceError, self).__init__( directive, - "Package '%s' cannot pass itself to %s." % (package, directive)) + "Package '%s' cannot pass itself to %s" % (package, directive)) self.package = package diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 29d87b65b33..08c23627f46 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -335,7 +335,7 @@ def _extension_map(self, spec): if not dag_hash in by_hash: raise InvalidExtensionSpecError( - "Spec %s not found in %s." % (dag_hash, prefix)) + "Spec %s not found in %s" % (dag_hash, prefix)) ext_spec = by_hash[dag_hash] if not prefix == ext_spec.prefix: @@ -450,7 +450,7 @@ class ExtensionConflictError(DirectoryLayoutError): """Raised when an extension is added to a package that already has it.""" def __init__(self, spec, ext_spec, conflict): super(ExtensionConflictError, self).__init__( - "%s cannot be installed in %s because it conflicts with %s."% ( + "%s cannot be installed in %s because it conflicts with %s"% ( ext_spec.short_spec, spec.short_spec, conflict.short_spec)) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 83a2dbb59c5..ec17cb97f14 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -153,7 +153,7 @@ def fetch(self): self.stage.chdir() if self.archive_file: - tty.msg("Already downloaded %s." % self.archive_file) + tty.msg("Already downloaded %s" % self.archive_file) return tty.msg("Trying to fetch from %s" % self.url) @@ -275,8 +275,8 @@ def check(self): checker = crypto.Checker(self.digest) if not checker.check(self.archive_file): raise ChecksumError( - "%s checksum failed for %s." % (checker.hash_name, self.archive_file), - "Expected %s but got %s." % (self.digest, checker.sum)) + "%s checksum failed for %s" % (checker.hash_name, self.archive_file), + "Expected %s but got %s" % (self.digest, checker.sum)) @_needs_stage def reset(self): @@ -312,7 +312,7 @@ def __init__(self, name, *rev_types, **kwargs): # Ensure that there's only one of the rev_types if sum(k in kwargs for k in rev_types) > 1: raise FetchStrategyError( - "Supply only one of %s to fetch with %s." % ( + "Supply only one of %s to fetch with %s" % ( comma_or(rev_types), name)) # Set attributes for each rev type. @@ -321,7 +321,7 @@ def __init__(self, name, *rev_types, **kwargs): @_needs_stage def check(self): - tty.msg("No checksum needed when fetching with %s." % self.name) + tty.msg("No checksum needed when fetching with %s" % self.name) @_needs_stage def expand(self): @@ -395,7 +395,7 @@ def fetch(self): self.stage.chdir() if self.stage.source_path: - tty.msg("Already fetched %s." % self.stage.source_path) + tty.msg("Already fetched %s" % self.stage.source_path) return args = [] @@ -505,7 +505,7 @@ def fetch(self): self.stage.chdir() if self.stage.source_path: - tty.msg("Already fetched %s." % self.stage.source_path) + tty.msg("Already fetched %s" % self.stage.source_path) return tty.msg("Trying to check out svn repository: %s" % self.url) @@ -584,7 +584,7 @@ def fetch(self): self.stage.chdir() if self.stage.source_path: - tty.msg("Already fetched %s." % self.stage.source_path) + tty.msg("Already fetched %s" % self.stage.source_path) return args = [] diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index fa29e208036..58e31c2c7b5 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -73,7 +73,7 @@ def get_matching_versions(specs, **kwargs): # Skip any package that has no known versions. if not pkg.versions: - tty.msg("No safe (checksummed) versions for package %s." % pkg.name) + tty.msg("No safe (checksummed) versions for package %s" % pkg.name) continue num_versions = kwargs.get('num_versions', 0) @@ -203,7 +203,7 @@ def create(path, specs, **kwargs): if spack.debug: sys.excepthook(*sys.exc_info()) else: - tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) + tty.warn("Error while fetching %s" % spec.format('$_$@'), e.message) error.append(spec) finally: pkg.stage.destroy() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 9f1825ca211..fb96f61de9e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -688,7 +688,7 @@ def do_fetch(self, mirror_only=False): if not ignore_checksum: raise FetchError( - "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg) + "Will not fetch %s" % self.spec.format('$_$@'), checksum_msg) self.stage.fetch(mirror_only) @@ -722,7 +722,7 @@ def do_patch(self): # If there are no patches, note it. if not self.patches and not has_patch_fun: - tty.msg("No patches needed for %s." % self.name) + tty.msg("No patches needed for %s" % self.name) return # Construct paths to special files in the archive dir used to @@ -745,7 +745,7 @@ def do_patch(self): tty.msg("Already patched %s" % self.name) return elif os.path.isfile(no_patches_file): - tty.msg("No patches needed for %s." % self.name) + tty.msg("No patches needed for %s" % self.name) return # Apply all the patches for specs that match this one @@ -766,10 +766,10 @@ def do_patch(self): if has_patch_fun: try: self.patch() - tty.msg("Ran patch() for %s." % self.name) + tty.msg("Ran patch() for %s" % self.name) patched = True except: - tty.msg("patch() function failed for %s." % self.name) + tty.msg("patch() function failed for %s" % self.name) touch(bad_file) raise @@ -838,7 +838,7 @@ def do_install(self, raise ValueError("Can only install concrete packages.") if os.path.exists(self.prefix): - tty.msg("%s is already installed in %s." % (self.name, self.prefix)) + tty.msg("%s is already installed in %s" % (self.name, self.prefix)) return tty.msg("Installing %s" % self.name) @@ -874,7 +874,7 @@ def cleanup(): def real_work(): try: - tty.msg("Building %s." % self.name) + tty.msg("Building %s" % self.name) # Run the pre-install hook in the child process after # the directory is created. @@ -918,8 +918,8 @@ def real_work(): self._total_time = time.time() - start_time build_time = self._total_time - self._fetch_time - tty.msg("Successfully installed %s." % self.name, - "Fetch: %s. Build: %s. Total: %s." + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s" % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) print_pkg(self.prefix) @@ -1025,7 +1025,7 @@ def do_uninstall(self, force=False): # Uninstalling in Spack only requires removing the prefix. self.remove_prefix() spack.installed_db.remove(self.spec) - tty.msg("Successfully uninstalled %s." % self.spec.short_spec) + tty.msg("Successfully uninstalled %s" % self.spec.short_spec) # Once everything else is done, run post install hooks spack.hooks.post_uninstall(self) @@ -1072,7 +1072,7 @@ def do_activate(self, force=False): self.extendee_spec.package.activate(self, **self.extendee_args) spack.install_layout.add_extension(self.extendee_spec, self.spec) - tty.msg("Activated extension %s for %s." + tty.msg("Activated extension %s for %s" % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) @@ -1124,7 +1124,7 @@ def do_deactivate(self, **kwargs): if self.activated: spack.install_layout.remove_extension(self.extendee_spec, self.spec) - tty.msg("Deactivated extension %s for %s." + tty.msg("Deactivated extension %s for %s" % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) @@ -1320,7 +1320,7 @@ class PackageVersionError(PackageError): """Raised when a version URL cannot automatically be determined.""" def __init__(self, version): super(PackageVersionError, self).__init__( - "Cannot determine a URL automatically for version %s." % version, + "Cannot determine a URL automatically for version %s" % version, "Please provide a url for this version in the package.py file.") diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 8d06fefe7f0..3c3ba08bcc2 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -156,7 +156,7 @@ def _add(self, repo): if repo.namespace in self.by_namespace: raise DuplicateRepoError( - "Package repos '%s' and '%s' both provide namespace %s." + "Package repos '%s' and '%s' both provide namespace %s" % (repo.root, self.by_namespace[repo.namespace].root, repo.namespace)) # Add repo to the pkg indexes @@ -545,7 +545,7 @@ def get(self, spec, new=False): raise UnknownPackageError(spec.name) if spec.namespace and spec.namespace != self.namespace: - raise UnknownPackageError("Repository %s does not contain package %s." + raise UnknownPackageError("Repository %s does not contain package %s" % (self.namespace, spec.fullname)) key = hash(spec) @@ -825,7 +825,7 @@ class UnknownPackageError(PackageLoadError): def __init__(self, name, repo=None): msg = None if repo: - msg = "Package %s not found in repository %s." % (name, repo) + msg = "Package %s not found in repository %s" % (name, repo) else: msg = "Package %s not found." % name super(UnknownPackageError, self).__init__(msg) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index f217450d42d..5591cb9ba5a 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -306,9 +306,9 @@ def expand_archive(self): archive_dir = self.source_path if not archive_dir: self.fetcher.expand() - tty.msg("Created stage in %s." % self.path) + tty.msg("Created stage in %s" % self.path) else: - tty.msg("Already staged %s in %s." % (self.name, self.path)) + tty.msg("Already staged %s in %s" % (self.name, self.path)) def chdir_to_source(self): """Changes directory to the expanded archive directory. From 310099d88838c8e08d15fb34028263c9294b62e9 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 4 Mar 2016 09:31:03 +0100 Subject: [PATCH 496/910] atlas : provides blas and lapack --- .../repos/builtin/packages/atlas/package.py | 62 +++++++++---------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/var/spack/repos/builtin/packages/atlas/package.py b/var/spack/repos/builtin/packages/atlas/package.py index fc683363a70..f9b50417b00 100644 --- a/var/spack/repos/builtin/packages/atlas/package.py +++ b/var/spack/repos/builtin/packages/atlas/package.py @@ -1,31 +1,34 @@ from spack import * from spack.util.executable import Executable -import os +import os.path class Atlas(Package): """ - Automatically Tuned Linear Algebra Software, generic shared - ATLAS is an approach for the automatic generation and optimization of - numerical software. Currently ATLAS supplies optimized versions for the - complete set of linear algebra kernels known as the Basic Linear Algebra - Subroutines (BLAS), and a subset of the linear algebra routines in the - LAPACK library. + Automatically Tuned Linear Algebra Software, generic shared ATLAS is an approach for the automatic generation and + optimization of numerical software. Currently ATLAS supplies optimized versions for the complete set of linear + algebra kernels known as the Basic Linear Algebra Subroutines (BLAS), and a subset of the linear algebra routines + in the LAPACK library. """ homepage = "http://math-atlas.sourceforge.net/" + version('3.10.2', 'a4e21f343dec8f22e7415e339f09f6da', + url='http://downloads.sourceforge.net/project/math-atlas/Stable/3.10.2/atlas3.10.2.tar.bz2', preferred=True) + resource(name='lapack', + url='http://www.netlib.org/lapack/lapack-3.5.0.tgz', + md5='b1d3e3e425b2e44a06760ff173104bdf', + destination='spack-resource-lapack', + when='@3:') + version('3.11.34', '0b6c5389c095c4c8785fd0f724ec6825', url='http://sourceforge.net/projects/math-atlas/files/Developer%20%28unstable%29/3.11.34/atlas3.11.34.tar.bz2/download') - version('3.10.2', 'a4e21f343dec8f22e7415e339f09f6da', - url='http://downloads.sourceforge.net/project/math-atlas/Stable/3.10.2/atlas3.10.2.tar.bz2') - # TODO: make this provide BLAS once it works better. Create a way - # TODO: to mark "beta" packages and require explicit invocation. - - # provides('blas') + variant('shared', default=True, description='Builds shared library') + provides('blas') + provides('lapack') def patch(self): - # Disable thraed check. LLNL's environment does not allow + # Disable thread check. LLNL's environment does not allow # disabling of CPU throttling in a way that ATLAS actually # understands. filter_file(r'^\s+if \(thrchk\) exit\(1\);', 'if (0) exit(1);', @@ -33,26 +36,21 @@ def patch(self): # TODO: investigate a better way to add the check back in # TODO: using, say, MSRs. Or move this to a variant. - @when('@:3.10') def install(self, spec, prefix): - with working_dir('ATLAS-Build', create=True): + + options = [] + if '+shared' in spec: + options.append('--shared') + + # Lapack resource + lapack_stage = self.stage[1] + lapack_tarfile = os.path.basename(lapack_stage.fetcher.url) + lapack_tarfile_path = join_path(lapack_stage.path, lapack_tarfile) + options.append('--with-netlib-lapack-tarfile=%s' % lapack_tarfile_path) + + with working_dir('spack-build', create=True): configure = Executable('../configure') - configure('--prefix=%s' % prefix, '-C', 'ic', 'cc', '-C', 'if', 'f77', "--dylibs") - make() - make('check') - make('ptcheck') - make('time') - make("install") - - - def install(self, spec, prefix): - with working_dir('ATLAS-Build', create=True): - configure = Executable('../configure') - configure('--incdir=%s' % prefix.include, - '--libdir=%s' % prefix.lib, - '--cc=cc', - "--shared") - + configure('--prefix=%s' % prefix, *options) make() make('check') make('ptcheck') From a2af4940aaf169f3358f19986fc94a02222a8096 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 4 Mar 2016 09:59:16 +0100 Subject: [PATCH 497/910] atlas : disabling parallel builds --- var/spack/repos/builtin/packages/atlas/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/atlas/package.py b/var/spack/repos/builtin/packages/atlas/package.py index f9b50417b00..b5504122b74 100644 --- a/var/spack/repos/builtin/packages/atlas/package.py +++ b/var/spack/repos/builtin/packages/atlas/package.py @@ -27,6 +27,8 @@ class Atlas(Package): provides('blas') provides('lapack') + parallel = False + def patch(self): # Disable thread check. LLNL's environment does not allow # disabling of CPU throttling in a way that ATLAS actually From 0eb6ef2cd031d1668e1891425bdc6234df0594df Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 4 Mar 2016 11:09:40 -0500 Subject: [PATCH 498/910] 1. Removed fake MPI dependency from netcdf-cxx4 and netcdf-fortran. 2. Removed Fortran bootstrap variant from netcdf. Users who need NetCDF Fortran interface should install netcdf-fortran. 3. Added result of ./configure --help on the netcdf-fortran. Verified the package has no additional options that should be exposed through Spack. --- .../builtin/packages/netcdf-cxx4/package.py | 6 - .../packages/netcdf-fortran/package.py | 156 +++++++++++++++++- .../repos/builtin/packages/netcdf/package.py | 12 +- 3 files changed, 154 insertions(+), 20 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py index 8d51a106794..fb4c2886cd8 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -7,12 +7,6 @@ class NetcdfCxx4(Package): version('4.2', 'd019853802092cf686254aaba165fc81') - - variant('mpi', default=True, description='Enables MPI parallelism') - - # netcdf-cxx4 doesn't really depend (directly) on MPI. However... this - # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) - depends_on('mpi', when='+mpi') depends_on('netcdf') def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py index 9e4aee95fb8..4e0b14d0121 100644 --- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -10,12 +10,162 @@ class NetcdfFortran(Package): variant('mpi', default=True, description='Enables MPI parallelism') - # netcdf-fortran doesn't really depend (directly) on MPI. However... this - # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) - depends_on('mpi', when='+mpi') depends_on('netcdf') def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() make("install") + + + + +# netcdf-fortran configure parameters are below +# --------------------------------------------- +# +# `configure' configures netCDF-Fortran 4.4.3 to adapt to many kinds of systems. +# +# Usage: ./configure [OPTION]... [VAR=VALUE]... +# +# To assign environment variables (e.g., CC, CFLAGS...), specify them as +# VAR=VALUE. See below for descriptions of some of the useful variables. +# +# Defaults for the options are specified in brackets. +# +# Configuration: +# -h, --help display this help and exit +# --help=short display options specific to this package +# --help=recursive display the short help of all the included packages +# -V, --version display version information and exit +# -q, --quiet, --silent do not print `checking ...' messages +# --cache-file=FILE cache test results in FILE [disabled] +# -C, --config-cache alias for `--cache-file=config.cache' +# -n, --no-create do not create output files +# --srcdir=DIR find the sources in DIR [configure dir or `..'] +# +# Installation directories: +# --prefix=PREFIX install architecture-independent files in PREFIX +# [/usr/local] +# --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX +# [PREFIX] +# +# By default, `make install' will install all the files in +# `/usr/local/bin', `/usr/local/lib' etc. You can specify +# an installation prefix other than `/usr/local' using `--prefix', +# for instance `--prefix=$HOME'. +# +# For better control, use the options below. +# +# Fine tuning of the installation directories: +# --bindir=DIR user executables [EPREFIX/bin] +# --sbindir=DIR system admin executables [EPREFIX/sbin] +# --libexecdir=DIR program executables [EPREFIX/libexec] +# --sysconfdir=DIR read-only single-machine data [PREFIX/etc] +# --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] +# --localstatedir=DIR modifiable single-machine data [PREFIX/var] +# --libdir=DIR object code libraries [EPREFIX/lib] +# --includedir=DIR C header files [PREFIX/include] +# --oldincludedir=DIR C header files for non-gcc [/usr/include] +# --datarootdir=DIR read-only arch.-independent data root [PREFIX/share] +# --datadir=DIR read-only architecture-independent data [DATAROOTDIR] +# --infodir=DIR info documentation [DATAROOTDIR/info] +# --localedir=DIR locale-dependent data [DATAROOTDIR/locale] +# --mandir=DIR man documentation [DATAROOTDIR/man] +# --docdir=DIR documentation root [DATAROOTDIR/doc/netcdf-fortran] +# --htmldir=DIR html documentation [DOCDIR] +# --dvidir=DIR dvi documentation [DOCDIR] +# --pdfdir=DIR pdf documentation [DOCDIR] +# --psdir=DIR ps documentation [DOCDIR] +# +# Program names: +# --program-prefix=PREFIX prepend PREFIX to installed program names +# --program-suffix=SUFFIX append SUFFIX to installed program names +# --program-transform-name=PROGRAM run sed PROGRAM on installed program names +# +# System types: +# --build=BUILD configure for building on BUILD [guessed] +# --host=HOST cross-compile to build programs to run on HOST [BUILD] +# --target=TARGET configure for building compilers for TARGET [HOST] +# +# Optional Features: +# --disable-option-checking ignore unrecognized --enable/--with options +# --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) +# --enable-FEATURE[=ARG] include FEATURE [ARG=yes] +# --enable-silent-rules less verbose build output (undo: "make V=1") +# --disable-silent-rules verbose build output (undo: "make V=0") +# --enable-maintainer-mode +# enable make rules and dependencies not useful (and +# sometimes confusing) to the casual installer +# --enable-valgrind-tests build with valgrind-tests (valgrind is required, +# static builds only) +# --enable-parallel-tests Run extra parallel IO tests. Ignored if netCDF-4 is +# not enabled, or built on a system without parallel +# I/O support. +# --enable-extra-tests run some extra tests that may not pass because of +# known issues +# --enable-doxygen Enable generation of documentation with doxygen. +# --enable-dot Use dot (provided by graphviz) to generate charts +# and graphs in the doxygen-based documentation. +# --enable-internal-docs Include documentation of library internals. This is +# of interest only to those developing the netCDF +# library. +# --enable-dependency-tracking +# do not reject slow dependency extractors +# --disable-dependency-tracking +# speeds up one-time build +# --disable-f03-compiler-check +# disable check of ISO_C_BINDING support in Fortran +# compiler +# --disable-f03 suppress netCDF Fortran 2003 native code +# --disable-fortran-type-check +# cause the Fortran type sizes checks to be skipped +# --enable-large-file-tests +# Run tests which create very large data files (~13 GB +# disk space required, but it will be recovered when +# tests are complete). See option --with-temp-large to +# specify temporary directory +# --enable-benchmarks Run benchmarks. This is an experimental feature. +# --enable-shared[=PKGS] build shared libraries [default=yes] +# --enable-static[=PKGS] build static libraries [default=yes] +# --enable-fast-install[=PKGS] +# optimize for fast installation [default=yes] +# --disable-libtool-lock avoid locking (might break parallel builds) +# --disable-largefile omit support for large files +# --enable-extra-example-tests +# Run extra example tests; requires GNU sed. Ignored +# if netCDF-4 is not enabled. +# --enable-dll build a win32 DLL (only works on mingw) +# +# Optional Packages: +# --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] +# --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) +# --with-temp-large= +# specify directory where large files (i.e. >2 GB) +# will be written, if large files tests are run with +# --enable-large-file-tests +# --with-pic[=PKGS] try to use only PIC/non-PIC objects [default=use +# both] +# --with-gnu-ld assume the C compiler uses GNU ld [default=no] +# --with-sysroot=DIR Search for dependent libraries within DIR +# (or the compiler's sysroot if not specified). +# +# Some influential environment variables: +# CC C compiler command +# CFLAGS C compiler flags +# LDFLAGS linker flags, e.g. -L if you have libraries in a +# nonstandard directory +# LIBS libraries to pass to the linker, e.g. -l +# CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if +# you have headers in a nonstandard directory +# FC Fortran compiler command +# FCFLAGS Fortran compiler flags +# F77 Fortran 77 compiler command +# FFLAGS Fortran 77 compiler flags +# CPP C preprocessor +# +# Use these variables to override the choices made by `configure' or to help +# it to find libraries and programs with nonstandard names/locations. +# +# Report bugs to . +# +# from spack import * diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 41a0d2b6f92..0b112a59ce9 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -13,7 +13,6 @@ class Netcdf(Package): version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') variant('mpi', default=True, description='Enables MPI parallelism') - variant('fortran', default=False, description="Download and install NetCDF-Fortran") variant('hdf4', default=False, description="Enable HDF4 support") # Dependencies: @@ -66,11 +65,7 @@ def install(self, spec, prefix): # Fortran support # In version 4.2+, NetCDF-C and NetCDF-Fortran have split. - # They can be installed separately, but this bootstrap procedure - # should be able to install both at the same time. - # Note: this is a new experimental feature. - if '+fortran' in spec: - config_args.append("--enable-remote-fortran-bootstrap") + # Use the netcdf-fortran package to install Fortran support. config_args.append('CPPFLAGS=%s' % ' '.join(CPPFLAGS)) config_args.append('LDFLAGS=%s' % ' '.join(LDFLAGS)) @@ -79,8 +74,3 @@ def install(self, spec, prefix): configure(*config_args) make() make("install") - - # After installing NetCDF-C, install NetCDF-Fortran - if '+fortran' in spec: - make("build-netcdf-fortran") - make("install-netcdf-fortran") From 5db967390450659f794d202bfb5c4b4f11fb964b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 4 Mar 2016 10:51:41 -0600 Subject: [PATCH 499/910] Typo fix in compiler docstring --- lib/spack/spack/compiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 12c02e0ea2b..d38c0b00b12 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -256,12 +256,12 @@ def find(cls, *path): def __repr__(self): - """Return a string represntation of the compiler toolchain.""" + """Return a string representation of the compiler toolchain.""" return self.__str__() def __str__(self): - """Return a string represntation of the compiler toolchain.""" + """Return a string representation of the compiler toolchain.""" return "%s(%s)" % ( self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) From b043c4a5b83f36ad4837fc7302f0919a3d759940 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Fri, 4 Mar 2016 12:04:41 -0500 Subject: [PATCH 500/910] Update package.py Removed comments on configure options. --- .../packages/netcdf-fortran/package.py | 153 ------------------ 1 file changed, 153 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py index 4e0b14d0121..954e7dc3e81 100644 --- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -16,156 +16,3 @@ def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() make("install") - - - - -# netcdf-fortran configure parameters are below -# --------------------------------------------- -# -# `configure' configures netCDF-Fortran 4.4.3 to adapt to many kinds of systems. -# -# Usage: ./configure [OPTION]... [VAR=VALUE]... -# -# To assign environment variables (e.g., CC, CFLAGS...), specify them as -# VAR=VALUE. See below for descriptions of some of the useful variables. -# -# Defaults for the options are specified in brackets. -# -# Configuration: -# -h, --help display this help and exit -# --help=short display options specific to this package -# --help=recursive display the short help of all the included packages -# -V, --version display version information and exit -# -q, --quiet, --silent do not print `checking ...' messages -# --cache-file=FILE cache test results in FILE [disabled] -# -C, --config-cache alias for `--cache-file=config.cache' -# -n, --no-create do not create output files -# --srcdir=DIR find the sources in DIR [configure dir or `..'] -# -# Installation directories: -# --prefix=PREFIX install architecture-independent files in PREFIX -# [/usr/local] -# --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX -# [PREFIX] -# -# By default, `make install' will install all the files in -# `/usr/local/bin', `/usr/local/lib' etc. You can specify -# an installation prefix other than `/usr/local' using `--prefix', -# for instance `--prefix=$HOME'. -# -# For better control, use the options below. -# -# Fine tuning of the installation directories: -# --bindir=DIR user executables [EPREFIX/bin] -# --sbindir=DIR system admin executables [EPREFIX/sbin] -# --libexecdir=DIR program executables [EPREFIX/libexec] -# --sysconfdir=DIR read-only single-machine data [PREFIX/etc] -# --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] -# --localstatedir=DIR modifiable single-machine data [PREFIX/var] -# --libdir=DIR object code libraries [EPREFIX/lib] -# --includedir=DIR C header files [PREFIX/include] -# --oldincludedir=DIR C header files for non-gcc [/usr/include] -# --datarootdir=DIR read-only arch.-independent data root [PREFIX/share] -# --datadir=DIR read-only architecture-independent data [DATAROOTDIR] -# --infodir=DIR info documentation [DATAROOTDIR/info] -# --localedir=DIR locale-dependent data [DATAROOTDIR/locale] -# --mandir=DIR man documentation [DATAROOTDIR/man] -# --docdir=DIR documentation root [DATAROOTDIR/doc/netcdf-fortran] -# --htmldir=DIR html documentation [DOCDIR] -# --dvidir=DIR dvi documentation [DOCDIR] -# --pdfdir=DIR pdf documentation [DOCDIR] -# --psdir=DIR ps documentation [DOCDIR] -# -# Program names: -# --program-prefix=PREFIX prepend PREFIX to installed program names -# --program-suffix=SUFFIX append SUFFIX to installed program names -# --program-transform-name=PROGRAM run sed PROGRAM on installed program names -# -# System types: -# --build=BUILD configure for building on BUILD [guessed] -# --host=HOST cross-compile to build programs to run on HOST [BUILD] -# --target=TARGET configure for building compilers for TARGET [HOST] -# -# Optional Features: -# --disable-option-checking ignore unrecognized --enable/--with options -# --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) -# --enable-FEATURE[=ARG] include FEATURE [ARG=yes] -# --enable-silent-rules less verbose build output (undo: "make V=1") -# --disable-silent-rules verbose build output (undo: "make V=0") -# --enable-maintainer-mode -# enable make rules and dependencies not useful (and -# sometimes confusing) to the casual installer -# --enable-valgrind-tests build with valgrind-tests (valgrind is required, -# static builds only) -# --enable-parallel-tests Run extra parallel IO tests. Ignored if netCDF-4 is -# not enabled, or built on a system without parallel -# I/O support. -# --enable-extra-tests run some extra tests that may not pass because of -# known issues -# --enable-doxygen Enable generation of documentation with doxygen. -# --enable-dot Use dot (provided by graphviz) to generate charts -# and graphs in the doxygen-based documentation. -# --enable-internal-docs Include documentation of library internals. This is -# of interest only to those developing the netCDF -# library. -# --enable-dependency-tracking -# do not reject slow dependency extractors -# --disable-dependency-tracking -# speeds up one-time build -# --disable-f03-compiler-check -# disable check of ISO_C_BINDING support in Fortran -# compiler -# --disable-f03 suppress netCDF Fortran 2003 native code -# --disable-fortran-type-check -# cause the Fortran type sizes checks to be skipped -# --enable-large-file-tests -# Run tests which create very large data files (~13 GB -# disk space required, but it will be recovered when -# tests are complete). See option --with-temp-large to -# specify temporary directory -# --enable-benchmarks Run benchmarks. This is an experimental feature. -# --enable-shared[=PKGS] build shared libraries [default=yes] -# --enable-static[=PKGS] build static libraries [default=yes] -# --enable-fast-install[=PKGS] -# optimize for fast installation [default=yes] -# --disable-libtool-lock avoid locking (might break parallel builds) -# --disable-largefile omit support for large files -# --enable-extra-example-tests -# Run extra example tests; requires GNU sed. Ignored -# if netCDF-4 is not enabled. -# --enable-dll build a win32 DLL (only works on mingw) -# -# Optional Packages: -# --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] -# --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) -# --with-temp-large= -# specify directory where large files (i.e. >2 GB) -# will be written, if large files tests are run with -# --enable-large-file-tests -# --with-pic[=PKGS] try to use only PIC/non-PIC objects [default=use -# both] -# --with-gnu-ld assume the C compiler uses GNU ld [default=no] -# --with-sysroot=DIR Search for dependent libraries within DIR -# (or the compiler's sysroot if not specified). -# -# Some influential environment variables: -# CC C compiler command -# CFLAGS C compiler flags -# LDFLAGS linker flags, e.g. -L if you have libraries in a -# nonstandard directory -# LIBS libraries to pass to the linker, e.g. -l -# CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if -# you have headers in a nonstandard directory -# FC Fortran compiler command -# FCFLAGS Fortran compiler flags -# F77 Fortran 77 compiler command -# FFLAGS Fortran 77 compiler flags -# CPP C preprocessor -# -# Use these variables to override the choices made by `configure' or to help -# it to find libraries and programs with nonstandard names/locations. -# -# Report bugs to . -# -# from spack import * From 648d08eb305e04d95ee702b819923f31fd829955 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Fri, 4 Mar 2016 12:05:43 -0500 Subject: [PATCH 501/910] Update package.py Updated homepage URL to general NetCDF homepage. --- var/spack/repos/builtin/packages/netcdf-cxx4/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py index fb4c2886cd8..ab717ac6ffb 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -2,7 +2,7 @@ class NetcdfCxx4(Package): """C++ interface for NetCDF4""" - homepage = "http://www.unidata.ucar.edu/downloads/netcdf/netcdf-cxx/index.jsp" + homepage = "http://www.unidata.ucar.edu/software/netcdf" url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx4-4.2.tar.gz" version('4.2', 'd019853802092cf686254aaba165fc81') From a7b918837eab02ebff5af783d06e56ec7f868d9e Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 4 Mar 2016 15:22:28 -0600 Subject: [PATCH 502/910] GNU m4 depends on libsigsegv --- .../repos/builtin/packages/libsigsegv/package.py | 15 +++++++++++++++ var/spack/repos/builtin/packages/m4/package.py | 2 ++ 2 files changed, 17 insertions(+) create mode 100644 var/spack/repos/builtin/packages/libsigsegv/package.py diff --git a/var/spack/repos/builtin/packages/libsigsegv/package.py b/var/spack/repos/builtin/packages/libsigsegv/package.py new file mode 100644 index 00000000000..4b486198ec6 --- /dev/null +++ b/var/spack/repos/builtin/packages/libsigsegv/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Libsigsegv(Package): + """GNU libsigsegv is a library for handling page faults in user mode.""" + homepage = "https://www.gnu.org/software/libsigsegv/" + url = "ftp://ftp.gnu.org/gnu/libsigsegv/libsigsegv-2.10.tar.gz" + + version('2.10', '7f96fb1f65b3b8cbc1582fb7be774f0f') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, + '--enable-shared') + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index 5d76d8866bc..3890663ad12 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -7,6 +7,8 @@ class M4(Package): version('1.4.17', 'a5e9954b1dae036762f7b13673a2cf76') + depends_on('libsigsegv') + def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() From f90eaa5f46bfd64b9b0adb4ab5a58d169efe3c69 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Fri, 4 Mar 2016 14:53:08 -0800 Subject: [PATCH 503/910] Fixed unmatched function signature for do_fetch in jdk package --- var/spack/repos/builtin/packages/jdk/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/jdk/package.py b/var/spack/repos/builtin/packages/jdk/package.py index f8f5fc21bdd..cbcc53ac0a7 100644 --- a/var/spack/repos/builtin/packages/jdk/package.py +++ b/var/spack/repos/builtin/packages/jdk/package.py @@ -28,7 +28,7 @@ class Jdk(Package): '-H', # specify required License Agreement cookie 'Cookie: oraclelicense=accept-securebackup-cookie'] - def do_fetch(self): + def do_fetch(self, mirror_only=False): # Add our custom curl commandline options tty.msg( "[Jdk] Adding required commandline options to curl " + @@ -39,7 +39,7 @@ def do_fetch(self): spack.curl.add_default_arg(option) # Now perform the actual fetch - super(Jdk, self).do_fetch() + super(Jdk, self).do_fetch(mirror_only) def install(self, spec, prefix): From f663d37da75e4ef0f266c63707e77d29a651e07d Mon Sep 17 00:00:00 2001 From: Mark Miller Date: Fri, 4 Mar 2016 17:08:11 -0800 Subject: [PATCH 504/910] Adding zfp package. --- .../repos/builtin/packages/zfp/package.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 var/spack/repos/builtin/packages/zfp/package.py diff --git a/var/spack/repos/builtin/packages/zfp/package.py b/var/spack/repos/builtin/packages/zfp/package.py new file mode 100644 index 00000000000..620fe9d4565 --- /dev/null +++ b/var/spack/repos/builtin/packages/zfp/package.py @@ -0,0 +1,26 @@ +from spack import * + +class Zfp(Package): + """zfp is an open source C library for compressed floating-point arrays that supports + very high throughput read and write random acces, target error bounds or bit rates. + Although bit-for-bit lossless compression is not always possible, zfp is usually + accurate to within machine epsilon in near-lossless mode, and is often orders of + magnitude more accurate than other lossy compressors. + """ + + homepage = "http://computation.llnl.gov/projects/floating-point-compression" + url = "http://computation.llnl.gov/projects/floating-point-compression/download/zfp-0.5.0.tar.gz" + + version('0.5.0', '2ab29a852e65ad85aae38925c5003654') + + def install(self, spec, prefix): + make("shared") + + # No install provided + mkdirp(prefix.lib) + mkdirp(prefix.include) + install('lib/libzfp.so', prefix.lib) + install('inc/zfp.h', prefix.include) + install('inc/types.h', prefix.include) + install('inc/bitstream.h', prefix.include) + install('inc/system.h', prefix.include) From fc6aa7374a243928af38fd1e583c6c67146951cd Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 04:18:48 -0800 Subject: [PATCH 505/910] Fix #104, #54: issues with overlong shebang in deep directories. This does several things: - Add `sbang`: a script to run scripts with long shebang lines. - Documentation for `sbang` is in `bin/sbang`. - Add an `sbang` hook that filters the `bin` directory after install and modifies any scripts wtih shebangs that are too long to use `sbang` instead. - `sbang` is at the top level, so it should be runnable (not much we can do if spack itself is too deep for shebang) - `sbang`, when used as the interpreter, runs the *second* shebang line it finds in a script. - shoud fix issues with too long shebang paths. --- bin/sbang | 84 +++++++++++++++++++++++++++++++ lib/spack/llnl/util/filesystem.py | 11 +++- lib/spack/spack/hooks/sbang.py | 74 +++++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 1 deletion(-) create mode 100755 bin/sbang create mode 100644 lib/spack/spack/hooks/sbang.py diff --git a/bin/sbang b/bin/sbang new file mode 100755 index 00000000000..ebfbe2e7a16 --- /dev/null +++ b/bin/sbang @@ -0,0 +1,84 @@ +#!/bin/bash +# +# `sbang`: Run scripts with long shebang lines. +# +# Many operating systems limit the length of shebang lines, making it +# hard to use interpreters that are deep in the directory hierarchy. +# `sbang` can run such scripts, either as a shebang interpreter, or +# directly on the command line. +# +# Usage +# ----------------------------- +# Suppose you have a script, long-shebang.sh, like this: +# +# 1 #!/very/long/path/to/some/interpreter +# 2 +# 3 echo "success!" +# +# Invoking this script will result in an error on some OS's. On +# Linux, you get this: +# +# $ ./long-shebang.sh +# -bash: ./long: /very/long/path/to/some/interp: bad interpreter: +# No such file or directory +# +# On Mac OS X, the system simply assumes the interpreter is the shell +# and tries to run with it, which is likely not what you want. +# +# +# `sbang` on the command line +# ----------------------------- +# You can use `sbang` in two ways. The first is to use it directly, +# from the command line, like this: +# +# $ sbang ./long-shebang.sh +# success! +# +# +# `sbang` as the interpreter +# ----------------------------- +# You can also use `sbang` *as* the interpreter for your script. Put +# `#!/bin/bash /path/to/sbang` on line 1, and move the original +# shebang to line 2 of the script: +# +# 1 #!/bin/bash /path/to/sbang +# 2 #!/long/path/to/real/interpreter with arguments +# 3 +# 4 echo "success!" +# +# $ ./long-shebang.sh +# success! +# +# On Linux, you could shorten line 1 to `#!/path/to/sbang`, but other +# operating systems like Mac OS X require the interpreter to be a +# binary, so it's best to use `sbang` as a `bash` argument. +# Obviously, for this to work, `sbang` needs to have a short enough +# path that *it* will run without hitting OS limits. +# +# +# How it works +# ----------------------------- +# `sbang` is a very simple bash script. It looks at the first two +# lines of a script argument and runs the last line starting with +# `#!`, with the script as an argument. It also forwards arguments. +# + +# First argument is the script we want to actually run. +script="$1" + +# Search the first two lines of script for interpreters. +lines=0 +while read line && ((lines < 2)) ; do + if [[ "$line" = '#!'* ]]; then + interpreter="${line#\#!}" + fi + lines=$((lines+1)) +done < "$script" + +# Invoke any interpreter found, or raise an error if none was found. +if [ -n "$interpreter" ]; then + exec $interpreter "$@" +else + echo "error: sbang found no interpreter in $script" + exit 1 +fi diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index da3cf96050f..10d25bdce8d 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,8 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', + 'copy_mode', 'unset_executable_mode'] import os import sys @@ -158,6 +159,14 @@ def copy_mode(src, dest): os.chmod(dest, dest_mode) +def unset_executable_mode(path): + mode = os.stat(path).st_mode + mode &= ~stat.S_IXUSR + mode &= ~stat.S_IXGRP + mode &= ~stat.S_IXOTH + os.chmod(path, mode) + + def install(src, dest): """Manually install a file to a particular location.""" tty.debug("Installing %s to %s" % (src, dest)) diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py new file mode 100644 index 00000000000..6117c4809dc --- /dev/null +++ b/lib/spack/spack/hooks/sbang.py @@ -0,0 +1,74 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os + +from llnl.util.filesystem import * +import llnl.util.tty as tty + +import spack +import spack.modules + +# Character limit for shebang line. Using Linux's 127 characters +# here, as it is the shortest I could find on a modern OS. +shebang_limit = 127 + +def shebang_too_long(path): + """Detects whether an file has a shebang line that is too long.""" + with open(path, 'r') as script: + bytes = script.read(2) + if bytes != '#!': + return False + + line = bytes + script.readline() + return len(line) > shebang_limit + + +def filter_shebang(path): + """Adds a second shebang line, using sbang, at the beginning of a file.""" + backup = path + ".shebang.bak" + os.rename(path, backup) + + with open(backup, 'r') as bak_file: + original = bak_file.read() + + with open(path, 'w') as new_file: + new_file.write('#!/bin/bash %s/bin/sbang\n' % spack.spack_root) + new_file.write(original) + + copy_mode(backup, path) + unset_executable_mode(backup) + + tty.warn("Patched overly long shebang in %s" % path) + + +def post_install(pkg): + """This hook edits scripts so that they call /bin/bash + $spack_prefix/bin/sbang instead of something longer than the + shebang limit.""" + for file in os.listdir(pkg.prefix.bin): + path = os.path.join(pkg.prefix.bin, file) + if shebang_too_long(path): + filter_shebang(path) + From bfce2c7508ada675ebe4398395b30ede82b67093 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 04:42:52 -0800 Subject: [PATCH 506/910] Fix bug in hook: ensure bin directory exists before listing. --- lib/spack/spack/hooks/sbang.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 6117c4809dc..3390ecea295 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -67,6 +67,9 @@ def post_install(pkg): """This hook edits scripts so that they call /bin/bash $spack_prefix/bin/sbang instead of something longer than the shebang limit.""" + if not os.path.isdir(pkg.prefix.bin): + return + for file in os.listdir(pkg.prefix.bin): path = os.path.join(pkg.prefix.bin, file) if shebang_too_long(path): From 3dd630d0a567964acd11bee23ebbc52c7d1c61c5 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 14:33:23 -0800 Subject: [PATCH 507/910] Make openssl a variant in libevent. --- var/spack/repos/builtin/packages/libevent/package.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/libevent/package.py b/var/spack/repos/builtin/packages/libevent/package.py index 2a44c49325d..714a155dc00 100644 --- a/var/spack/repos/builtin/packages/libevent/package.py +++ b/var/spack/repos/builtin/packages/libevent/package.py @@ -22,12 +22,16 @@ class Libevent(Package): version('2.0.13', 'af786b4b3f790c9d3279792edf7867fc') version('2.0.12', '42986228baf95e325778ed328a93e070') - - depends_on('openssl') - + variant('openssl', default=True, description="Build with encryption enabled at the libevent level.") + depends_on('openssl', when='+openssl') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + configure_args = [] + if '+openssl' in spec: + configure_args.append('--enable-openssl') + else: + configure_args.append('--enable-openssl') + configure("--prefix=%s" % prefix, *configure_args) make() make("install") From 220b6a9fee0f307d4de1e48b29093812f7dd10ec Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Sat, 5 Mar 2016 16:46:32 -0600 Subject: [PATCH 508/910] Make libsigsegv an optional dependency --- var/spack/repos/builtin/packages/m4/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index 3890663ad12..9d522dfccf3 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -7,7 +7,9 @@ class M4(Package): version('1.4.17', 'a5e9954b1dae036762f7b13673a2cf76') - depends_on('libsigsegv') + variant('sigsegv', default=True, description="Build the libsigsegv dependency") + + depends_on('libsigsegv', when='+sigsegv') def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 7d847f4dc4f8895d57e492c11cfd7c3fbb794945 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 15:14:21 -0800 Subject: [PATCH 509/910] Minor tweaks to m4 -- be sure to add sigsegv args explicitly. --- var/spack/repos/builtin/packages/m4/package.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index 9d522dfccf3..d6829dbcd4b 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -12,6 +12,12 @@ class M4(Package): depends_on('libsigsegv', when='+sigsegv') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + configure_args = [] + if 'libsigsegv' in spec: + configure_args.append('--with-libsigsegv-prefix=%s' % spec['libsigsegv'].prefix) + else: + configure_args.append('--without-libsigsegv-prefix') + + configure("--prefix=%s" % prefix, *configure_args) make() make("install") From 14d48eba461596b3c5b03cfe269318946406e02c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 18:40:28 -0800 Subject: [PATCH 510/910] Revert c5d9ee8924 for bug fixes. - This reverts commit c5d9ee89246b3d2aeddb756a04588424051d3295. - merged too soon before - reverting and fixing bugs now. Conflicts: lib/spack/spack/mirror.py lib/spack/spack/package.py --- lib/spack/llnl/util/filesystem.py | 35 +++- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/mirror.py | 55 +++--- lib/spack/spack/package.py | 175 +++++++++---------- lib/spack/spack/stage.py | 186 ++++++++++----------- lib/spack/spack/test/concretize.py | 2 - lib/spack/spack/test/config.py | 6 +- lib/spack/spack/test/configure_guess.py | 21 +-- lib/spack/spack/test/database.py | 11 +- lib/spack/spack/test/directory_layout.py | 13 +- lib/spack/spack/test/git_fetch.py | 46 ++--- lib/spack/spack/test/hg_fetch.py | 44 +++-- lib/spack/spack/test/install.py | 9 +- lib/spack/spack/test/link_tree.py | 7 +- lib/spack/spack/test/lock.py | 8 +- lib/spack/spack/test/make_executable.py | 6 +- lib/spack/spack/test/mirror.py | 69 ++++---- lib/spack/spack/test/mock_packages_test.py | 8 +- lib/spack/spack/test/mock_repo.py | 4 - lib/spack/spack/test/multimethod.py | 5 +- lib/spack/spack/test/namespace_trie.py | 1 + lib/spack/spack/test/optional_deps.py | 4 +- lib/spack/spack/test/packages.py | 6 +- lib/spack/spack/test/python_version.py | 3 +- lib/spack/spack/test/spec_dag.py | 2 - lib/spack/spack/test/spec_semantics.py | 1 - lib/spack/spack/test/spec_syntax.py | 3 +- lib/spack/spack/test/stage.py | 130 ++++++-------- lib/spack/spack/test/svn_fetch.py | 49 +++--- lib/spack/spack/test/tally_plugin.py | 4 +- lib/spack/spack/test/unit_install.py | 3 +- lib/spack/spack/test/url_extrapolate.py | 3 - lib/spack/spack/test/url_parse.py | 2 +- lib/spack/spack/test/url_substitution.py | 1 - lib/spack/spack/test/versions.py | 1 + lib/spack/spack/test/yaml.py | 1 + 36 files changed, 422 insertions(+), 504 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index da3cf96050f..366237ef8f3 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links', 'remove_linked_tree'] import os import sys @@ -240,7 +240,7 @@ def touchp(path): def force_symlink(src, dest): try: os.symlink(src, dest) - except OSError, e: + except OSError as e: os.remove(dest) os.symlink(src, dest) @@ -344,3 +344,34 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): if order == 'post': yield (source_path, dest_path) + +def remove_dead_links(root): + """ + Removes any dead link that is present in root + + Args: + root: path where to search for dead links + + """ + for file in os.listdir(root): + path = join_path(root, file) + if os.path.islink(path): + real_path = os.path.realpath(path) + if not os.path.exists(real_path): + os.unlink(path) + +def remove_linked_tree(path): + """ + Removes a directory and its contents. If the directory is a symlink, follows the link and removes the real + directory before removing the link. + + Args: + path: directory to be removed + + """ + if os.path.exists(path): + if os.path.islink(path): + shutil.rmtree(os.path.realpath(path), True) + os.unlink(path) + else: + shutil.rmtree(path, True) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 6e7179122c1..0c8bd1d5281 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -43,4 +43,4 @@ def clean(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.do_clean() + package.stage.destroy() diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 58e31c2c7b5..fdc4e7967f5 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -110,7 +110,6 @@ def suggest_archive_basename(resource): return basename - def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with package archives. @@ -158,17 +157,29 @@ def create(path, specs, **kwargs): "Cannot create directory '%s':" % mirror_root, str(e)) # Things to keep track of while parsing specs. - present = [] - mirrored = [] - error = [] + categories = { + 'present': [], + 'mirrored': [], + 'error': [] + } # Iterate through packages and download all the safe tarballs for each of them - everything_already_exists = True for spec in version_specs: - pkg = spec.package - tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) - try: - for ii, stage in enumerate(pkg.stage): + add_single_spec(spec, mirror_root, categories, **kwargs) + + return categories['present'], categories['mirrored'], categories['error'] + + +def add_single_spec(spec, mirror_root, categories, **kwargs): + tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) + spec_exists_in_mirror = True + try: + with spec.package.stage: + # fetcher = stage.fetcher + # fetcher.fetch() + # ... + # fetcher.archive(archive_path) + for ii, stage in enumerate(spec.package.stage): fetcher = stage.fetcher if ii == 0: # create a subdirectory for the current package@version @@ -184,7 +195,7 @@ def create(path, specs, **kwargs): if os.path.exists(archive_path): tty.msg("{name} : already added".format(name=name)) else: - everything_already_exists = False + spec_exists_in_mirror = False fetcher.fetch() if not kwargs.get('no_checksum', False): fetcher.check() @@ -195,20 +206,16 @@ def create(path, specs, **kwargs): fetcher.archive(archive_path) tty.msg("{name} : added".format(name=name)) - if everything_already_exists: - present.append(spec) - else: - mirrored.append(spec) - except Exception, e: - if spack.debug: - sys.excepthook(*sys.exc_info()) - else: - tty.warn("Error while fetching %s" % spec.format('$_$@'), e.message) - error.append(spec) - finally: - pkg.stage.destroy() - - return (present, mirrored, error) + if spec_exists_in_mirror: + categories['present'].append(spec) + else: + categories['mirrored'].append(spec) + except Exception as e: + if spack.debug: + sys.excepthook(*sys.exc_info()) + else: + tty.warn("Error while fetching %s" % spec.format('$_$@'), e.message) + categories['error'].append(spec) class MirrorError(spack.error.SpackError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index fb96f61de9e..be45415b759 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -293,7 +293,6 @@ class SomePackage(Package): .. code-block:: python - p.do_clean() # removes the stage directory entirely p.do_restage() # removes the build directory and # re-expands the archive. @@ -503,7 +502,6 @@ def fetcher(self): self._fetcher = self._make_fetcher() return self._fetcher - @fetcher.setter def fetcher(self, f): self._fetcher = f @@ -735,7 +733,7 @@ def do_patch(self): # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. if os.path.isfile(bad_file): - tty.msg("Patching failed last time. Restaging.") + tty.msg("Patching failed last time. Restaging.") self.stage.restage() self.stage.chdir_to_source() @@ -850,102 +848,102 @@ def do_install(self, make_jobs=make_jobs) start_time = time.time() - if not fake: - if not skip_patch: - self.do_patch() - else: - self.do_stage() - - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) - - def cleanup(): - if not keep_prefix: - # If anything goes wrong, remove the install prefix - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + - "Manually remove this directory to fix:", - self.prefix, wrap=True) - - - def real_work(): - try: - tty.msg("Building %s" % self.name) - - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) - - # Set up process's build environment before running install. - if fake: - self.do_fake_install() + with self.stage: + if not fake: + if not skip_patch: + self.do_patch() else: - # Do the real install in the source directory. - self.stage.chdir_to_source() + self.do_stage() - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) - # This redirects I/O to a build log (and optionally to the terminal) - log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), True): - dump_environment(env_path) - self.install(self.spec, self.prefix) + def cleanup(): + if not keep_prefix: + # If anything goes wrong, remove the install prefix + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed." + + "Manually remove this directory to fix:", + self.prefix, wrap=True) - # Ensure that something was actually installed. - self._sanity_check_install() + def real_work(): + try: + tty.msg("Building %s" % self.name) - # Move build log into install directory on success - if not fake: - log_install_path = spack.install_layout.build_log_path(self.spec) - env_install_path = spack.install_layout.build_env_path(self.spec) - install(log_path, log_install_path) - install(env_path, env_install_path) + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) - packages_dir = spack.install_layout.build_packages_path(self.spec) - dump_packages(self.spec, packages_dir) + # Set up process's build environment before running install. + if fake: + self.do_fake_install() + else: + # Do the real install in the source directory. + self.stage.chdir_to_source() - # On successful install, remove the stage. - if not keep_stage: - self.stage.destroy() + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + # This redirects I/O to a build log (and optionally to the terminal) + log_path = join_path(os.getcwd(), 'spack-build.out') + log_file = open(log_path, 'w') + with log_output(log_file, verbose, sys.stdout.isatty(), True): + dump_environment(env_path) + self.install(self.spec, self.prefix) - tty.msg("Successfully installed %s" % self.name, - "Fetch: %s. Build: %s. Total: %s" - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) + # Ensure that something was actually installed. + self._sanity_check_install() - except ProcessError, e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e + # Move build log into install directory on success + if not fake: + log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) + install(log_path, log_install_path) + install(env_path, env_install_path) - except: - # other exceptions just clean up and raise. - cleanup() - raise + packages_dir = spack.install_layout.build_packages_path(self.spec) + dump_packages(self.spec, packages_dir) - # Set parallelism before starting build. - self.make_jobs = make_jobs + # On successful install, remove the stage. + if not keep_stage: + self.stage.destroy() - # Do the build. - spack.build_environment.fork(self, real_work) + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - # note: PARENT of the build process adds the new package to - # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, self.prefix) + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) - # Once everything else is done, run post install hooks - spack.hooks.post_install(self) + except ProcessError as e: + # Annotate with location of build log. + e.build_log = log_path + cleanup() + raise e + + except: + # other exceptions just clean up and raise. + cleanup() + raise + + # Set parallelism before starting build. + self.make_jobs = make_jobs + + # Do the build. + spack.build_environment.fork(self, real_work) + + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, self.prefix) + + # Once everything else is done, run post install hooks + spack.hooks.post_install(self) def _sanity_check_install(self): @@ -1149,13 +1147,6 @@ def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() - - def do_clean(self): - """Removes the package's build stage and source tarball.""" - if os.path.exists(self.stage.path): - self.stage.destroy() - - def format_doc(self, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get('indent', 0) @@ -1192,7 +1183,7 @@ def fetch_remote_versions(self): try: return spack.util.web.find_versions_of_archive( *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) - except spack.error.NoNetworkConnectionError, e: + except spack.error.NoNetworkConnectionError as e: tty.die("Package.fetch_versions couldn't connect to:", e.url, e.message) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 5591cb9ba5a..a22982a6d43 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -42,33 +42,26 @@ class Stage(object): - """A Stage object manages a directory where some source code is - downloaded and built before being installed. It handles - fetching the source code, either as an archive to be expanded - or by checking it out of a repository. A stage's lifecycle - looks like this: + """ + A Stage object is a context manager that handles a directory where some source code is downloaded and built + before being installed. It handles fetching the source code, either as an archive to be expanded or by checking + it out of a repository. A stage's lifecycle looks like this: - Stage() - Constructor creates the stage directory. - fetch() - Fetch a source archive into the stage. - expand_archive() - Expand the source archive. - - Build and install the archive. This is handled by the Package class. - destroy() - Remove the stage once the package has been installed. + ``` + with Stage() as stage: # Context manager creates and destroys the stage directory + fetch() # Fetch a source archive into the stage. + expand_archive() # Expand the source archive. + # Build and install the archive. This is handled by the Package class. + ``` - If spack.use_tmp_stage is True, spack will attempt to create stages - in a tmp directory. Otherwise, stages are created directly in - spack.stage_path. + If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory. + Otherwise, stages are created directly in spack.stage_path. - There are two kinds of stages: named and unnamed. Named stages can - persist between runs of spack, e.g. if you fetched a tarball but - didn't finish building it, you won't have to fetch it again. + There are two kinds of stages: named and unnamed. Named stages can persist between runs of spack, e.g. if you + fetched a tarball but didn't finish building it, you won't have to fetch it again. - Unnamed stages are created using standard mkdtemp mechanisms or - similar, and are intended to persist for only one run of spack. + Unnamed stages are created using standard mkdtemp mechanisms or similar, and are intended to persist for + only one run of spack. """ def __init__(self, url_or_fetch_strategy, **kwargs): @@ -96,21 +89,46 @@ def __init__(self, url_or_fetch_strategy, **kwargs): self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. - self.name = kwargs.get('name') + # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name + # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root + self.name = kwargs.get('name') if 'name' in kwargs else STAGE_PREFIX + next(tempfile._get_candidate_names()) self.mirror_path = kwargs.get('mirror_path') self.tmp_root = find_tmp_root() - self.path = None - self._setup() + # Try to construct here a temporary name for the stage directory + # If this is a named stage, then construct a named path. + self.path = join_path(spack.stage_path, self.name) + # Flag to decide whether to delete the stage folder on exit or not + self.delete_on_exit = True - def _cleanup_dead_links(self): - """Remove any dead links in the stage directory.""" - for file in os.listdir(spack.stage_path): - path = join_path(spack.stage_path, file) - if os.path.islink(path): - real_path = os.path.realpath(path) - if not os.path.exists(path): - os.unlink(path) + def __enter__(self): + """ + Entering a stage context will create the stage directory + + Returns: + self + """ + self.create() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Exiting from a stage context will delete the stage directory unless: + - it was explicitly requested not to do so + - an exception has been raised + + Args: + exc_type: exception type + exc_val: exception value + exc_tb: exception traceback + + Returns: + Boolean + """ + self.delete_on_exit = False if exc_type is not None else self.delete_on_exit + + if self.delete_on_exit: + self.destroy() def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we @@ -148,54 +166,6 @@ def _need_to_create_path(self): return False - def _setup(self): - """Creates the stage directory. - If spack.use_tmp_stage is False, the stage directory is created - directly under spack.stage_path. - - If spack.use_tmp_stage is True, this will attempt to create a - stage in a temporary directory and link it into spack.stage_path. - Spack will use the first writable location in spack.tmp_dirs to - create a stage. If there is no valid location in tmp_dirs, fall - back to making the stage inside spack.stage_path. - """ - # Create the top-level stage directory - mkdirp(spack.stage_path) - self._cleanup_dead_links() - - # If this is a named stage, then construct a named path. - if self.name is not None: - self.path = join_path(spack.stage_path, self.name) - - # If this is a temporary stage, them make the temp directory - tmp_dir = None - if self.tmp_root: - if self.name is None: - # Unnamed tmp root. Link the path in - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - self.name = os.path.basename(tmp_dir) - self.path = join_path(spack.stage_path, self.name) - if self._need_to_create_path(): - os.symlink(tmp_dir, self.path) - - else: - if self._need_to_create_path(): - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) - - # if we're not using a tmp dir, create the stage directly in the - # stage dir, rather than linking to it. - else: - if self.name is None: - self.path = tempfile.mkdtemp('', STAGE_PREFIX, spack.stage_path) - self.name = os.path.basename(self.path) - else: - if self._need_to_create_path(): - mkdirp(self.path) - - # Make sure we can actually do something with the stage we made. - ensure_access(self.path) - @property def archive_file(self): """Path to the source archive within this stage directory.""" @@ -276,7 +246,7 @@ def fetch(self, mirror_only=False): self.fetcher = fetcher self.fetcher.fetch() break - except spack.error.SpackError, e: + except spack.error.SpackError as e: tty.msg("Fetching from %s failed." % fetcher) tty.debug(e) continue @@ -328,8 +298,34 @@ def restage(self): """ self.fetcher.reset() + def create(self): + """ + Creates the stage directory + + If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise + this will attempt to create a stage in a temporary directory and link it into spack.stage_path. + + Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location + in tmp_dirs, fall back to making the stage inside spack.stage_path. + """ + # Create the top-level stage directory + mkdirp(spack.stage_path) + remove_dead_links(spack.stage_path) + # If a tmp_root exists then create a directory there and then link it in the stage area, + # otherwise create the stage directory in self.path + if self._need_to_create_path(): + if self.tmp_root: + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + os.symlink(tmp_dir, self.path) + else: + mkdirp(self.path) + # Make sure we can actually do something with the stage we made. + ensure_access(self.path) + def destroy(self): - """Remove this stage directory.""" + """ + Removes this stage directory + """ remove_linked_tree(self.path) # Make sure we don't end up in a removed directory @@ -389,6 +385,15 @@ def source_path(self): def path(self): return self[0].path + def __enter__(self): + for item in self: + item.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + for item in reversed(self): + item.__exit__(exc_type, exc_val, exc_tb) + def chdir_to_source(self): return self[0].chdir_to_source() @@ -439,19 +444,6 @@ def ensure_access(file=spack.stage_path): tty.die("Insufficient permissions for %s" % file) -def remove_linked_tree(path): - """Removes a directory and its contents. If the directory is a symlink, - follows the link and reamoves the real directory before removing the - link. - """ - if os.path.exists(path): - if os.path.islink(path): - shutil.rmtree(os.path.realpath(path), True) - os.unlink(path) - else: - shutil.rmtree(path, True) - - def purge(): """Remove all build directories in the top-level stage path.""" if os.path.isdir(spack.stage_path): diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 7f2938aec5a..794344fb6ae 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -22,8 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest - import spack from spack.spec import Spec, CompilerSpec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index d8be5a855b8..0562d2d6209 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -22,13 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest -import shutil import os +import shutil from tempfile import mkdtemp -from ordereddict_backport import OrderedDict + import spack import spack.config +from ordereddict_backport import OrderedDict from spack.test.mock_packages_test import * # Some sample compiler config data diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index a4e8565b62b..2440d120e5d 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -23,20 +23,15 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest import shutil import tempfile +import unittest from llnl.util.filesystem import * - from spack.cmd.create import ConfigureGuesser from spack.stage import Stage - -from spack.fetch_strategy import URLFetchStrategy -from spack.directory_layout import YamlDirectoryLayout -from spack.util.executable import which from spack.test.mock_packages_test import * -from spack.test.mock_repo import MockArchive +from spack.util.executable import which class InstallTest(unittest.TestCase): @@ -52,8 +47,6 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) - if self.stage: - self.stage.destroy() os.chdir(self.orig_dir) @@ -64,12 +57,12 @@ def check_archive(self, filename, system): url = 'file://' + join_path(os.getcwd(), 'archive.tar.gz') print url - self.stage = Stage(url) - self.stage.fetch() + with Stage(url) as stage: + stage.fetch() - guesser = ConfigureGuesser() - guesser(self.stage) - self.assertEqual(system, guesser.build_system) + guesser = ConfigureGuesser() + guesser(stage) + self.assertEqual(system, guesser.build_system) def test_python(self): diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 0205f4b8ce2..9a57e1f03ea 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -26,19 +26,18 @@ These tests check the database is functioning properly, both in memory and in its file """ -import tempfile -import shutil import multiprocessing - -from llnl.util.lock import * -from llnl.util.filesystem import join_path +import shutil +import tempfile import spack +from llnl.util.filesystem import join_path +from llnl.util.lock import * +from llnl.util.tty.colify import colify from spack.database import Database from spack.directory_layout import YamlDirectoryLayout from spack.test.mock_packages_test import * -from llnl.util.tty.colify import colify def _print_ref_counts(): """Print out all ref counts for the graph used here, for debugging""" diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 925cb648edd..d814572d4a0 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -25,20 +25,17 @@ """\ This test verifies that the Spack directory layout works properly. """ -import unittest -import tempfile -import shutil import os - -from llnl.util.filesystem import * +import shutil +import tempfile import spack -from spack.spec import Spec -from spack.repository import RepoPath +from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout +from spack.repository import RepoPath +from spack.spec import Spec from spack.test.mock_packages_test import * - # number of packages to test (to reduce test time) max_packages = 10 diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index d84433176a8..35780441166 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -23,19 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest -import shutil -import tempfile - -from llnl.util.filesystem import * import spack -from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which - +from llnl.util.filesystem import * from spack.test.mock_packages_test import * from spack.test.mock_repo import MockGitRepo +from spack.version import ver class GitFetchTest(MockPackagesTest): @@ -52,19 +45,15 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def assert_rev(self, rev): """Check that the current git revision is equal to the supplied rev.""" self.assertEqual(self.repo.rev_hash('HEAD'), self.repo.rev_hash(rev)) - def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with @@ -76,26 +65,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('git')] = args - self.pkg.do_stage() - self.assert_rev(rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked_file = 'foobarbaz' - touch(untracked_file) - self.assertTrue(os.path.isfile(untracked_file)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked_file)) + untracked_file = 'foobarbaz' + touch(untracked_file) + self.assertTrue(os.path.isfile(untracked_file)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked_file)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_master(self): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index bbcb64e4c1a..b8a0c1ec460 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -23,16 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest - -from llnl.util.filesystem import * - import spack + from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which -from spack.test.mock_packages_test import * from spack.test.mock_repo import MockHgRepo +from llnl.util.filesystem import * +from spack.test.mock_packages_test import * class HgFetchTest(MockPackagesTest): @@ -49,13 +45,10 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(HgFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def try_fetch(self, rev, test_file, args): """Tries to: @@ -68,26 +61,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('hg')] = args - self.pkg.do_stage() - self.assertEqual(self.repo.get_rev(), rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assertEqual(self.repo.get_rev(), rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assertEqual(self.repo.get_rev(), rev) + self.assertEqual(self.repo.get_rev(), rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 8863d13c426..8297893f012 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -22,18 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os -import unittest import shutil import tempfile -from llnl.util.filesystem import * - import spack -from spack.stage import Stage -from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite +from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout -from spack.util.executable import which +from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite from spack.test.mock_packages_test import * from spack.test.mock_repo import MockArchive diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index 886b7ef4c5d..ee37e765c7b 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -24,8 +24,6 @@ ############################################################################## import os import unittest -import shutil -import tempfile from llnl.util.filesystem import * from llnl.util.link_tree import LinkTree @@ -38,6 +36,7 @@ class LinkTreeTest(unittest.TestCase): def setUp(self): self.stage = Stage('link-tree-test') + self.stage.create() with working_dir(self.stage.path): touchp('source/1') @@ -51,10 +50,8 @@ def setUp(self): source_path = os.path.join(self.stage.path, 'source') self.link_tree = LinkTree(source_path) - def tearDown(self): - if self.stage: - self.stage.destroy() + self.stage.destroy() def check_file_link(self, filename): diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py index bc68df01dbe..3b11d18da49 100644 --- a/lib/spack/spack/test/lock.py +++ b/lib/spack/spack/test/lock.py @@ -25,15 +25,13 @@ """ These tests ensure that our lock works correctly. """ -import unittest -import os -import tempfile import shutil +import tempfile +import unittest from multiprocessing import Process -from llnl.util.lock import * from llnl.util.filesystem import join_path, touch - +from llnl.util.lock import * from spack.util.multiproc import Barrier # This is the longest a failed test will take, as the barriers will diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index d568a28d443..a2606acf19b 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -28,13 +28,13 @@ This just tests whether the right args are getting passed to make. """ import os -import unittest -import tempfile import shutil +import tempfile +import unittest from llnl.util.filesystem import * -from spack.util.environment import path_put_first from spack.build_environment import MakeExecutable +from spack.util.environment import path_put_first class MakeExecutableTest(unittest.TestCase): diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index f83cc8090c9..e707adfe9d5 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -23,11 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from filecmp import dircmp - import spack import spack.mirror -from spack.util.compression import decompressor_for + +from filecmp import dircmp from spack.test.mock_packages_test import * from spack.test.mock_repo import * @@ -74,14 +73,14 @@ def set_up_package(self, name, MockRepoClass, url_attr): def check_mirror(self): - stage = Stage('spack-mirror-test') - mirror_root = join_path(stage.path, 'test-mirror') + with Stage('spack-mirror-test') as stage: + mirror_root = join_path(stage.path, 'test-mirror') + + # register mirror with spack config + mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } + spack.config.update_config('mirrors', mirrors) - # register mirror with spack config - mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } - spack.config.update_config('mirrors', mirrors) - try: os.chdir(stage.path) spack.mirror.create( mirror_root, self.repos, no_checksum=True) @@ -97,38 +96,28 @@ def check_mirror(self): files = os.listdir(subdir) self.assertEqual(len(files), 1) - # Now try to fetch each package. - for name, mock_repo in self.repos.items(): - spec = Spec(name).concretized() - pkg = spec.package + # Now try to fetch each package. + for name, mock_repo in self.repos.items(): + spec = Spec(name).concretized() + pkg = spec.package - pkg._stage = None - saved_checksum_setting = spack.do_checksum - try: - # Stage the archive from the mirror and cd to it. - spack.do_checksum = False - pkg.do_stage(mirror_only=True) - - # Compare the original repo with the expanded archive - original_path = mock_repo.path - if 'svn' in name: - # have to check out the svn repo to compare. - original_path = join_path(mock_repo.path, 'checked_out') - svn('checkout', mock_repo.url, original_path) - - dcmp = dircmp(original_path, pkg.stage.source_path) - - # make sure there are no new files in the expanded tarball - self.assertFalse(dcmp.right_only) - - # and that all original files are present. - self.assertTrue(all(l in exclude for l in dcmp.left_only)) - - finally: - spack.do_checksum = saved_checksum_setting - pkg.do_clean() - finally: - stage.destroy() + saved_checksum_setting = spack.do_checksum + with pkg.stage: + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) + # Compare the original repo with the expanded archive + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) + dcmp = dircmp(original_path, pkg.stage.source_path) + # make sure there are no new files in the expanded tarball + self.assertFalse(dcmp.right_only) + # and that all original files are present. + self.assertTrue(all(l in exclude for l in dcmp.left_only)) + spack.do_checksum = saved_checksum_setting def test_git_mirror(self): diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index e9f1f95df55..0b8867b61e2 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -22,17 +22,15 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys import os import shutil -import unittest import tempfile -from ordereddict_backport import OrderedDict - -from llnl.util.filesystem import mkdirp +import unittest import spack import spack.config +from llnl.util.filesystem import mkdirp +from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index ed94023b0eb..a8bdfb55710 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -26,13 +26,9 @@ import shutil from llnl.util.filesystem import * - -import spack -from spack.version import ver from spack.stage import Stage from spack.util.executable import which - # # VCS Systems used by mock repo code. # diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 7bf4ff0a0a5..2d4b8cd584a 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -25,14 +25,11 @@ """ Test for multi_method dispatch. """ -import unittest import spack from spack.multimethod import * -from spack.version import * -from spack.spec import Spec -from spack.multimethod import when from spack.test.mock_packages_test import * +from spack.version import * class MultiMethodTest(MockPackagesTest): diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index 647976df210..2023ba6d96f 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest + from spack.util.naming import NamespaceTrie diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index ebd72819992..55f35ea4c96 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -22,10 +22,8 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest -import spack -from spack.spec import Spec, CompilerSpec +from spack.spec import Spec from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 83984dc5f65..f0b5e05f3b5 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -22,14 +22,12 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest - -from llnl.util.filesystem import join_path import spack +from llnl.util.filesystem import join_path from spack.repository import Repo -from spack.util.naming import mod_to_class from spack.test.mock_packages_test import * +from spack.util.naming import mod_to_class class PackagesTest(MockPackagesTest): diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index d74d3b9b7d7..42949753044 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -28,12 +28,11 @@ Spack was originally 2.7, but enough systems in 2014 are still using 2.6 on their frontend nodes that we need 2.6 to get adopted. """ -import unittest import os import re +import unittest import llnl.util.tty as tty - import pyqver2 import spack diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 632f777cdeb..5e6162b6e66 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -31,8 +31,6 @@ import spack import spack.package -from llnl.util.lang import list_modules - from spack.spec import Spec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 44a09cbd7fb..8c33d1ff6e7 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -22,7 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest from spack.spec import * from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 1daaa4be8fc..6e08e30e135 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -23,9 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest + import spack.spec -from spack.spec import * from spack.parse import Token +from spack.spec import * # Sample output for a complex lexing. complex_lex = [Token(ID, 'mvapich_foo'), diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index c1b2a2a5730..dbcf89d8645 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -25,15 +25,13 @@ """\ Test that the Stage class works correctly. """ -import unittest -import shutil import os -import getpass +import shutil +import unittest from contextlib import * -from llnl.util.filesystem import * - import spack +from llnl.util.filesystem import * from spack.stage import Stage from spack.util.executable import which @@ -192,116 +190,90 @@ def check_destroy(self, stage, stage_name): def test_setup_and_destroy_name_with_tmp(self): with use_tmp(True): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + self.check_setup(stage, stage_name) self.check_destroy(stage, stage_name) def test_setup_and_destroy_name_without_tmp(self): with use_tmp(False): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + self.check_setup(stage, stage_name) self.check_destroy(stage, stage_name) def test_setup_and_destroy_no_name_with_tmp(self): with use_tmp(True): - stage = Stage(archive_url) - self.check_setup(stage, None) - - stage.destroy() + with Stage(archive_url) as stage: + self.check_setup(stage, None) self.check_destroy(stage, None) def test_setup_and_destroy_no_name_without_tmp(self): with use_tmp(False): - stage = Stage(archive_url) - self.check_setup(stage, None) - - stage.destroy() + with Stage(archive_url) as stage: + self.check_setup(stage, None) self.check_destroy(stage, None) def test_chdir(self): - stage = Stage(archive_url, name=stage_name) - - stage.chdir() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.chdir() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) self.check_destroy(stage, stage_name) def test_fetch(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) self.check_destroy(stage, stage_name) def test_expand_archive(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.expand_archive() - self.check_expand_archive(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.expand_archive() + self.check_expand_archive(stage, stage_name) self.check_destroy(stage, stage_name) def test_expand_archive(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) self.check_destroy(stage, stage_name) def test_restage(self): - stage = Stage(archive_url, name=stage_name) + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) - stage.fetch() - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) + # Try to make a file in the old archive dir + with open('foobar', 'w') as file: + file.write("this file is to be destroyed.") - # Try to make a file in the old archive dir - with open('foobar', 'w') as file: - file.write("this file is to be destroyed.") + self.assertTrue('foobar' in os.listdir(stage.source_path)) - self.assertTrue('foobar' in os.listdir(stage.source_path)) - - # Make sure the file is not there after restage. - stage.restage() - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.chdir_to_source() - self.check_chdir_to_source(stage, stage_name) - self.assertFalse('foobar' in os.listdir(stage.source_path)) - - stage.destroy() + # Make sure the file is not there after restage. + stage.restage() + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.chdir_to_source() + self.check_chdir_to_source(stage, stage_name) + self.assertFalse('foobar' in os.listdir(stage.source_path)) self.check_destroy(stage, stage_name) diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 454a7f1d1ff..1ee4ee700e7 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -24,18 +24,12 @@ ############################################################################## import os import re -import unittest -import shutil -import tempfile - -from llnl.util.filesystem import * - import spack -from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which -from spack.test.mock_packages_test import * + from spack.test.mock_repo import svn, MockSvnRepo +from spack.version import ver +from spack.test.mock_packages_test import * +from llnl.util.filesystem import * class SvnFetchTest(MockPackagesTest): @@ -51,13 +45,10 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(SvnFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def assert_rev(self, rev): """Check that the current revision is equal to the supplied rev.""" @@ -70,7 +61,6 @@ def get_rev(): return match.group(1) self.assertEqual(get_rev(), rev) - def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with @@ -82,26 +72,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('svn')] = args - self.pkg.do_stage() - self.assert_rev(rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index e0b9618e0cb..4163ab95dd6 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -22,10 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from nose.plugins import Plugin - import os +from nose.plugins import Plugin + class Tally(Plugin): name = 'tally' diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index ccc409dd602..18615b7efe8 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -22,10 +22,11 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest import itertools +import unittest import spack + test_install = __import__("spack.cmd.test-install", fromlist=["BuildId", "create_test_output", "TestResult"]) diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py index 87adf894017..068a335b491 100644 --- a/lib/spack/spack/test/url_extrapolate.py +++ b/lib/spack/spack/test/url_extrapolate.py @@ -25,10 +25,7 @@ """\ Tests ability of spack to extrapolate URL versions from existing versions. """ -import spack import spack.url as url -from spack.spec import Spec -from spack.version import ver from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index efde7c0c73b..561d4658a10 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -27,8 +27,8 @@ detection in Homebrew. """ import unittest + import spack.url as url -from pprint import pprint class UrlParseTest(unittest.TestCase): diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index aec8baf4eaa..2be38af0d34 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -27,7 +27,6 @@ """ import unittest -import spack import spack.url as url diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 108450e098e..2732006eb3d 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -28,6 +28,7 @@ where it makes sense. """ import unittest + from spack.version import * diff --git a/lib/spack/spack/test/yaml.py b/lib/spack/spack/test/yaml.py index 5a357b8e696..b930c022f26 100644 --- a/lib/spack/spack/test/yaml.py +++ b/lib/spack/spack/test/yaml.py @@ -26,6 +26,7 @@ Test Spack's custom YAML format. """ import unittest + import spack.util.spack_yaml as syaml test_file = """\ From ad103dcafa652a839590f5fce28b2e2ce3b5a56d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 19:55:07 -0800 Subject: [PATCH 511/910] Small refactor: add keep parameter to stage, get rid of stage.destroy call. - package.py uses context manager more effectively. - Stage.__init__ has easier to understand method signature now. - keep can be used to override the default behavior either to keep the stage ALL the time or to delete the stage ALL the time. --- lib/spack/llnl/util/filesystem.py | 5 +- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/package.py | 108 +++++++++++++++------------- lib/spack/spack/stage.py | 116 ++++++++++++++++++++---------- 4 files changed, 141 insertions(+), 90 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 366237ef8f3..a92cb0706d2 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -362,8 +362,9 @@ def remove_dead_links(root): def remove_linked_tree(path): """ - Removes a directory and its contents. If the directory is a symlink, follows the link and removes the real - directory before removing the link. + Removes a directory and its contents. If the directory is a + symlink, follows the link and removes the real directory before + removing the link. Args: path: directory to be removed diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 0c8bd1d5281..6e7179122c1 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -43,4 +43,4 @@ def clean(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.stage.destroy() + package.do_clean() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index be45415b759..47d259968aa 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -293,6 +293,7 @@ class SomePackage(Package): .. code-block:: python + p.do_clean() # removes the stage directory entirely p.do_restage() # removes the build directory and # re-expands the archive. @@ -455,7 +456,7 @@ def _make_stage(self): # Construct a composite stage on top of the composite FetchStrategy composite_fetcher = self.fetcher composite_stage = StageComposite() - resources = self._get_resources() + resources = self._get_needed_resources() for ii, fetcher in enumerate(composite_fetcher): if ii == 0: # Construct root stage first @@ -484,12 +485,14 @@ def stage(self, stage): def _make_fetcher(self): - # Construct a composite fetcher that always contains at least one element (the root package). In case there - # are resources associated with the package, append their fetcher to the composite. + # Construct a composite fetcher that always contains at least + # one element (the root package). In case there are resources + # associated with the package, append their fetcher to the + # composite. root_fetcher = fs.for_package_version(self, self.version) fetcher = fs.FetchStrategyComposite() # Composite fetcher fetcher.append(root_fetcher) # Root fetcher is always present - resources = self._get_resources() + resources = self._get_needed_resources() for resource in resources: fetcher.append(resource.fetcher) return fetcher @@ -706,6 +709,7 @@ def do_stage(self, mirror_only=False): self.stage.expand_archive() self.stage.chdir_to_source() + def do_patch(self): """Calls do_stage(), then applied patches to the expanded tarball if they haven't been applied already.""" @@ -798,7 +802,7 @@ def do_fake_install(self): mkdirp(self.prefix.man1) - def _get_resources(self): + def _get_needed_resources(self): resources = [] # Select the resources that are needed for this build for when_spec, resource_list in self.resources.items(): @@ -816,7 +820,7 @@ def _resource_stage(self, resource): def do_install(self, - keep_prefix=False, keep_stage=False, ignore_deps=False, + keep_prefix=False, keep_stage=None, ignore_deps=False, skip_patch=False, verbose=False, make_jobs=None, fake=False): """Called by commands to install a package and its dependencies. @@ -825,7 +829,8 @@ def do_install(self, Args: keep_prefix -- Keep install prefix on failure. By default, destroys it. - keep_stage -- Keep stage on successful build. By default, destroys it. + keep_stage -- Set to True or false to always keep or always delete stage. + By default, stage is destroyed only if there are no exceptions. ignore_deps -- Do not install dependencies before installing this package. fake -- Don't really build -- install fake stub files instead. skip_patch -- Skip patch stage of build if True. @@ -848,32 +853,33 @@ def do_install(self, make_jobs=make_jobs) start_time = time.time() - with self.stage: - if not fake: - if not skip_patch: - self.do_patch() - else: - self.do_stage() + if not fake: + if not skip_patch: + self.do_patch() + else: + self.do_stage() - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) - def cleanup(): - if not keep_prefix: - # If anything goes wrong, remove the install prefix - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + - "Manually remove this directory to fix:", - self.prefix, wrap=True) + def cleanup(): + if not keep_prefix: + # If anything goes wrong, remove the install prefix + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed." + + "Manually remove this directory to fix:", + self.prefix, wrap=True) - def real_work(): - try: - tty.msg("Building %s" % self.name) + def real_work(): + try: + tty.msg("Building %s" % self.name) + self.stage.keep = keep_stage + with self.stage: # Run the pre-install hook in the child process after # the directory is created. spack.hooks.pre_install(self) @@ -888,7 +894,7 @@ def real_work(): # Save the build environment in a file before building. env_path = join_path(os.getcwd(), 'spack-build.env') - # This redirects I/O to a build log (and optionally to the terminal) + # Redirect I/O to a build log (and optionally to the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') log_file = open(log_path, 'w') with log_output(log_file, verbose, sys.stdout.isatty(), True): @@ -908,29 +914,25 @@ def real_work(): packages_dir = spack.install_layout.build_packages_path(self.spec) dump_packages(self.spec, packages_dir) - # On successful install, remove the stage. - if not keep_stage: - self.stage.destroy() + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) - tty.msg("Successfully installed %s" % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) + except ProcessError as e: + # Annotate with location of build log. + e.build_log = log_path + cleanup() + raise e - except ProcessError as e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e - - except: - # other exceptions just clean up and raise. - cleanup() - raise + except: + # other exceptions just clean up and raise. + cleanup() + raise # Set parallelism before starting build. self.make_jobs = make_jobs @@ -1147,6 +1149,12 @@ def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() + + def do_clean(self): + """Removes the package's build stage and source tarball.""" + self.stage.destroy() + + def format_doc(self, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get('indent', 0) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index a22982a6d43..b117c76aa17 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -42,29 +42,53 @@ class Stage(object): - """ - A Stage object is a context manager that handles a directory where some source code is downloaded and built - before being installed. It handles fetching the source code, either as an archive to be expanded or by checking - it out of a repository. A stage's lifecycle looks like this: + """Manages a temporary stage directory for building. + + A Stage object is a context manager that handles a directory where + some source code is downloaded and built before being installed. + It handles fetching the source code, either as an archive to be + expanded or by checking it out of a repository. A stage's + lifecycle looks like this: ``` - with Stage() as stage: # Context manager creates and destroys the stage directory - fetch() # Fetch a source archive into the stage. - expand_archive() # Expand the source archive. - # Build and install the archive. This is handled by the Package class. + with Stage() as stage: # Context manager creates and destroys the stage directory + stage.fetch() # Fetch a source archive into the stage. + stage.expand_archive() # Expand the source archive. + # Build and install the archive. (handled by user of Stage) ``` - If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory. - Otherwise, stages are created directly in spack.stage_path. + When used as a context manager, the stage is automatically + destroyed if no exception is raised by the context. If an + excpetion is raised, the stage is left in the filesystem and NOT + destroyed, for potential reuse later. - There are two kinds of stages: named and unnamed. Named stages can persist between runs of spack, e.g. if you - fetched a tarball but didn't finish building it, you won't have to fetch it again. + You can also use the stage's create/destroy functions manually, + like this: - Unnamed stages are created using standard mkdtemp mechanisms or similar, and are intended to persist for - only one run of spack. + ``` + stage = Stage() + try: + stage.create() # Explicitly create the stage directory. + stage.fetch() # Fetch a source archive into the stage. + stage.expand_archive() # Expand the source archive. + # Build and install the archive. (handled by user of Stage) + finally: + stage.destroy() # Explicitly destroy the stage directory. + ``` + + If spack.use_tmp_stage is True, spack will attempt to create + stages in a tmp directory. Otherwise, stages are created directly + in spack.stage_path. + + There are two kinds of stages: named and unnamed. Named stages + can persist between runs of spack, e.g. if you fetched a tarball + but didn't finish building it, you won't have to fetch it again. + + Unnamed stages are created using standard mkdtemp mechanisms or + similar, and are intended to persist for only one run of spack. """ - def __init__(self, url_or_fetch_strategy, **kwargs): + def __init__(self, url_or_fetch_strategy, name=None, mirror_path=None, keep=None): """Create a stage object. Parameters: url_or_fetch_strategy @@ -76,6 +100,18 @@ def __init__(self, url_or_fetch_strategy, **kwargs): and will persist between runs (or if you construct another stage object later). If name is not provided, then this stage will be given a unique name automatically. + + mirror_path + If provided, Stage will search Spack's mirrors for + this archive at the mirror_path, before using the + default fetch strategy. + + keep + By default, when used as a context manager, the Stage + is cleaned up when everything goes well, and it is + kept intact when an exception is raised. You can + override this behavior by setting keep to True + (always keep) or False (always delete). """ # TODO: fetch/stage coupling needs to be reworked -- the logic # TODO: here is convoluted and not modular enough. @@ -91,15 +127,19 @@ def __init__(self, url_or_fetch_strategy, **kwargs): # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root - self.name = kwargs.get('name') if 'name' in kwargs else STAGE_PREFIX + next(tempfile._get_candidate_names()) - self.mirror_path = kwargs.get('mirror_path') + self.name = name + if name is None: + self.name = STAGE_PREFIX + next(tempfile._get_candidate_names()) + self.mirror_path = mirror_path self.tmp_root = find_tmp_root() # Try to construct here a temporary name for the stage directory # If this is a named stage, then construct a named path. self.path = join_path(spack.stage_path, self.name) + # Flag to decide whether to delete the stage folder on exit or not - self.delete_on_exit = True + self.keep = keep + def __enter__(self): """ @@ -111,6 +151,7 @@ def __enter__(self): self.create() return self + def __exit__(self, exc_type, exc_val, exc_tb): """ Exiting from a stage context will delete the stage directory unless: @@ -125,11 +166,15 @@ def __exit__(self, exc_type, exc_val, exc_tb): Returns: Boolean """ - self.delete_on_exit = False if exc_type is not None else self.delete_on_exit + if self.keep is None: + # Default: delete when there are no exceptions. + if exc_type is None: self.destroy() - if self.delete_on_exit: + elif not self.keep: + # Overridden. Either always keep or always delete. self.destroy() + def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we looked at path. Returns True if path already exists and is ok. @@ -201,7 +246,7 @@ def chdir(self): if os.path.isdir(self.path): os.chdir(self.path) else: - tty.die("Setup failed: no such directory: " + self.path) + raise ChdirError("Setup failed: no such directory: " + self.path) def fetch(self, mirror_only=False): """Downloads an archive or checks out code from a repository.""" @@ -302,11 +347,14 @@ def create(self): """ Creates the stage directory - If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise - this will attempt to create a stage in a temporary directory and link it into spack.stage_path. + If self.tmp_root evaluates to False, the stage directory is + created directly under spack.stage_path, otherwise this will + attempt to create a stage in a temporary directory and link it + into spack.stage_path. - Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location - in tmp_dirs, fall back to making the stage inside spack.stage_path. + Spack will use the first writable location in spack.tmp_dirs + to create a stage. If there is no valid location in tmp_dirs, + fall back to making the stage inside spack.stage_path. """ # Create the top-level stage directory mkdirp(spack.stage_path) @@ -323,9 +371,7 @@ def create(self): ensure_access(self.path) def destroy(self): - """ - Removes this stage directory - """ + """Removes this stage directory.""" remove_linked_tree(self.path) # Make sure we don't end up in a removed directory @@ -370,7 +416,7 @@ def expand_archive(self): shutil.move(source_path, destination_path) -@pattern.composite(method_list=['fetch', 'check', 'expand_archive', 'restage', 'destroy']) +@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy']) class StageComposite: """ Composite for Stage type objects. The first item in this composite is considered to be the root package, and @@ -410,7 +456,7 @@ def chdir(self): if os.path.isdir(self.path): os.chdir(self.path) else: - tty.die("Setup failed: no such directory: " + self.path) + raise ChdirError("Setup failed: no such directory: " + self.path) def chdir_to_source(self): self.chdir() @@ -472,19 +518,15 @@ def find_tmp_root(): class StageError(spack.error.SpackError): - def __init__(self, message, long_message=None): - super(self, StageError).__init__(message, long_message) + """"Superclass for all errors encountered during staging.""" class RestageError(StageError): - def __init__(self, message, long_msg=None): - super(RestageError, self).__init__(message, long_msg) + """"Error encountered during restaging.""" class ChdirError(StageError): - def __init__(self, message, long_msg=None): - super(ChdirError, self).__init__(message, long_msg) - + """Raised when Spack can't change directories.""" # Keep this in namespace for convenience FailedDownloadError = fs.FailedDownloadError From ae87948a232c17982a7987e5423ac77acdc59194 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sat, 5 Mar 2016 23:05:45 -0500 Subject: [PATCH 512/910] Fixed URLs Removed vestigal mpi variant in netcdf-fortran --- var/spack/repos/builtin/packages/netcdf-cxx4/package.py | 1 - var/spack/repos/builtin/packages/netcdf-fortran/package.py | 4 +--- var/spack/repos/builtin/packages/netcdf/package.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py index ab717ac6ffb..b83e964b00a 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -11,6 +11,5 @@ class NetcdfCxx4(Package): def install(self, spec, prefix): configure('--prefix=%s' % prefix) - make() make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py index 954e7dc3e81..e4e33445e54 100644 --- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -3,13 +3,11 @@ class NetcdfFortran(Package): """Fortran interface for NetCDF4""" - homepage = "http://www.unidata.ucar.edu/downloads/netcdf/netcdf-cxx/index.jsp" + homepage = "http://www.unidata.ucar.edu/software/netcdf" url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-fortran-4.4.3.tar.gz" version('4.4.3', 'bfd4ae23a34635b273d3eb0d91cbde9e') - variant('mpi', default=True, description='Enables MPI parallelism') - depends_on('netcdf') def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 0b112a59ce9..227362399a9 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -6,7 +6,7 @@ class Netcdf(Package): data formats that support the creation, access, and sharing of array-oriented scientific data.""" - homepage = "http://www.unidata.ucar.edu/software/netcdf/" + homepage = "http://www.unidata.ucar.edu/software/netcdf" url = "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.3.3.tar.gz" version('4.4.0', 'cffda0cbd97fdb3a06e9274f7aef438e') From e515042a36e7aa21e52943dab1e8b5594f3f0e94 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 6 Mar 2016 01:41:48 -0800 Subject: [PATCH 513/910] Fix stage creation bug, simplify do_install code. --- lib/spack/spack/package.py | 73 +++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 47d259968aa..ce8cce27e23 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -467,6 +467,11 @@ def _make_stage(self): stage = self._make_resource_stage(composite_stage[0], fetcher, resource) # Append the item to the composite composite_stage.append(stage) + + # Create stage on first access. Needed because fetch, stage, + # patch, and install can be called independently of each + # other, so `with self.stage:` in do_install isn't sufficient. + composite_stage.create() return composite_stage @property @@ -846,35 +851,38 @@ def do_install(self, tty.msg("Installing %s" % self.name) + # First, install dependencies recursively. if not ignore_deps: self.do_install_dependencies( keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps, - fake=fake, skip_patch=skip_patch, verbose=verbose, - make_jobs=make_jobs) - - start_time = time.time() - if not fake: - if not skip_patch: - self.do_patch() - else: - self.do_stage() - - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) + fake=fake, skip_patch=skip_patch, verbose=verbose, make_jobs=make_jobs) def cleanup(): + """Handles removing install prefix on error.""" if not keep_prefix: - # If anything goes wrong, remove the install prefix self.remove_prefix() else: tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + + "Spack will think this package is installed. " + "Manually remove this directory to fix:", self.prefix, wrap=True) + # Then install the package itself. def real_work(): + """Forked for each build. Has its own process and python + module space set up by build_environment.fork().""" + start_time = time.time() + if not fake: + if not skip_patch: + self.do_patch() + else: + self.do_stage() + + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) + try: tty.msg("Building %s" % self.name) @@ -884,9 +892,9 @@ def real_work(): # the directory is created. spack.hooks.pre_install(self) - # Set up process's build environment before running install. if fake: self.do_fake_install() + else: # Do the real install in the source directory. self.stage.chdir_to_source() @@ -901,18 +909,17 @@ def real_work(): dump_environment(env_path) self.install(self.spec, self.prefix) - # Ensure that something was actually installed. - self._sanity_check_install() + # Ensure that something was actually installed. + self._sanity_check_install() - # Move build log into install directory on success - if not fake: + # Copy provenance into the install directory on success log_install_path = spack.install_layout.build_log_path(self.spec) env_install_path = spack.install_layout.build_env_path(self.spec) + packages_dir = spack.install_layout.build_packages_path(self.spec) + install(log_path, log_install_path) install(env_path, env_install_path) - - packages_dir = spack.install_layout.build_packages_path(self.spec) - dump_packages(self.spec, packages_dir) + dump_packages(self.spec, packages_dir) # Stop timer. self._total_time = time.time() - start_time @@ -934,18 +941,18 @@ def real_work(): cleanup() raise - # Set parallelism before starting build. - self.make_jobs = make_jobs + # Set parallelism before starting build. + self.make_jobs = make_jobs - # Do the build. - spack.build_environment.fork(self, real_work) + # Do the build. + spack.build_environment.fork(self, real_work) - # note: PARENT of the build process adds the new package to - # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, self.prefix) + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, self.prefix) - # Once everything else is done, run post install hooks - spack.hooks.post_install(self) + # Once everything else is done, run post install hooks + spack.hooks.post_install(self) def _sanity_check_install(self): From 240ada5775c7857932279d86e4305ef001d33717 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 6 Mar 2016 16:51:09 -0800 Subject: [PATCH 514/910] Add `expand=False` option for URL downloads. - Allows skipping the expand step for downloads. - Fixed stage so that it knows expansion didn't fail when there is a no-expand URLFetchStrategy. - Updated docs to reflect new option, and provided an example. --- lib/spack/docs/packaging_guide.rst | 29 ++++++++++++++++++ lib/spack/llnl/util/filesystem.py | 9 +++++- lib/spack/spack/fetch_strategy.py | 7 ++++- lib/spack/spack/mirror.py | 13 ++++++-- lib/spack/spack/stage.py | 48 ++++++++++++++++++++---------- 5 files changed, 86 insertions(+), 20 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 59ba63fa355..bae8c34d522 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -401,6 +401,35 @@ construct the new one for ``8.2.1``. When you supply a custom URL for a version, Spack uses that URL *verbatim* and does not perform extrapolation. +Skipping the expand step +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spack normally expands archives automatically after downloading +them. If you want to skip this step (e.g., for self-extracting +executables and other custom archive types), you can add +``expand=False`` to a ``version`` directive. + +.. code-block:: python + + version('8.2.1', '4136d7b4c04df68b686570afa26988ac', + url='http://example.com/foo-8.2.1-special-version.tar.gz', 'expand=False') + +When ``expand`` is set to ``False``, Spack sets the current working +directory to the directory containing the downloaded archive before it +calls your ``install`` method. Within ``install``, the path to the +downloaded archive is available as ``self.stage.archive_file``. + +Here is an example snippet for packages distribuetd as self-extracting +archives. The example sets permissions on the downloaded file to make +it executable, then runs it with some arguments. + +.. code-block:: python + + def install(self, spec, prefix): + set_executable(self.stage.archive_file) + installer = Executable(self.stage.archive_file) + installer('--prefix=%s' % prefix, 'arg1', 'arg2', 'etc.') + Checksums ~~~~~~~~~~~~~~~~~ diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index a92cb0706d2..f218b7c424b 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,8 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links', 'remove_linked_tree'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', + 'set_executable', 'remove_dead_links', 'remove_linked_tree'] import os import sys @@ -345,6 +346,12 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): if order == 'post': yield (source_path, dest_path) + +def set_executable(path): + st = os.stat(path) + os.chmod(path, st.st_mode | stat.S_IEXEC) + + def remove_dead_links(root): """ Removes any dead link that is present in root diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index ec17cb97f14..0d0a7db8a91 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -82,7 +82,6 @@ class FetchStrategy(object): class __metaclass__(type): """This metaclass registers all fetch strategies in a list.""" - def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) if cls.enabled: all_strategies.append(cls) @@ -145,6 +144,8 @@ def __init__(self, url=None, digest=None, **kwargs): self.digest = kwargs.get('md5', None) if not self.digest: self.digest = digest + self.expand_archive = kwargs.get('expand', True) + if not self.url: raise ValueError("URLFetchStrategy requires a url for fetching.") @@ -218,6 +219,10 @@ def archive_file(self): @_needs_stage def expand(self): + if not self.expand_archive: + tty.msg("Skipping expand step for %s" % self.archive_file) + return + tty.msg("Staging archive: %s" % self.archive_file) self.stage.chdir() diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index fdc4e7967f5..6981f69ac0e 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -51,13 +51,20 @@ def mirror_archive_filename(spec, fetcher): raise ValueError("mirror.path requires spec with concrete version.") if isinstance(fetcher, fs.URLFetchStrategy): - # If we fetch this version with a URLFetchStrategy, use URL's archive type - ext = url.downloaded_file_extension(fetcher.url) + if fetcher.expand_archive: + # If we fetch this version with a URLFetchStrategy, use URL's archive type + ext = url.downloaded_file_extension(fetcher.url) + else: + # If the archive shouldn't be expanded, don't check for its extension. + ext = None else: # Otherwise we'll make a .tar.gz ourselves ext = 'tar.gz' - return "%s-%s.%s" % (spec.package.name, spec.version, ext) + filename = "%s-%s" % (spec.package.name, spec.version) + if ext: + filename += ".%s" % ext + return filename def mirror_archive_path(spec, fetcher): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index b117c76aa17..b405915a752 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -229,13 +229,22 @@ def archive_file(self): @property def source_path(self): - """Returns the path to the expanded/checked out source code - within this fetch strategy's path. + """Returns the path to the expanded/checked out source code. - This assumes nothing else is going ot be put in the - FetchStrategy's path. It searches for the first - subdirectory of the path it can find, then returns that. + To find the source code, this method searches for the first + subdirectory of the stage that it can find, and returns it. + This assumes nothing besides the archive file will be in the + stage path, but it has the advantage that we don't need to + know the name of the archive or its contents. + + If the fetch strategy is not supposed to expand the downloaded + file, it will just return the stage path. If the archive needs + to be expanded, it will return None when no archive is found. """ + if isinstance(self.fetcher, fs.URLFetchStrategy): + if not self.fetcher.expand_archive: + return self.path + for p in [os.path.join(self.path, f) for f in os.listdir(self.path)]: if os.path.isdir(p): return p @@ -416,21 +425,15 @@ def expand_archive(self): shutil.move(source_path, destination_path) -@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy']) +@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy']) class StageComposite: """ Composite for Stage type objects. The first item in this composite is considered to be the root package, and operations that return a value are forwarded to it. """ - - @property - def source_path(self): - return self[0].source_path - - @property - def path(self): - return self[0].path - + # + # __enter__ and __exit__ delegate to all stages in the composite. + # def __enter__(self): for item in self: item.__enter__() @@ -440,9 +443,24 @@ def __exit__(self, exc_type, exc_val, exc_tb): for item in reversed(self): item.__exit__(exc_type, exc_val, exc_tb) + # + # Below functions act only on the *first* stage in the composite. + # + @property + def source_path(self): + return self[0].source_path + + @property + def path(self): + return self[0].path + def chdir_to_source(self): return self[0].chdir_to_source() + @property + def archive_file(self): + return self[0].archive_file + class DIYStage(object): """Simple class that allows any directory to be a spack stage.""" From 1ee90a6fe792bce1730bfd3fddb4fa12b4597361 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 6 Mar 2016 16:52:34 -0800 Subject: [PATCH 515/910] Make spack_cc, spack_cxx, spack_f77, spack_fc available in build env. - Add for convenience for packages that need to refer to these. - Added an example in the documentation. --- lib/spack/docs/packaging_guide.rst | 9 +++++++++ lib/spack/spack/build_environment.py | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index bae8c34d522..983adb28b01 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -2137,6 +2137,15 @@ Filtering functions Examples: + #. Filtering a Makefile to force it to use Spack's compiler wrappers: + + .. code-block:: python + + filter_file(r'^CC\s*=.*', spack_cc, 'Makefile') + filter_file(r'^CXX\s*=.*', spack_cxx, 'Makefile') + filter_file(r'^F77\s*=.*', spack_f77, 'Makefile') + filter_file(r'^FC\s*=.*', spack_fc, 'Makefile') + #. Replacing ``#!/usr/bin/perl`` with ``#!/usr/bin/env perl`` in ``bib2xhtml``: .. code-block:: python diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 1b877780801..e22597a789b 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -214,6 +214,13 @@ def set_module_variables_for_package(pkg, m): m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE') m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg))) + # Put spack compiler paths in module scope. + link_dir = spack.build_env_path + m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc']) + m.spack_cxx = join_path(link_dir, pkg.compiler.link_paths['cxx']) + m.spack_f77 = join_path(link_dir, pkg.compiler.link_paths['f77']) + m.spack_f90 = join_path(link_dir, pkg.compiler.link_paths['fc']) + # Emulate some shell commands for convenience m.pwd = os.getcwd m.cd = os.chdir From 108277fb5dc648c0e93fbd22e6ea6613329acc9d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 6 Mar 2016 19:32:00 -0800 Subject: [PATCH 516/910] Fix docs that have gone stale since repos were added. --- lib/spack/docs/developer_guide.rst | 15 ++++++++++++++- lib/spack/docs/features.rst | 2 +- lib/spack/docs/packaging_guide.rst | 20 ++++++++++---------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index db47de80f59..0b618aa6830 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -73,19 +73,32 @@ with a high level view of Spack's directory structure:: spack/ <- installation root bin/ spack <- main spack executable + + etc/ + spack/ <- Spack config files. + Can be overridden by files in ~/.spack. + var/ spack/ <- build & stage directories + repos/ <- contains package repositories + builtin/ <- pkg repository that comes with Spack + repo.yaml <- descriptor for the builtin repository + packages/ <- directories under here contain packages + opt/ spack/ <- packages are installed here + lib/ spack/ docs/ <- source for this documentation env/ <- compiler wrappers for build environment + external/ <- external libs included in Spack distro + llnl/ <- some general-use libraries + spack/ <- spack module; contains Python code cmd/ <- each file in here is a spack subcommand compilers/ <- compiler description files - packages/ <- each file in here is a spack package test/ <- unit test modules util/ <- common code diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst index fcb810086da..0998ba8da4d 100644 --- a/lib/spack/docs/features.rst +++ b/lib/spack/docs/features.rst @@ -103,7 +103,7 @@ creates a simple python file: It doesn't take much python coding to get from there to a working package: -.. literalinclude:: ../../../var/spack/packages/libelf/package.py +.. literalinclude:: ../../../var/spack/repos/builtin/packages/libelf/package.py :lines: 25- Spack also provides wrapper functions around common commands like diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 59ba63fa355..a3e1f4de592 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -84,7 +84,7 @@ always choose to download just one tarball initially, and run If it fails entirely, you can get minimal boilerplate by using :ref:`spack-edit-f`, or you can manually create a directory and - ``package.py`` file for the package in ``var/spack/packages``. + ``package.py`` file for the package in ``var/spack/repos/builtin/packages``. .. note:: @@ -203,7 +203,7 @@ edit`` command: So, if you used ``spack create`` to create a package, then saved and closed the resulting file, you can get back to it with ``spack edit``. The ``cmake`` package actually lives in -``$SPACK_ROOT/var/spack/packages/cmake/package.py``, but this provides +``$SPACK_ROOT/var/spack/repos/builtin/packages/cmake/package.py``, but this provides a much simpler shortcut and saves you the trouble of typing the full path. @@ -269,18 +269,18 @@ live in Spack's directory structure. In general, `spack-create`_ and `spack-edit`_ handle creating package files for you, so you can skip most of the details here. -``var/spack/packages`` +``var/spack/repos/builtin/packages`` ~~~~~~~~~~~~~~~~~~~~~~~ A Spack installation directory is structured like a standard UNIX install prefix (``bin``, ``lib``, ``include``, ``var``, ``opt``, etc.). Most of the code for Spack lives in ``$SPACK_ROOT/lib/spack``. -Packages themselves live in ``$SPACK_ROOT/var/spack/packages``. +Packages themselves live in ``$SPACK_ROOT/var/spack/repos/builtin/packages``. If you ``cd`` to that directory, you will see directories for each package: -.. command-output:: cd $SPACK_ROOT/var/spack/packages; ls -CF +.. command-output:: cd $SPACK_ROOT/var/spack/repos/builtin/packages; ls -CF :shell: :ellipsis: 10 @@ -288,7 +288,7 @@ Each directory contains a file called ``package.py``, which is where all the python code for the package goes. For example, the ``libelf`` package lives in:: - $SPACK_ROOT/var/spack/packages/libelf/package.py + $SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py Alongside the ``package.py`` file, a package may contain extra directories or files (like patches) that it needs to build. @@ -301,7 +301,7 @@ Packages are named after the directory containing ``package.py``. So, ``libelf``'s ``package.py`` lives in a directory called ``libelf``. The ``package.py`` file defines a class called ``Libelf``, which extends Spack's ``Package`` class. for example, here is -``$SPACK_ROOT/var/spack/packages/libelf/package.py``: +``$SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py``: .. code-block:: python :linenos: @@ -328,7 +328,7 @@ these: $ spack install libelf@0.8.13 Spack sees the package name in the spec and looks for -``libelf/package.py`` in ``var/spack/packages``. Likewise, if you say +``libelf/package.py`` in ``var/spack/repos/builtin/packages``. Likewise, if you say ``spack install py-numpy``, then Spack looks for ``py-numpy/package.py``. @@ -703,7 +703,7 @@ supply is a filename, then the patch needs to live within the spack source tree. For example, the patch above lives in a directory structure like this:: - $SPACK_ROOT/var/spack/packages/ + $SPACK_ROOT/var/spack/repos/builtin/packages/ mvapich2/ package.py ad_lustre_rwcontig_open_source.patch @@ -1533,7 +1533,7 @@ The last element of a package is its ``install()`` method. This is where the real work of installation happens, and it's the main part of the package you'll need to customize for each piece of software. -.. literalinclude:: ../../../var/spack/packages/libelf/package.py +.. literalinclude:: ../../../var/spack/repos/builtin/packages/libelf/package.py :start-after: 0.8.12 :linenos: From 265ef337a8cdb7397aa01858077787ca4c2669fb Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 3 Mar 2016 10:40:06 -0500 Subject: [PATCH 517/910] Don't quote -rpath literal --- lib/spack/env/cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index c3d11357224..fb0df79d334 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -176,17 +176,17 @@ while [ -n "$1" ]; do -Wl,*) arg="${1#-Wl,}" # TODO: Handle multiple -Wl, continuations of -Wl,-rpath - if [[ $arg == '-rpath='* ]]; then + if [[ $arg == -rpath=* ]]; then arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg == '-rpath,'* ]]; then + elif [[ $arg == -rpath,* ]]; then arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg == '-rpath' ]]; then + elif [[ $arg == -rpath ]]; then shift; arg="$1" if [[ $arg != '-Wl,'* ]]; then die "-Wl,-rpath was not followed by -Wl,*" From a399451e1e32c9a294f5b735237cef93ddbd4131 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 6 Mar 2016 23:34:48 -0500 Subject: [PATCH 518/910] Update OpenSSL to 1.0.2g --- var/spack/repos/builtin/packages/openssl/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index c73102f05d3..70afaf4038a 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -17,6 +17,7 @@ class Openssl(Package): version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') version('1.0.2f', 'b3bf73f507172be9292ea2a8c28b659d') + version('1.0.2g', 'f3c710c045cdee5fd114feb69feba7aa') depends_on("zlib") parallel = False From 547933e7e4997a175bf03363b5c0cf7e687f2000 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 7 Mar 2016 01:08:08 -0800 Subject: [PATCH 519/910] Fix bug in -Xlinker argument handling - also update test to match. --- lib/spack/env/cc | 15 ++++++++------- lib/spack/spack/test/cc.py | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 5e6ed939852..a19346ce971 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -199,19 +199,20 @@ while [ -n "$1" ]; do other_args+=("-Wl,$arg") fi ;; - -Xlinker,*) - arg="${1#-Xlinker,}" - if [ -z "$arg" ]; then shift; arg="$1"; fi + -Xlinker) + shift; arg="$1"; if [[ $arg = -rpath=* ]]; then rpaths+=("${arg#-rpath=}") elif [[ $arg = -rpath ]]; then shift; arg="$1" - if [[ $arg != -Xlinker,* ]]; then - die "-Xlinker,-rpath was not followed by -Xlinker,*" + if [[ $arg != -Xlinker ]]; then + die "-Xlinker -rpath was not followed by -Xlinker " fi - rpaths+=("${arg#-Xlinker,}") + shift; arg="$1" + rpaths+=("$arg") else - other_args+=("-Xlinker,$arg") + other_args+=("-Xlinker") + other_args+=("$arg") fi ;; *) diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 11420ec44a1..f3f6d4a22e6 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -43,7 +43,7 @@ '-llib1', '-llib2', 'arg4', '-Wl,--end-group', - '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath', '-Xlinker,/fourth/rpath', + '-Xlinker', '-rpath', '-Xlinker', '/third/rpath', '-Xlinker', '-rpath', '-Xlinker', '/fourth/rpath', '-llib3', '-llib4', 'arg5', 'arg6'] From 12ddf241a53f85d871c981307e5f682d2e979c75 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 7 Mar 2016 02:39:14 -0800 Subject: [PATCH 520/910] Fix code block in docs. --- lib/spack/spack/multimethod.py | 36 ++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index 51c6a8e89d9..3cd17e796a3 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -138,7 +138,7 @@ class when(object): methods like install() that depend on the package's spec. For example: - .. code-block:: + .. code-block:: python class SomePackage(Package): ... @@ -163,26 +163,28 @@ def install(self, prefix): if you only have part of the install that is platform specific, you could do this: - class SomePackage(Package): - ... - # virtual dependence on MPI. - # could resolve to mpich, mpich2, OpenMPI - depends_on('mpi') + .. code-block:: python - def setup(self): - # do nothing in the default case - pass + class SomePackage(Package): + ... + # virtual dependence on MPI. + # could resolve to mpich, mpich2, OpenMPI + depends_on('mpi') - @when('^openmpi') - def setup(self): - # do something special when this is built with OpenMPI for - # its MPI implementations. + def setup(self): + # do nothing in the default case + pass + + @when('^openmpi') + def setup(self): + # do something special when this is built with OpenMPI for + # its MPI implementations. - def install(self, prefix): - # Do common install stuff - self.setup() - # Do more common install stuff + def install(self, prefix): + # Do common install stuff + self.setup() + # Do more common install stuff There must be one (and only one) @when clause that matches the package's spec. If there is more than one, or if none match, From 1e7d946d9116dee4900b95def1614bd3f788f6f0 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 7 Mar 2016 12:37:51 -0600 Subject: [PATCH 521/910] Add patch to allow PGI to build M4 --- var/spack/repos/builtin/packages/m4/inline-pgi.patch | 10 ++++++++++ var/spack/repos/builtin/packages/m4/package.py | 2 ++ 2 files changed, 12 insertions(+) create mode 100644 var/spack/repos/builtin/packages/m4/inline-pgi.patch diff --git a/var/spack/repos/builtin/packages/m4/inline-pgi.patch b/var/spack/repos/builtin/packages/m4/inline-pgi.patch new file mode 100644 index 00000000000..da5eb57a939 --- /dev/null +++ b/var/spack/repos/builtin/packages/m4/inline-pgi.patch @@ -0,0 +1,10 @@ +--- a/m4/extern-inline.m4 ++++ b/m4/extern-inline.m4 +@@ -34,6 +34,7 @@ + ? defined __GNUC_STDC_INLINE__ && __GNUC_STDC_INLINE__ \ + : (199901L <= __STDC_VERSION__ \ + && !defined __HP_cc \ ++ && !defined __PGI \ + && !(defined __SUNPRO_C && __STDC__))) \ + && !defined _GL_EXTERN_INLINE_STDHEADER_BUG) + # define _GL_INLINE inline diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index d6829dbcd4b..ef70add18aa 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -7,6 +7,8 @@ class M4(Package): version('1.4.17', 'a5e9954b1dae036762f7b13673a2cf76') + patch('inline-pgi.patch', when='@1.4.17') + variant('sigsegv', default=True, description="Build the libsigsegv dependency") depends_on('libsigsegv', when='+sigsegv') From b7750cf61c51704d87d679efeba9ca8cb8c2b768 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 7 Mar 2016 15:52:22 -0600 Subject: [PATCH 522/910] Autoreconf is necessary after patch --- var/spack/repos/builtin/packages/m4/inline-pgi.patch | 12 ++++++++++++ var/spack/repos/builtin/packages/m4/package.py | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/var/spack/repos/builtin/packages/m4/inline-pgi.patch b/var/spack/repos/builtin/packages/m4/inline-pgi.patch index da5eb57a939..bc31ad918ed 100644 --- a/var/spack/repos/builtin/packages/m4/inline-pgi.patch +++ b/var/spack/repos/builtin/packages/m4/inline-pgi.patch @@ -8,3 +8,15 @@ && !(defined __SUNPRO_C && __STDC__))) \ && !defined _GL_EXTERN_INLINE_STDHEADER_BUG) # define _GL_INLINE inline +--- a/configure.ac ++++ b/configure.ac +@@ -22,7 +22,7 @@ + [bug-m4@gnu.org]) + AC_CONFIG_AUX_DIR([build-aux]) + +-AM_INIT_AUTOMAKE([1.11.6 dist-bzip2 dist-xz color-tests parallel-tests ++AM_INIT_AUTOMAKE([dist-bzip2 dist-xz color-tests parallel-tests + silent-rules subdir-objects gnu]) + + m4_pattern_forbid([^M4_[A-Z]]) + diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index ef70add18aa..aa0c775f083 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -14,6 +14,10 @@ class M4(Package): depends_on('libsigsegv', when='+sigsegv') def install(self, spec, prefix): + # After patch, update generated configuration files that depend on extern-inline.m4 + autoreconf = which('autoreconf') + autoreconf() + configure_args = [] if 'libsigsegv' in spec: configure_args.append('--with-libsigsegv-prefix=%s' % spec['libsigsegv'].prefix) From 77ec27c73013eb14821a4249f6a08a8321e10eef Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 8 Mar 2016 11:09:41 +0100 Subject: [PATCH 523/910] fixed bug : similar issues in checksum and md5 as were solved in ad103dcafa652a839590f5fce28b2e2ce3b5a56d --- lib/spack/spack/cmd/checksum.py | 39 +++++++++--------------- lib/spack/spack/cmd/md5.py | 54 ++++++++++++++++----------------- lib/spack/spack/cmd/stage.py | 4 ++- lib/spack/spack/stage.py | 1 + 4 files changed, 45 insertions(+), 53 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 966ff9a5e96..5504673c9e4 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -22,23 +22,18 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os -import re import argparse import hashlib -from pprint import pprint -from subprocess import CalledProcessError import llnl.util.tty as tty -from llnl.util.tty.colify import colify - import spack import spack.cmd import spack.util.crypto from spack.stage import Stage, FailedDownloadError from spack.version import * -description ="Checksum available versions of a package." +description = "Checksum available versions of a package." + def setup_parser(subparser): subparser.add_argument( @@ -60,30 +55,24 @@ def get_checksums(versions, urls, **kwargs): hashes = [] i = 0 for url, version in zip(urls, versions): - stage = Stage(url) try: - stage.fetch() - if i == 0 and first_stage_function: - first_stage_function(stage) + with Stage(url) as stage: + stage.delete_on_exit = not keep_stage + stage.fetch() + if i == 0 and first_stage_function: + first_stage_function(stage) - hashes.append((version, - spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) - except FailedDownloadError, e: + hashes.append((version, + spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) + i += 1 + except FailedDownloadError as e: tty.msg("Failed to fetch %s" % url) - continue - except Exception, e: + except Exception as e: tty.msg('Something failed on %s, skipping.\n (%s)' % (url, e)) - continue - - finally: - if not keep_stage: - stage.destroy() - i += 1 return hashes - def checksum(parser, args): # get the package we're going to generate checksums for pkg = spack.repo.get(args.package) @@ -106,8 +95,8 @@ def checksum(parser, args): tty.msg("Found %s versions of %s" % (len(versions), pkg.name), *spack.cmd.elide_list( - ["%-10s%s" % (v, versions[v]) for v in sorted_versions])) - print + ["%-10s%s" % (v, versions[v]) for v in sorted_versions])) + print() archives_to_fetch = tty.get_number( "How many would you like to checksum?", default=5, abort='q') diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index 879ef9f7b74..20508abf99c 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -22,51 +22,51 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os import argparse import hashlib - -from contextlib import contextmanager +import os import llnl.util.tty as tty -from llnl.util.filesystem import * - import spack.util.crypto from spack.stage import Stage, FailedDownloadError description = "Calculate md5 checksums for files/urls." -@contextmanager -def stager(url): - _cwd = os.getcwd() - _stager = Stage(url) - try: - _stager.fetch() - yield _stager - except FailedDownloadError: - tty.msg("Failed to fetch %s" % url) - finally: - _stager.destroy() - os.chdir(_cwd) # the Stage class changes the current working dir so it has to be restored def setup_parser(subparser): setup_parser.parser = subparser subparser.add_argument('files', nargs=argparse.REMAINDER, help="Files to checksum.") + +def compute_md5_checksum(url): + if not os.path.isfile(url): + with Stage(url) as stage: + stage.fetch() + value = spack.util.crypto.checksum(hashlib.md5, stage.archive_file) + else: + value = spack.util.crypto.checksum(hashlib.md5, url) + return value + + def md5(parser, args): if not args.files: setup_parser.parser.print_help() return 1 - for f in args.files: - if not os.path.isfile(f): - with stager(f) as stage: - checksum = spack.util.crypto.checksum(hashlib.md5, stage.archive_file) - print "%s %s" % (checksum, f) - else: - if not can_access(f): - tty.die("Cannot read file: %s" % f) + results = [] + for url in args.files: + try: + checksum = compute_md5_checksum(url) + results.append((checksum, url)) + except FailedDownloadError as e: + tty.warn("Failed to fetch %s" % url) + tty.warn("%s" % e) + except IOError as e: + tty.warn("Error when reading %s" % url) + tty.warn("%s" % e) - checksum = spack.util.crypto.checksum(hashlib.md5, f) - print "%s %s" % (checksum, f) + # Dump the MD5s at last without interleaving them with downloads + tty.msg("Number of MD5 check-sums computed: %s " % len(results)) + for checksum, url in results: + tty.msg("%s %s" % (checksum, url)) diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index 5786780efb4..b575f6c4566 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -50,4 +50,6 @@ def stage(parser, args): specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.do_stage() + with package.stage as stage: + stage.delete_on_exit = False + package.do_stage() diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index b117c76aa17..9404e12e849 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -438,6 +438,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): for item in reversed(self): + item.delete_on_exit = getattr(self, 'delete_on_exit', True) item.__exit__(exc_type, exc_val, exc_tb) def chdir_to_source(self): From 18ce5ccf8fb31dae71848f541f3e139bd582953c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 02:49:11 -0800 Subject: [PATCH 524/910] Avoid race in pango's `make install`, set parallel=False --- var/spack/repos/builtin/packages/pango/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/pango/package.py b/var/spack/repos/builtin/packages/pango/package.py index df43625bf5c..79dad3a3d2f 100644 --- a/var/spack/repos/builtin/packages/pango/package.py +++ b/var/spack/repos/builtin/packages/pango/package.py @@ -16,4 +16,4 @@ class Pango(Package): def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() - make("install") + make("install", parallel=False) From 5aadb6df19ea6d7575ee5f5e5f18d92934245d98 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 02:50:26 -0800 Subject: [PATCH 525/910] Fixes #502. Create install prefix before build, clean up do_install. - Fix bug introduced during merge of stage refactor. - install prefix was not created before build_environment.fork() - build_environment.fork() calls setup_dependent_environment - python's setup_dependent_environment can inadvertently create the install prefix before directory_layout expects it. - Clean up Package.do_install: - simplify control flow: parent process now entirely responsible for creating/destroying the install prefix. cleanup is now in one place. - Hoisting cleanup out of the child improves nesting of try/catch in `real_work`. - `real_work` renamed to `build_process` --- lib/spack/spack/directory_layout.py | 56 +++++++++----- lib/spack/spack/package.py | 115 +++++++++++++--------------- 2 files changed, 91 insertions(+), 80 deletions(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 08c23627f46..242eb1afa04 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -85,6 +85,16 @@ def create_install_directory(self, spec): raise NotImplementedError() + def check_installed(self, spec): + """Checks whether a spec is installed. + + Return the spec's prefix, if it is installed, None otherwise. + + Raise an exception if the install is inconsistent or corrupt. + """ + raise NotImplementedError() + + def extension_map(self, spec): """Get a dict of currently installed extension packages for a spec. @@ -246,26 +256,36 @@ def build_packages_path(self, spec): def create_install_directory(self, spec): _check_concrete(spec) + prefix = self.check_installed(spec) + if prefix: + raise InstallDirectoryAlreadyExistsError(prefix) + + mkdirp(self.metadata_path(spec)) + self.write_spec(spec, self.spec_file_path(spec)) + + + def check_installed(self, spec): + _check_concrete(spec) path = self.path_for_spec(spec) spec_file_path = self.spec_file_path(spec) - if os.path.isdir(path): - if not os.path.isfile(spec_file_path): - raise InconsistentInstallDirectoryError( - 'No spec file found at path %s' % spec_file_path) + if not os.path.isdir(path): + return None - installed_spec = self.read_spec(spec_file_path) - if installed_spec == self.spec: - raise InstallDirectoryAlreadyExistsError(path) + if not os.path.isfile(spec_file_path): + raise InconsistentInstallDirectoryError( + 'Inconsistent state: install prefix exists but contains no spec.yaml:', + " " + path) - if spec.dag_hash() == installed_spec.dag_hash(): - raise SpecHashCollisionError(installed_hash, spec_hash) - else: - raise InconsistentInstallDirectoryError( - 'Spec file in %s does not match hash!' % spec_file_path) + installed_spec = self.read_spec(spec_file_path) + if installed_spec == spec: + return path - mkdirp(self.metadata_path(spec)) - self.write_spec(spec, spec_file_path) + if spec.dag_hash() == installed_spec.dag_hash(): + raise SpecHashCollisionError(installed_hash, spec_hash) + else: + raise InconsistentInstallDirectoryError( + 'Spec file in %s does not match hash!' % spec_file_path) def all_specs(self): @@ -399,8 +419,8 @@ def remove_extension(self, spec, ext_spec): class DirectoryLayoutError(SpackError): """Superclass for directory layout errors.""" - def __init__(self, message): - super(DirectoryLayoutError, self).__init__(message) + def __init__(self, message, long_msg=None): + super(DirectoryLayoutError, self).__init__(message, long_msg) class SpecHashCollisionError(DirectoryLayoutError): @@ -422,8 +442,8 @@ def __init__(self, installed_spec, prefix, error): class InconsistentInstallDirectoryError(DirectoryLayoutError): """Raised when a package seems to be installed to the wrong place.""" - def __init__(self, message): - super(InconsistentInstallDirectoryError, self).__init__(message) + def __init__(self, message, long_msg=None): + super(InconsistentInstallDirectoryError, self).__init__(message, long_msg) class InstallDirectoryAlreadyExistsError(DirectoryLayoutError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index d4acbf50243..972a0410b9a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -845,7 +845,8 @@ def do_install(self, if not self.spec.concrete: raise ValueError("Can only install concrete packages.") - if os.path.exists(self.prefix): + # Ensure package is not already installed + if spack.install_layout.check_installed(self.spec): tty.msg("%s is already installed in %s" % (self.name, self.prefix)) return @@ -857,18 +858,11 @@ def do_install(self, keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps, fake=fake, skip_patch=skip_patch, verbose=verbose, make_jobs=make_jobs) - def cleanup(): - """Handles removing install prefix on error.""" - if not keep_prefix: - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed. " + - "Manually remove this directory to fix:", - self.prefix, wrap=True) + # Set parallelism before starting build. + self.make_jobs = make_jobs # Then install the package itself. - def real_work(): + def build_process(): """Forked for each build. Has its own process and python module space set up by build_environment.fork().""" start_time = time.time() @@ -878,30 +872,24 @@ def real_work(): else: self.do_stage() - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) + tty.msg("Building %s" % self.name) - try: - tty.msg("Building %s" % self.name) + self.stage.keep = keep_stage + with self.stage: + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) - self.stage.keep = keep_stage - with self.stage: - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) + if fake: + self.do_fake_install() + else: + # Do the real install in the source directory. + self.stage.chdir_to_source() - if fake: - self.do_fake_install() - - else: - # Do the real install in the source directory. - self.stage.chdir_to_source() - - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') + try: # Redirect I/O to a build log (and optionally to the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') log_file = open(log_path, 'w') @@ -909,43 +897,46 @@ def real_work(): dump_environment(env_path) self.install(self.spec, self.prefix) - # Ensure that something was actually installed. - self._sanity_check_install() + except ProcessError as e: + # Annotate ProcessErrors with the location of the build log. + e.build_log = log_path + raise e - # Copy provenance into the install directory on success - log_install_path = spack.install_layout.build_log_path(self.spec) - env_install_path = spack.install_layout.build_env_path(self.spec) - packages_dir = spack.install_layout.build_packages_path(self.spec) + # Ensure that something was actually installed. + self._sanity_check_install() - install(log_path, log_install_path) - install(env_path, env_install_path) - dump_packages(self.spec, packages_dir) + # Copy provenance into the install directory on success + log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) + packages_dir = spack.install_layout.build_packages_path(self.spec) - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + install(log_path, log_install_path) + install(env_path, env_install_path) + dump_packages(self.spec, packages_dir) - tty.msg("Successfully installed %s" % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - except ProcessError as e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) - except: - # other exceptions just clean up and raise. - cleanup() - raise - - # Set parallelism before starting build. - self.make_jobs = make_jobs - - # Do the build. - spack.build_environment.fork(self, real_work) + try: + # Create the install prefix and fork the build process. + spack.install_layout.create_install_directory(self.spec) + spack.build_environment.fork(self, build_process) + except: + # remove the install prefix if anything went wrong during install. + if not keep_prefix: + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed. " + + "Manually remove this directory to fix:", + self.prefix, wrap=True) + raise # note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. From e0e545774aaddb5386367bd900a31b85692abc51 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 8 Mar 2016 12:15:40 -0600 Subject: [PATCH 526/910] Add more versions to Autotools --- var/spack/repos/builtin/packages/autoconf/package.py | 1 + var/spack/repos/builtin/packages/automake/package.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/autoconf/package.py b/var/spack/repos/builtin/packages/autoconf/package.py index 5189faf0548..6412e810a6a 100644 --- a/var/spack/repos/builtin/packages/autoconf/package.py +++ b/var/spack/repos/builtin/packages/autoconf/package.py @@ -6,6 +6,7 @@ class Autoconf(Package): url = "http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz" version('2.69', '82d05e03b93e45f5a39b828dc9c6c29b') + version('2.62', '6c1f3b3734999035d77da5024aab4fbd') def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/repos/builtin/packages/automake/package.py b/var/spack/repos/builtin/packages/automake/package.py index 9115822730e..2172a42030f 100644 --- a/var/spack/repos/builtin/packages/automake/package.py +++ b/var/spack/repos/builtin/packages/automake/package.py @@ -5,7 +5,9 @@ class Automake(Package): homepage = "http://www.gnu.org/software/automake/" url = "http://ftp.gnu.org/gnu/automake/automake-1.14.tar.gz" + version('1.15', '716946a105ca228ab545fc37a70df3a3') version('1.14.1', 'd052a3e884631b9c7892f2efce542d75') + version('1.11.6', '0286dc30295b62985ca51919202ecfcc') depends_on('autoconf') From a4861a30729e32aef493b3418d03874837eecc08 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 10:39:46 -0800 Subject: [PATCH 527/910] Minor changes/bugfixes on md5/checksum PR --- lib/spack/spack/cmd/checksum.py | 5 ++--- lib/spack/spack/cmd/md5.py | 4 ++-- lib/spack/spack/cmd/stage.py | 4 +--- lib/spack/spack/stage.py | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 5504673c9e4..518d2703dcd 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -56,8 +56,7 @@ def get_checksums(versions, urls, **kwargs): i = 0 for url, version in zip(urls, versions): try: - with Stage(url) as stage: - stage.delete_on_exit = not keep_stage + with Stage(url, keep=keep_stage) as stage: stage.fetch() if i == 0 and first_stage_function: first_stage_function(stage) @@ -96,7 +95,7 @@ def checksum(parser, args): tty.msg("Found %s versions of %s" % (len(versions), pkg.name), *spack.cmd.elide_list( ["%-10s%s" % (v, versions[v]) for v in sorted_versions])) - print() + print archives_to_fetch = tty.get_number( "How many would you like to checksum?", default=5, abort='q') diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index 20508abf99c..f99fc0f8c2b 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -67,6 +67,6 @@ def md5(parser, args): tty.warn("%s" % e) # Dump the MD5s at last without interleaving them with downloads - tty.msg("Number of MD5 check-sums computed: %s " % len(results)) + tty.msg("%d MD5 checksums:" % len(results)) for checksum, url in results: - tty.msg("%s %s" % (checksum, url)) + print "%s %s" % (checksum, url) diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index b575f6c4566..5786780efb4 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -50,6 +50,4 @@ def stage(parser, args): specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: package = spack.repo.get(spec) - with package.stage as stage: - stage.delete_on_exit = False - package.do_stage() + package.do_stage() diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 0d35511c347..d2ed03c271e 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -441,7 +441,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): for item in reversed(self): - item.delete_on_exit = getattr(self, 'delete_on_exit', True) + item.keep = getattr(self, 'keep', None) item.__exit__(exc_type, exc_val, exc_tb) # From 0c7d0c0b6c650cd0fd6f4814a255613e3efa1814 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Tue, 8 Mar 2016 12:26:40 -0800 Subject: [PATCH 528/910] Variants and fixes to thrift package --- .../repos/builtin/packages/thrift/package.py | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/var/spack/repos/builtin/packages/thrift/package.py b/var/spack/repos/builtin/packages/thrift/package.py index 0e15052f644..ec3b13e5632 100644 --- a/var/spack/repos/builtin/packages/thrift/package.py +++ b/var/spack/repos/builtin/packages/thrift/package.py @@ -12,33 +12,45 @@ class Thrift(Package): version('0.9.2', '89f63cc4d0100912f4a1f8a9dee63678') - extends("python") + # Currently only support for c-family and python + variant('c', default=True, description="Build support for C-family languages") + variant('python', default=True, description="Build support for python") - depends_on("autoconf") - depends_on("automake") - depends_on("bison") - depends_on("boost") - depends_on("flex") - depends_on("jdk") - depends_on("libtool") - depends_on("openssl") - depends_on("python") + depends_on('jdk') + depends_on('autoconf') + depends_on('automake') + depends_on('libtool') + depends_on('boost@1.53:') + depends_on('bison') + depends_on('flex') + depends_on('openssl') + + # Variant dependencies + extends('python', when='+python') + depends_on('python', when='+python') + + depends_on('zlib', when='+c') + depends_on('libevent', when='+c') - # Compilation fails for most languages, fortunately cpp installs fine - # All other languages (yes, including C) are omitted until someone needs them def install(self, spec, prefix): - env["PY_PREFIX"] = prefix - env["JAVA_PREFIX"] = prefix + env['PY_PREFIX'] = prefix + env['JAVA_HOME'] = spec['jdk'].prefix - configure("--prefix=%s" % prefix, - "--with-boost=%s" % spec['boost'].prefix, - "--with-c=no", - "--with-go=no", - "--with-python=yes", - "--with-lua=no", - "--with-php=no", - "--with-qt4=no", - "--enable-tests=no") + # configure options + options = ['--prefix=%s' % prefix] + + options.append('--with-boost=%s' % spec['boost'].prefix) + options.append('--enable-tests=no') + + options.append('--with-c=%s' % ('yes' if '+c' in spec else 'no')) + options.append('--with-python=%s' % ('yes' if '+python' in spec else 'no')) + options.append('--with-java=%s' % ('yes' if '+java' in spec else 'no')) + options.append('--with-go=%s' % ('yes' if '+go' in spec else 'no')) + options.append('--with-lua=%s' % ('yes' if '+lua' in spec else 'no')) + options.append('--with-php=%s' % ('yes' if '+php' in spec else 'no')) + options.append('--with-qt4=%s' % ('yes' if '+qt4' in spec else 'no')) + + configure(*options) make() make("install") From a1be45d0e7f1921176b40b2aa497309029f1f7ad Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 13:56:44 -0800 Subject: [PATCH 529/910] Fix bug with setting module-scope vars in derived package classes. --- lib/spack/spack/build_environment.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index e22597a789b..87fc310b5a3 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -177,8 +177,6 @@ def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. """ - m = pkg.module - # number of jobs spack will to build with. jobs = multiprocessing.cpu_count() if not pkg.parallel: From df84677d1640139673c26a4c407698486905fbb3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 13:57:13 -0800 Subject: [PATCH 530/910] Make diy generate verbose build output by default. - added -q option to shut it up --- lib/spack/spack/cmd/diy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 9df53312f80..2c3a8761ab6 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -45,6 +45,9 @@ def setup_parser(subparser): subparser.add_argument( '--skip-patch', action='store_true', help="Skip patching for the DIY build.") + subparser.add_argument( + '-q', '--quiet', action='store_true', dest='quiet', + help="Do not display verbose build output while installing.") subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="specs to use for install. Must contain package AND verison.") @@ -92,4 +95,5 @@ def diy(self, args): package.do_install( keep_prefix=args.keep_prefix, ignore_deps=args.ignore_deps, + verbose=not args.quiet, keep_stage=True) # don't remove source dir for DIY. From 9c6184373752746e1219263c1afd9d1955911891 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 13:58:41 -0800 Subject: [PATCH 531/910] Fix bugs in DIYStage: fetch & context handling. - DIYStage needs to be a context handler - DIYStage.fetch needs to take 2 args. --- lib/spack/spack/stage.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index d2ed03c271e..5354135e6a3 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -477,10 +477,14 @@ def chdir(self): else: raise ChdirError("Setup failed: no such directory: " + self.path) + # DIY stages do nothing as context managers. + def __enter__(self): pass + def __exit__(self, exc_type, exc_val, exc_tb): pass + def chdir_to_source(self): self.chdir() - def fetch(self): + def fetch(self, mirror_only): tty.msg("No need to fetch for DIY.") def check(self): From 5b22873b3dce37c0d7bad418bf6e0b73d381d19e Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Wed, 9 Mar 2016 00:18:20 -0500 Subject: [PATCH 532/910] Fixed issues with hypre: 1. --with-lapack-lib was wrong. 2. --with-MPI was wrong; set env vars for MPI wrappers instead. 3. Added version 2.10.1 4. Added shared library variant (True by default). Hypre can build shared or static libraries, but not both in the same build. --- .../repos/builtin/packages/hypre/package.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py index 0f7f14dd89f..242ee100d7a 100644 --- a/var/spack/repos/builtin/packages/hypre/package.py +++ b/var/spack/repos/builtin/packages/hypre/package.py @@ -1,4 +1,5 @@ from spack import * +import os class Hypre(Package): """Hypre is a library of high performance preconditioners that @@ -8,8 +9,11 @@ class Hypre(Package): homepage = "http://computation.llnl.gov/project/linear_solvers/software.php" url = "http://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz" + version('2.10.1', 'dc048c4cabb3cd549af72591474ad674') version('2.10.0b', '768be38793a35bb5d055905b271f5b8e') + variant('shared', default=True, description="Build shared library version (disables static library)") + depends_on("mpi") depends_on("blas") depends_on("lapack") @@ -17,16 +21,26 @@ class Hypre(Package): def install(self, spec, prefix): blas_dir = spec['blas'].prefix lapack_dir = spec['lapack'].prefix + mpi_dir = spec['mpi'].prefix + + os.environ['CC'] = os.path.join(mpi_dir, 'bin', 'mpicc') + os.environ['CXX'] = os.path.join(mpi_dir, 'bin', 'mpicxx') + os.environ['F77'] = os.path.join(mpi_dir, 'bin', 'mpif77') + + + configure_args = [ + "--prefix=%s" % prefix, + "--with-lapack-libs=lapack", + "--with-lapack-lib-dirs=%s/lib" % lapack_dir, + "--with-blas-libs=blas", + "--with-blas-lib-dirs=%s/lib" % blas_dir] + if '+shared' in self.spec: + configure_args.append("--enable-shared") # Hypre's source is staged under ./src so we'll have to manually # cd into it. with working_dir("src"): - configure( - "--prefix=%s" % prefix, - "--with-blas-libs=blas", - "--with-blas-lib-dirs=%s/lib" % blas_dir, - "--with-lapack-libs=\"lapack blas\"", - "--with-lapack-lib-dirs=%s/lib" % lapack_dir, - "--with-MPI") + configure(*configure_args) + make() make("install") From c67b922185ebf5b7686d5b4958e2b31b8f0f1469 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Wed, 9 Mar 2016 00:34:08 -0500 Subject: [PATCH 533/910] Added shared library capability. --- var/spack/repos/builtin/packages/petsc/package.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 87f700629df..e42425bcefa 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -11,8 +11,11 @@ class Petsc(Package): version('3.5.3', 'd4fd2734661e89f18ac6014b5dd1ef2f') version('3.5.2', 'ad170802b3b058b5deb9cd1f968e7e13') version('3.5.1', 'a557e029711ebf425544e117ffa44d8f') + version('3.4.4', '7edbc68aa6d8d6a3295dd5f6c2f6979d') - depends_on("python @2.6:2.9") # requires Python for building + variant('shared', default=True, description="Build shared library version") + + depends_on("python @2.6:2.7") # requires Python for building depends_on("boost") depends_on("blas") @@ -33,7 +36,7 @@ def install(self, spec, prefix): "--with-metis-dir=%s" % spec['metis'].prefix, "--with-hdf5-dir=%s" % spec['hdf5'].prefix, "--with-mpi-dir=%s" % spec['mpi'].prefix, - "--with-shared-libraries=0") + "--with-shared-libraries=%d" % (1 if '+shared' in self.spec else 0)) # PETSc has its own way of doing parallel make. make('MAKE_NP=%s' % make_jobs, parallel=False) From f01d1c4385f05dd7bda7efb3a9e7f2c8008e177a Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 9 Mar 2016 14:36:37 +0100 Subject: [PATCH 534/910] petsc : added variants and logic to build various flavors --- .../repos/builtin/packages/petsc/package.py | 92 ++++++++++++++----- 1 file changed, 69 insertions(+), 23 deletions(-) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 87f700629df..5be187f3487 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -1,40 +1,86 @@ +import os from spack import * + class Petsc(Package): - """PETSc is a suite of data structures and routines for the - scalable (parallel) solution of scientific applications modeled by - partial differential equations.""" + """ + PETSc is a suite of data structures and routines for the scalable (parallel) solution of scientific applications + modeled by partial differential equations. + """ homepage = "http://www.mcs.anl.gov/petsc/index.html" - url = "http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.5.3.tar.gz" + url = "http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.5.3.tar.gz" + version('3.6.3', '91dd3522de5a5ef039ff8f50800db606') version('3.5.3', 'd4fd2734661e89f18ac6014b5dd1ef2f') version('3.5.2', 'ad170802b3b058b5deb9cd1f968e7e13') version('3.5.1', 'a557e029711ebf425544e117ffa44d8f') - depends_on("python @2.6:2.9") # requires Python for building + variant('shared', default=True, description='Enables the build of shared libraries') + variant('mpi', default=True, description='Activates MPI support') + variant('double', default=True, description='Switches between single and double precision') - depends_on("boost") - depends_on("blas") - depends_on("lapack") - depends_on("hypre") - depends_on("parmetis") - depends_on("metis") - depends_on("hdf5+mpi") - depends_on("mpi") + variant('metis', default=True, description='Activates support for metis and parmetis') + variant('hdf5', default=True, description='Activates support for HDF5 (only parallel)') + variant('boost', default=True, description='Activates support for Boost') + variant('hypre', default=True, description='Activates support for Hypre') + + # Build dependencies + depends_on('python @2.6:2.9') # requires Python for building + + # Virtual dependencies + depends_on('blas') + depends_on('lapack') + depends_on('mpi', when='+mpi') + + # Other dependencies + depends_on('boost', when='+boost') + depends_on('metis', when='+metis') + + depends_on('hdf5~cxx~unsupported+mpi', when='+hdf5+mpi') + depends_on('parmetis', when='+metis+mpi') + depends_on('hypre', when='+hypre+mpi') + + def mpi_dependent_options(self): + if '~mpi' in self.spec: + compiler_opts = [ + '--with-cc=%s' % os.environ['CC'], + '--with-cxx=%s' % (os.environ['CXX'] if self.compiler.cxx is not None else '0'), + '--with-fc=%s' % (os.environ['FC'] if self.compiler.fc is not None else '0'), + '--with-mpi=0' + ] + error_message_fmt = '\t{library} support requires "+mpi" to be activated' + errors = [error_message_fmt.format(library=x) for x in ('hdf5', 'hypre') if ('+'+x) in self.spec] + if errors: + errors = ['incompatible variants given'] + errors + raise RuntimeError('\n'.join(errors)) + else: + compiler_opts = [ + '--with-mpi=1', + '--with-mpi-dir=%s' % self.spec['mpi'].prefix, + ] + return compiler_opts def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-blas-lib=%s/libblas.a" % spec['blas'].prefix.lib, - "--with-lapack-lib=%s/liblapack.a" % spec['lapack'].prefix.lib, - "--with-boost-dir=%s" % spec['boost'].prefix, - "--with-hypre-dir=%s" % spec['hypre'].prefix, - "--with-parmetis-dir=%s" % spec['parmetis'].prefix, - "--with-metis-dir=%s" % spec['metis'].prefix, - "--with-hdf5-dir=%s" % spec['hdf5'].prefix, - "--with-mpi-dir=%s" % spec['mpi'].prefix, - "--with-shared-libraries=0") + options = [] + options.extend(self.mpi_dependent_options()) + options.extend([ + '--with-precision=%s' % ('double' if '+double' in spec else 'single'), + '--with-shared-libraries=%s' % ('1' if '+shared' in spec else '0'), + '--with-blas-lapack-dir=%s' % spec['lapack'].prefix + ]) + # Activates library support if needed + for library in ('metis', 'boost', 'hfd5', 'hypre', 'parmetis'): + options.append( + '--with-{library}={value}'.format(library=library, value=('1' if library in spec else '0')) + ) + if library in spec: + options.append( + '--with-{library}-dir={path}'.format(library=library, path=spec[library].prefix) + ) + + configure('--prefix=%s' % prefix, *options) # PETSc has its own way of doing parallel make. make('MAKE_NP=%s' % make_jobs, parallel=False) make("install") From 8e76cda200bb3da159cdc27726c7812f66c5a5ed Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 9 Mar 2016 10:41:31 -0600 Subject: [PATCH 535/910] Patch config.hin, not extern-inline.m4 --- .../builtin/packages/m4/inline-pgi.patch | 22 ------------------- .../repos/builtin/packages/m4/package.py | 6 +---- var/spack/repos/builtin/packages/m4/pgi.patch | 10 +++++++++ 3 files changed, 11 insertions(+), 27 deletions(-) delete mode 100644 var/spack/repos/builtin/packages/m4/inline-pgi.patch create mode 100644 var/spack/repos/builtin/packages/m4/pgi.patch diff --git a/var/spack/repos/builtin/packages/m4/inline-pgi.patch b/var/spack/repos/builtin/packages/m4/inline-pgi.patch deleted file mode 100644 index bc31ad918ed..00000000000 --- a/var/spack/repos/builtin/packages/m4/inline-pgi.patch +++ /dev/null @@ -1,22 +0,0 @@ ---- a/m4/extern-inline.m4 -+++ b/m4/extern-inline.m4 -@@ -34,6 +34,7 @@ - ? defined __GNUC_STDC_INLINE__ && __GNUC_STDC_INLINE__ \ - : (199901L <= __STDC_VERSION__ \ - && !defined __HP_cc \ -+ && !defined __PGI \ - && !(defined __SUNPRO_C && __STDC__))) \ - && !defined _GL_EXTERN_INLINE_STDHEADER_BUG) - # define _GL_INLINE inline ---- a/configure.ac -+++ b/configure.ac -@@ -22,7 +22,7 @@ - [bug-m4@gnu.org]) - AC_CONFIG_AUX_DIR([build-aux]) - --AM_INIT_AUTOMAKE([1.11.6 dist-bzip2 dist-xz color-tests parallel-tests -+AM_INIT_AUTOMAKE([dist-bzip2 dist-xz color-tests parallel-tests - silent-rules subdir-objects gnu]) - - m4_pattern_forbid([^M4_[A-Z]]) - diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index aa0c775f083..a4b9dcb6231 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -7,17 +7,13 @@ class M4(Package): version('1.4.17', 'a5e9954b1dae036762f7b13673a2cf76') - patch('inline-pgi.patch', when='@1.4.17') + patch('pgi.patch', when='@1.4.17') variant('sigsegv', default=True, description="Build the libsigsegv dependency") depends_on('libsigsegv', when='+sigsegv') def install(self, spec, prefix): - # After patch, update generated configuration files that depend on extern-inline.m4 - autoreconf = which('autoreconf') - autoreconf() - configure_args = [] if 'libsigsegv' in spec: configure_args.append('--with-libsigsegv-prefix=%s' % spec['libsigsegv'].prefix) diff --git a/var/spack/repos/builtin/packages/m4/pgi.patch b/var/spack/repos/builtin/packages/m4/pgi.patch new file mode 100644 index 00000000000..1ad63e2cf1c --- /dev/null +++ b/var/spack/repos/builtin/packages/m4/pgi.patch @@ -0,0 +1,10 @@ +--- a/lib/config.hin ++++ b/lib/config.hin +@@ -1510,6 +1510,7 @@ + ? defined __GNUC_STDC_INLINE__ && __GNUC_STDC_INLINE__ \ + : (199901L <= __STDC_VERSION__ \ + && !defined __HP_cc \ ++ && !defined __PGI \ + && !(defined __SUNPRO_C && __STDC__))) \ + && !defined _GL_EXTERN_INLINE_APPLE_BUG) + # define _GL_INLINE inline From bf4d51ea0b00e091728ecf8d4d51aa295448a416 Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Wed, 9 Mar 2016 14:45:29 +0100 Subject: [PATCH 536/910] fix error exit when doing spack patch without parameters, same as spack stage --- lib/spack/spack/cmd/patch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index 44fc8696db3..b04b4027384 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -24,6 +24,7 @@ ############################################################################## import argparse +import llnl.util.tty as tty import spack.cmd import spack From b43c277dc617020d7aaa1e292e01e79d0840999e Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Mar 2016 10:55:50 -0800 Subject: [PATCH 537/910] Merge @citibeth and @alalazo's petsc fixes from #515 and #517 --- .../repos/builtin/packages/petsc/package.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 5be187f3487..41fd8599452 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -15,6 +15,7 @@ class Petsc(Package): version('3.5.3', 'd4fd2734661e89f18ac6014b5dd1ef2f') version('3.5.2', 'ad170802b3b058b5deb9cd1f968e7e13') version('3.5.1', 'a557e029711ebf425544e117ffa44d8f') + version('3.4.4', '7edbc68aa6d8d6a3295dd5f6c2f6979d') variant('shared', default=True, description='Enables the build of shared libraries') variant('mpi', default=True, description='Activates MPI support') @@ -25,21 +26,21 @@ class Petsc(Package): variant('boost', default=True, description='Activates support for Boost') variant('hypre', default=True, description='Activates support for Hypre') - # Build dependencies - depends_on('python @2.6:2.9') # requires Python for building - # Virtual dependencies depends_on('blas') depends_on('lapack') depends_on('mpi', when='+mpi') + # Build dependencies + depends_on('python @2.6:2.7') + # Other dependencies depends_on('boost', when='+boost') depends_on('metis', when='+metis') - depends_on('hdf5~cxx~unsupported+mpi', when='+hdf5+mpi') + depends_on('hdf5+mpi', when='+hdf5+mpi') depends_on('parmetis', when='+metis+mpi') - depends_on('hypre', when='+hypre+mpi') + depends_on('hypre', when='+hypre+mpi') def mpi_dependent_options(self): if '~mpi' in self.spec: @@ -50,7 +51,9 @@ def mpi_dependent_options(self): '--with-mpi=0' ] error_message_fmt = '\t{library} support requires "+mpi" to be activated' - errors = [error_message_fmt.format(library=x) for x in ('hdf5', 'hypre') if ('+'+x) in self.spec] + errors = [error_message_fmt.format(library=x) + for x in ('hdf5', 'hypre', 'parmetis') + if ('+'+x) in self.spec] if errors: errors = ['incompatible variants given'] + errors raise RuntimeError('\n'.join(errors)) @@ -70,7 +73,7 @@ def install(self, spec, prefix): '--with-blas-lapack-dir=%s' % spec['lapack'].prefix ]) # Activates library support if needed - for library in ('metis', 'boost', 'hfd5', 'hypre', 'parmetis'): + for library in ('metis', 'boost', 'hdf5', 'hypre', 'parmetis'): options.append( '--with-{library}={value}'.format(library=library, value=('1' if library in spec else '0')) ) @@ -79,8 +82,8 @@ def install(self, spec, prefix): '--with-{library}-dir={path}'.format(library=library, path=spec[library].prefix) ) - configure('--prefix=%s' % prefix, *options) + # PETSc has its own way of doing parallel make. make('MAKE_NP=%s' % make_jobs, parallel=False) make("install") From d06ebf23d4ffe3499edca3f34f60be9b561f5f8c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Mar 2016 11:16:35 -0800 Subject: [PATCH 538/910] Removing `unsupported` variant from HDF5. - `unsupported` shouldn't be a variant. --- var/spack/repos/builtin/packages/hdf5/package.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index ed4e7c35c99..513a38ee8a4 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -46,7 +46,6 @@ class Hdf5(Package): variant('cxx', default=True, description='Enable C++ support') variant('fortran', default=True, description='Enable Fortran support') - variant('unsupported', default=True, description='Enables unsupported configuration options') variant('mpi', default=False, description='Enable MPI support') variant('szip', default=False, description='Enable szip support') @@ -74,6 +73,13 @@ def install(self, spec, prefix): self.validate(spec) # Handle compilation after spec validation extra_args = [] + + # Always enable this option. This does not actually enable any + # features: it only *allows* the user to specify certain + # combinations of other arguments. Enabling it just skips a + # sanity check in configure, so this doesn't merit a variant. + extra_args.append("--enable-unsupported") + if '+debug' in spec: extra_args.append('--enable-debug=all') else: @@ -84,9 +90,6 @@ def install(self, spec, prefix): else: extra_args.append('--enable-static-exec') - if '+unsupported' in spec: - extra_args.append("--enable-unsupported") - if '+cxx' in spec: extra_args.append('--enable-cxx') From a4d40177fdfece3c25d7ebb48855e8fee2a7093d Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Tue, 8 Mar 2016 09:44:21 -0700 Subject: [PATCH 539/910] + Provide download/build instructions for qt/5.4.2. - This version provides updates to provided cmake scripts that are required for building cmake-gui. --- var/spack/repos/builtin/packages/qt/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 91afa420c17..ef5f05601f8 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -8,6 +8,9 @@ class Qt(Package): list_url = 'http://download.qt-project.org/official_releases/qt/' list_depth = 2 + version('5.4.2', 'fa1c4d819b401b267eb246a543a63ea5', + url='http://download.qt-project.org/official_releases/qt/5.4/5.4.2/single/qt-everywhere-opensource-src-5.4.2.tar.gz') + version('5.4.0', 'e8654e4b37dd98039ba20da7a53877e6', url='http://download.qt-project.org/official_releases/qt/5.4/5.4.0/single/qt-everywhere-opensource-src-5.4.0.tar.gz') From 383e73a5f53e0ab3d1f448789bc5a1b4a7c85292 Mon Sep 17 00:00:00 2001 From: Alfredo Adolfo Gimenez Date: Wed, 9 Mar 2016 11:25:51 -0800 Subject: [PATCH 540/910] Remove unneccessary depends_on --- var/spack/repos/builtin/packages/thrift/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/thrift/package.py b/var/spack/repos/builtin/packages/thrift/package.py index ec3b13e5632..6430f40e80d 100644 --- a/var/spack/repos/builtin/packages/thrift/package.py +++ b/var/spack/repos/builtin/packages/thrift/package.py @@ -27,7 +27,6 @@ class Thrift(Package): # Variant dependencies extends('python', when='+python') - depends_on('python', when='+python') depends_on('zlib', when='+c') depends_on('libevent', when='+c') From 45ef496dd553e023903b8b4d5bcace59c56eb486 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Mar 2016 11:26:37 -0800 Subject: [PATCH 541/910] Add some descriptive language to a list comprehension. --- var/spack/repos/builtin/packages/petsc/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 41fd8599452..efe172fc083 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -51,6 +51,9 @@ def mpi_dependent_options(self): '--with-mpi=0' ] error_message_fmt = '\t{library} support requires "+mpi" to be activated' + + # If mpi is disabled (~mpi), it's an error to have any of these enabled. + # This generates a list of any such errors. errors = [error_message_fmt.format(library=x) for x in ('hdf5', 'hypre', 'parmetis') if ('+'+x) in self.spec] From 6e82ab1f1501c8b64383f1d9a26d743284f4054f Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 9 Mar 2016 21:18:44 +0100 Subject: [PATCH 542/910] change of url for mpfr --- var/spack/repos/builtin/packages/mpfr/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/mpfr/package.py b/var/spack/repos/builtin/packages/mpfr/package.py index a1bd7529cfe..7e6e7d5bb6f 100644 --- a/var/spack/repos/builtin/packages/mpfr/package.py +++ b/var/spack/repos/builtin/packages/mpfr/package.py @@ -28,8 +28,9 @@ class Mpfr(Package): """The MPFR library is a C library for multiple-precision floating-point computations with correct rounding.""" homepage = "http://www.mpfr.org" - url = "http://www.mpfr.org/mpfr-current/mpfr-3.1.3.tar.bz2" + url = "https://gforge.inria.fr/frs/download.php/latestfile/159/mpfr-3.1.2.tar.bz2" + version('3.1.4', 'b8a2f6b0e68bef46e53da2ac439e1cf4') version('3.1.3', '5fdfa3cfa5c86514ee4a241a1affa138') version('3.1.2', 'ee2c3ac63bf0c2359bf08fc3ee094c19') From ca102295657adef7100f638be0dabc0e6d7cf37f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Mar 2016 14:56:21 -0800 Subject: [PATCH 543/910] Fixes #524 - Had attempted to add more functionality by assigning different meanign None, True, and False values "keep_stage" (where False was "always delete"). - Turns out that's not really worth the complexity. Having the third "always delete" sense is hardly ever useful but makes the code hard to understand. --- lib/spack/spack/package.py | 7 ++++--- lib/spack/spack/stage.py | 24 ++++++++++-------------- lib/spack/spack/test/stage.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 972a0410b9a..ca9e9c4bd1d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -825,7 +825,7 @@ def _resource_stage(self, resource): def do_install(self, - keep_prefix=False, keep_stage=None, ignore_deps=False, + keep_prefix=False, keep_stage=False, ignore_deps=False, skip_patch=False, verbose=False, make_jobs=None, fake=False): """Called by commands to install a package and its dependencies. @@ -834,8 +834,9 @@ def do_install(self, Args: keep_prefix -- Keep install prefix on failure. By default, destroys it. - keep_stage -- Set to True or false to always keep or always delete stage. - By default, stage is destroyed only if there are no exceptions. + keep_stage -- By default, stage is destroyed only if there are no + exceptions during build. Set to True to keep the stage + even with exceptions. ignore_deps -- Do not install dependencies before installing this package. fake -- Don't really build -- install fake stub files instead. skip_patch -- Skip patch stage of build if True. diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 5354135e6a3..f88f82fc2d6 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -88,7 +88,8 @@ class Stage(object): similar, and are intended to persist for only one run of spack. """ - def __init__(self, url_or_fetch_strategy, name=None, mirror_path=None, keep=None): + def __init__(self, url_or_fetch_strategy, + name=None, mirror_path=None, keep=False): """Create a stage object. Parameters: url_or_fetch_strategy @@ -108,10 +109,9 @@ def __init__(self, url_or_fetch_strategy, name=None, mirror_path=None, keep=None keep By default, when used as a context manager, the Stage - is cleaned up when everything goes well, and it is - kept intact when an exception is raised. You can - override this behavior by setting keep to True - (always keep) or False (always delete). + is deleted on exit when no exceptions are raised. + Pass True to keep the stage intact even if no + exceptions are raised. """ # TODO: fetch/stage coupling needs to be reworked -- the logic # TODO: here is convoluted and not modular enough. @@ -166,12 +166,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): Returns: Boolean """ - if self.keep is None: - # Default: delete when there are no exceptions. - if exc_type is None: self.destroy() - - elif not self.keep: - # Overridden. Either always keep or always delete. + # Delete when there are no exceptions, unless asked to keep. + if exc_type is None and not self.keep: self.destroy() @@ -195,8 +191,8 @@ def _need_to_create_path(self): real_tmp = os.path.realpath(self.tmp_root) if spack.use_tmp_stage: - # If we're using a tmp dir, it's a link, and it points at the right spot, - # then keep it. + # If we're using a tmp dir, it's a link, and it points at the + # right spot, then keep it. if (real_path.startswith(real_tmp) and os.path.exists(real_path)): return False else: @@ -441,7 +437,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): for item in reversed(self): - item.keep = getattr(self, 'keep', None) + item.keep = getattr(self, 'keep', False) item.__exit__(exc_type, exc_val, exc_tb) # diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index dbcf89d8645..ea425127c41 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -277,3 +277,38 @@ def test_restage(self): self.check_chdir_to_source(stage, stage_name) self.assertFalse('foobar' in os.listdir(stage.source_path)) self.check_destroy(stage, stage_name) + + + def test_no_keep_without_exceptions(self): + with Stage(archive_url, name=stage_name, keep=False) as stage: + pass + self.check_destroy(stage, stage_name) + + + def test_keep_without_exceptions(self): + with Stage(archive_url, name=stage_name, keep=True) as stage: + pass + path = self.get_stage_path(stage, stage_name) + self.assertTrue(os.path.isdir(path)) + + + def test_no_keep_with_exceptions(self): + try: + with Stage(archive_url, name=stage_name, keep=False) as stage: + raise Exception() + + path = self.get_stage_path(stage, stage_name) + self.assertTrue(os.path.isdir(path)) + except: + pass # ignore here. + + + def test_keep_exceptions(self): + try: + with Stage(archive_url, name=stage_name, keep=True) as stage: + raise Exception() + + path = self.get_stage_path(stage, stage_name) + self.assertTrue(os.path.isdir(path)) + except: + pass # ignore here. From a384ad5b1270140d71110e46d39144a0f0e9081e Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Wed, 9 Mar 2016 16:11:33 -0800 Subject: [PATCH 544/910] Fix problem with pure integer arguments in preferred versions list (e.g, 2 instead of 2.7.3) --- lib/spack/spack/config.py | 5 +++-- lib/spack/spack/preferred_packages.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 807a898644d..95a988f7ffe 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -214,7 +214,8 @@ 'version': { 'type' : 'array', 'default' : [], - 'items' : { 'type' : 'string' } }, #version strings + 'items' : { 'anyOf' : [ { 'type' : 'string' }, + { 'type' : 'number'}]}}, #version strings 'compiler': { 'type' : 'array', 'default' : [], @@ -573,7 +574,7 @@ def __init__(self, validation_error, data): # Try to get line number from erroneous instance and its parent instance_mark = getattr(validation_error.instance, '_start_mark', None) parent_mark = getattr(validation_error.parent, '_start_mark', None) - path = getattr(validation_error, 'path', None) + path = [str(s) for s in getattr(validation_error, 'path', None)] # Try really hard to get the parent (which sometimes is not # set) This digs it out of the validated structure if it's not diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index 2b0ba791b6b..eaea016a85a 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -45,7 +45,7 @@ def _order_for_package(self, pkgname, component, second_key, test_all=True): order = order.get(second_key, {}) if not order: continue - return [s.strip() for s in order] + return [str(s).strip() for s in order] return [] @@ -98,11 +98,11 @@ def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, secon b_index = None reverse = -1 if reverse_natural_compare else 1 for i, cspec in enumerate(specs): - if a_index == None and cspec.satisfies(a): + if a_index == None and (cspec.satisfies(a) or a.satisfies(cspec)): a_index = i if b_index: break - if b_index == None and cspec.satisfies(b): + if b_index == None and (cspec.satisfies(b) or b.satisfies(cspec)): b_index = i if a_index: break From 1f06dd40f7e65252568da23e9758bf5af02833eb Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Wed, 9 Mar 2016 16:11:53 -0800 Subject: [PATCH 545/910] Update documentation for new packages.yaml config format. --- lib/spack/docs/packaging_guide.rst | 41 +++++++++---------- lib/spack/docs/site_configuration.rst | 59 +++++++++++++-------------- 2 files changed, 48 insertions(+), 52 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index f368d0a4fa8..ef9fd89b62f 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1561,50 +1561,49 @@ be concretized on their system. For example, one user may prefer packages built with OpenMPI and the Intel compiler. Another user may prefer packages be built with MVAPICH and GCC. -Spack's ``preferred`` configuration can be used to set defaults for sites or users. -Spack uses this configuration to make decisions about which compilers, package -versions, depends_on, and variants it should prefer during concretization. - -The preferred configuration can be controlled by editing the -``~/.spack/preferred.yaml`` file for user configuations, or the +Spack can be configurated to prefer certain compilers, package +versions, depends_on, and variants during concretization. +The preferred configuration can be controlled via the +``~/.spack/packages.yaml`` file for user configuations, or the +``etc/spack/packages.yaml`` site configuration. -Here's an example preferred.yaml file: +Here's an example packages.yaml file that sets preferred packages: .. code-block:: sh - preferred: + packages: dyninst: - compiler: gcc@4.9 + compiler: [gcc@4.9] variants: +debug gperftools: - version: 2.2, 2.4, 2.3 + version: [2.2, 2.4, 2.3] all: - compiler: gcc@4.4.7, gcc@4.6:, intel, clang, pgi - providers: - mpi: mvapich, mpich, openmpi + compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi] + providers: + mpi: [mvapich, mpich, openmpi] + At a high level, this example is specifying how packages should be concretized. The dyninst package should prefer using gcc 4.9 and be built with debug options. The gperftools package should prefer version 2.2 over 2.4. Every package on the system should prefer mvapich for -its MPI and gcc 4.4.7 (except for Dyninst, which perfers gcc 4.9). +its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by perfering gcc 4.9). These options are used to fill in implicit defaults. Any of them can be overwritten on the command line if explicitly requested. -Each preferred.yaml file begin with the string ``preferred:`` and -each subsequent entry is indented underneath it. The next layer contains -package names or the special string ``all`` (which applies to -every package). Underneath each package name is +Each packages.yaml file begin with the string ``packages:`` and +package names are specified on the next level. The special string ``all`` +applies settings to each package. Underneath each package name is one or more components: ``compiler``, ``variants``, ``version``, or ``providers``. Each component has an ordered list of spec ``constraints``, with earlier entries in the list being prefered over -latter entries. +later entries. Sometimes a package installation may have constraints that forbid the first concretization rule, in which case Spack will use the first legal concretization rule. Going back to the example, if a user -requests gperftools 2.3 or latter, then Spack will install version 2.4 +requests gperftools 2.3 or later, then Spack will install version 2.4 as the 2.4 version of gperftools is preferred over 2.3. An explicit concretization rule in the preferred section will always @@ -1612,7 +1611,7 @@ take preference over unlisted concretizations. In the above example, xlc isn't listed in the compiler list. Every listed compiler from gcc to pgi will thus be preferred over the xlc compiler. -The syntax for the ``providers`` section differs slightly from other +The syntax for the ``provider`` section differs slightly from other concretization rules. A provider lists a value that packages may ``depend_on`` (e.g, mpi) and a list of rules for fulfilling that dependency. diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index a7211a9d95b..ebf04371067 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -56,44 +56,43 @@ directory is. External Packages ~~~~~~~~~~~~~~~~~~~~~ -It's possible for Spack to use certain externally-installed -packages rather than always rebuilding packages. This may be desirable +Spack can be configured to use externally-installed +packages rather than building its own packages. This may be desirable if machines ship with system packages, such as a customized MPI that should be used instead of Spack building its own MPI. External packages are configured through the ``packages.yaml`` file found in a Spack installation's ``etc/spack/`` or a user's ``~/.spack/`` -directory. Here's an example of an external configuration:: +directory. Here's an example of an external configuration: .. code-block:: yaml - packages: - - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: - path: /opt/openmpi-1.4.3 - - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: - path: /opt/openmpi-1.4.3-debug - - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: - path: /opt/openmpi-1.6.5-intel + packages: + openmpi: + paths: + openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel This example lists three installations of OpenMPI, one built with gcc, one built with gcc and debug information, and another built with OpenMPI. If Spack is asked to build a package that uses one of these MPIs as a -dependency, it link the package to the pre-installed OpenMPI in -the given directory. +dependency, it will use the the pre-installed OpenMPI in +the given directory. This example also specifies that Spack should never +build its own OpenMPI via the ``nobuild: True`` option. -Each ``packages.yaml`` should begin with a ``packages:`` token, followed -by a list of package specs. Specs in the ``packages.yaml`` have at most -one ``path`` tag, which specifies the top-level directory where the -spec is installed. - -Each spec should be as well-defined as reasonably possible. If a +Each ``packages.yaml`` begins with a ``packages:`` token, followed +by a list of package names. To specify externals, add a ``paths`` +token under the package name, which lists externals in a +``spec : /path`` format. Each spec should be as +well-defined as reasonably possible. If a package lacks a spec component, such as missing a compiler or package version, then Spack will guess the missing component based on its most-favored packages, and it may guess incorrectly. -All package versions and compilers listed in ``packages.yaml`` should +Each package version and compilers listed in an external should have entries in Spack's packages and compiler configuration, even -the package and compiler may not actually be used. +though the package and compiler may not every be built. The packages configuration can tell Spack to use an external location for certain package versions, but it does not restrict Spack to using @@ -103,27 +102,25 @@ rather than continue using the pre-installed OpenMPI versions. To prevent this, the ``packages.yaml`` configuration also allows packages to be flagged as non-buildable. The previous example could be modified to -be:: +be: .. code-block:: yaml packages: - - openmpi: - nobuild: True - - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: - path: /opt/openmpi-1.4.3 - - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: - path: /opt/openmpi-1.4.3-debug - - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: - path: /opt/openmpi-1.6.5-intel + openmpi: + paths: + openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel + nobuild: True The addition of the ``nobuild`` flag tells Spack that it should never build its own version of OpenMPI, and it will instead always rely on a pre-built OpenMPI. Similar to ``path``, ``nobuild`` is specified as a property under -a spec and will prevent building of anything that satisfies that spec. +a package name. The ``nobuild`` does not need to be paired with external packages. -It could also be used alone to forbid versions of packages that may be +It could also be used alone to forbid packages that may be buggy or otherwise undesirable. From 23cbc2b1d9dc5816c63f330640105c5435ccdc22 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Tue, 8 Mar 2016 09:44:21 -0700 Subject: [PATCH 546/910] + Provide download/build instructions for qt/5.4.2. - This version provides updates to provided cmake scripts that are required for building cmake-gui. + Provide download/build instructions for version 3.5.0. - When building the +qt variant, add a validate function to ensure that qt-5.4.0 is not used (this version of qt has errors related to cmake). --- .../repos/builtin/packages/cmake/package.py | 21 +++++++++++++++++-- .../repos/builtin/packages/qt/package.py | 3 +++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index f39a6812843..806c37a68cc 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -30,6 +30,7 @@ class Cmake(Package): homepage = 'https://www.cmake.org' url = 'https://cmake.org/files/v3.4/cmake-3.4.3.tar.gz' + version('3.5.0', '33c5d09d4c33d4ffcc63578a6ba8777e') version('3.4.3', '4cb3ff35b2472aae70f542116d616e63') version('3.4.0', 'cd3034e0a44256a0917e254167217fc8') version('3.3.1', '52638576f4e1e621fed6c3410d3a1b12') @@ -49,8 +50,25 @@ def url_for_version(self, version): """Handle CMake's version-based custom URLs.""" return 'https://cmake.org/files/v%s/cmake-%s.tar.gz' % (version.up_to(2), version) - def install(self, spec, prefix): + def validate(self, spec): + """ + Checks if incompatible versions of qt were specified + :param spec: spec of the package + :raises RuntimeError: in case of inconsistencies + """ + + print spec + + if '+qt' in spec and spec.satisfies('^qt@5.4.0'): + msg = 'qt-5.4.0 has broken CMake modules.' + raise RuntimeError(msg) + + def install(self, spec, prefix): + # Consistency check + self.validate(spec) + + # configure, build, install: options = ['--prefix=%s' % prefix] options.append('--parallel=%s' % str(make_jobs)) @@ -65,6 +83,5 @@ def install(self, spec, prefix): options.append('-DCMAKE_USE_OPENSSL=ON') configure(*options) - make() make('install') diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 91afa420c17..ef5f05601f8 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -8,6 +8,9 @@ class Qt(Package): list_url = 'http://download.qt-project.org/official_releases/qt/' list_depth = 2 + version('5.4.2', 'fa1c4d819b401b267eb246a543a63ea5', + url='http://download.qt-project.org/official_releases/qt/5.4/5.4.2/single/qt-everywhere-opensource-src-5.4.2.tar.gz') + version('5.4.0', 'e8654e4b37dd98039ba20da7a53877e6', url='http://download.qt-project.org/official_releases/qt/5.4/5.4.0/single/qt-everywhere-opensource-src-5.4.0.tar.gz') From f56939c16c36ee946f40571bdc330b432af93cce Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Mar 2016 17:01:01 -0800 Subject: [PATCH 547/910] Add unit test for sbang patching. --- lib/spack/spack/test/__init__.py | 13 ++--- lib/spack/spack/test/sbang.py | 93 ++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 lib/spack/spack/test/sbang.py diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 4b9a361d4bf..d5d8b647650 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -65,7 +65,8 @@ 'lock', 'database', 'namespace_trie', - 'yaml'] + 'yaml', + 'sbang'] def list_tests(): @@ -87,20 +88,20 @@ def run(names, outputDir, verbose=False): "Valid names are:") colify(sorted(test_names), indent=4) sys.exit(1) - + tally = Tally() for test in names: module = 'spack.test.' + test print module - + tty.msg("Running test: %s" % test) - + runOpts = ["--with-%s" % spack.test.tally_plugin.Tally.name] - + if outputDir: xmlOutputFname = "unittests-{0}.xml".format(test) xmlOutputPath = join_path(outputDir, xmlOutputFname) - runOpts += ["--with-xunit", + runOpts += ["--with-xunit", "--xunit-file={0}".format(xmlOutputPath)] argv = [""] + runOpts + [module] result = nose.run(argv=argv, addplugins=[tally]) diff --git a/lib/spack/spack/test/sbang.py b/lib/spack/spack/test/sbang.py new file mode 100644 index 00000000000..825bc4be982 --- /dev/null +++ b/lib/spack/spack/test/sbang.py @@ -0,0 +1,93 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""\ +Test that Spack's shebang filtering works correctly. +""" +import os +import unittest +import tempfile +import shutil + +from llnl.util.filesystem import * +from spack.hooks.sbang import filter_shebangs_in_directory +import spack + +short_line = "#!/this/is/short/bin/bash\n" +long_line = "#!/this/" + ('x' * 200) + "/is/long\n" +sbang_line = '#!/bin/bash %s/bin/sbang\n' % spack.spack_root +last_line = "last!\n" + +class SbangTest(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.mkdtemp() + + # make sure we can ignore non-files + directory = os.path.join(self.tempdir, 'dir') + mkdirp(directory) + + # Script with short shebang + self.short_shebang = os.path.join(self.tempdir, 'short') + with open(self.short_shebang, 'w') as f: + f.write(short_line) + f.write(last_line) + + # Script with long shebang + self.long_shebang = os.path.join(self.tempdir, 'long') + with open(self.long_shebang, 'w') as f: + f.write(long_line) + f.write(last_line) + + # Script already using sbang. + self.has_shebang = os.path.join(self.tempdir, 'shebang') + with open(self.has_shebang, 'w') as f: + f.write(sbang_line) + f.write(long_line) + f.write(last_line) + + + def tearDown(self): + shutil.rmtree(self.tempdir, ignore_errors=True) + + + + def test_shebang_handling(self): + filter_shebangs_in_directory(self.tempdir) + + # Make sure this is untouched + with open(self.short_shebang, 'r') as f: + self.assertEqual(f.readline(), short_line) + self.assertEqual(f.readline(), last_line) + + # Make sure this got patched. + with open(self.long_shebang, 'r') as f: + self.assertEqual(f.readline(), sbang_line) + self.assertEqual(f.readline(), long_line) + self.assertEqual(f.readline(), last_line) + + # Make sure this is untouched + with open(self.has_shebang, 'r') as f: + self.assertEqual(f.readline(), sbang_line) + self.assertEqual(f.readline(), long_line) + self.assertEqual(f.readline(), last_line) From 52cdcdde76b272ba4905a23642323d116b00a7e7 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Mar 2016 17:13:04 -0800 Subject: [PATCH 548/910] Fix #525: sbang handles symlinks & directories properly. --- lib/spack/spack/hooks/sbang.py | 43 +++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 3390ecea295..d78adb576e2 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -35,7 +35,7 @@ shebang_limit = 127 def shebang_too_long(path): - """Detects whether an file has a shebang line that is too long.""" + """Detects whether a file has a shebang line that is too long.""" with open(path, 'r') as script: bytes = script.read(2) if bytes != '#!': @@ -47,14 +47,21 @@ def shebang_too_long(path): def filter_shebang(path): """Adds a second shebang line, using sbang, at the beginning of a file.""" + with open(path, 'r') as original_file: + original = original_file.read() + + # This line will be prepended to file + new_sbang_line = '#!/bin/bash %s/bin/sbang\n' % spack.spack_root + + # Skip files that are already using sbang. + if original.startswith(new_sbang_line): + return + backup = path + ".shebang.bak" os.rename(path, backup) - with open(backup, 'r') as bak_file: - original = bak_file.read() - with open(path, 'w') as new_file: - new_file.write('#!/bin/bash %s/bin/sbang\n' % spack.spack_root) + new_file.write(new_sbang_line) new_file.write(original) copy_mode(backup, path) @@ -63,15 +70,29 @@ def filter_shebang(path): tty.warn("Patched overly long shebang in %s" % path) +def filter_shebangs_in_directory(directory): + for file in os.listdir(directory): + path = os.path.join(directory, file) + + # only handle files + if not os.path.isfile(path): + continue + + # only handle links that resolve within THIS package's prefix. + if os.path.islink(path): + real_path = os.path.realpath(path) + if not real_path.startswith(directory + os.sep): + continue + + # test the file for a long shebang, and filter + if shebang_too_long(path): + filter_shebang(path) + + def post_install(pkg): """This hook edits scripts so that they call /bin/bash $spack_prefix/bin/sbang instead of something longer than the shebang limit.""" if not os.path.isdir(pkg.prefix.bin): return - - for file in os.listdir(pkg.prefix.bin): - path = os.path.join(pkg.prefix.bin, file) - if shebang_too_long(path): - filter_shebang(path) - + filter_shebangs_in_directory(pkg.prefix.bin) From 267e83d8a70324d9b8a70a86085853dd762c7f93 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Wed, 9 Mar 2016 21:12:11 -0500 Subject: [PATCH 549/910] Added emacs package. --- .../repos/builtin/packages/emacs/package.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 var/spack/repos/builtin/packages/emacs/package.py diff --git a/var/spack/repos/builtin/packages/emacs/package.py b/var/spack/repos/builtin/packages/emacs/package.py new file mode 100644 index 00000000000..e496ffc3d1f --- /dev/null +++ b/var/spack/repos/builtin/packages/emacs/package.py @@ -0,0 +1,36 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install emacs +# +# You can always get back here to change things with: +# +# spack edit emacs +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class Emacs(Package): + """FIXME: put a proper description of your package here.""" + # FIXME: add a proper url for your package's homepage here. + homepage = "http://www.example.com" + url = "http://ftp.gnu.org/gnu/emacs/emacs-24.5.tar.gz" + + version('24.5', 'd74b597503a68105e61b5b9f6d065b44') + + # FIXME: Add dependencies if this package requires them. + depends_on('ncurses') + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + configure('--prefix=%s' % prefix) + + # FIXME: Add logic to build and install here + make() + make("install") From b701aa10d4b40eda3d5ea3f0c96b5aed7493c7a7 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Wed, 9 Mar 2016 21:21:25 -0500 Subject: [PATCH 550/910] Fixed up --- .../repos/builtin/packages/emacs/package.py | 29 ++++--------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/var/spack/repos/builtin/packages/emacs/package.py b/var/spack/repos/builtin/packages/emacs/package.py index e496ffc3d1f..09eb05d5a76 100644 --- a/var/spack/repos/builtin/packages/emacs/package.py +++ b/var/spack/repos/builtin/packages/emacs/package.py @@ -1,36 +1,19 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install emacs -# -# You can always get back here to change things with: -# -# spack edit emacs -# -# See the spack documentation for more information on building -# packages. -# from spack import * class Emacs(Package): - """FIXME: put a proper description of your package here.""" - # FIXME: add a proper url for your package's homepage here. - homepage = "http://www.example.com" + """The Emacs programmable text editor.""" + homepage = "https://www.gnu.org/software/emacs" url = "http://ftp.gnu.org/gnu/emacs/emacs-24.5.tar.gz" version('24.5', 'd74b597503a68105e61b5b9f6d065b44') - # FIXME: Add dependencies if this package requires them. depends_on('ncurses') + # Emacs also depends on: + # GTK or other widget library + # libtiff, png, etc. + # For now, we assume the system provides all that stuff. def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. configure('--prefix=%s' % prefix) - - # FIXME: Add logic to build and install here make() make("install") From b0377da771d9154956a408a59e97000049d7c2fb Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 31 Jan 2016 12:21:04 -0800 Subject: [PATCH 551/910] update mirror config documentation. - mirrors.yaml uses Spack's OrderedDict rather than lists. --- lib/spack/docs/mirrors.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index 7581a0e9ede..b20fedb55f1 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -186,7 +186,7 @@ Each mirror has a name so that you can refer to it again later. ``spack mirror list`` ---------------------------- -If you want to see all the mirrors Spack knows about you can run ``spack mirror list``:: +To see all the mirrors Spack knows about, run ``spack mirror list``:: $ spack mirror list local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 @@ -196,7 +196,7 @@ If you want to see all the mirrors Spack knows about you can run ``spack mirror ``spack mirror remove`` ---------------------------- -And, if you want to remove a mirror, just remove it by name:: +To remove a mirror by name:: $ spack mirror remove local_filesystem $ spack mirror list @@ -205,11 +205,11 @@ And, if you want to remove a mirror, just remove it by name:: Mirror precedence ---------------------------- -Adding a mirror really just adds a section in ``~/.spack/mirrors.yaml``:: +Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``:: mirrors: - - local_filesystem: file:///Users/gamblin2/spack-mirror-2014-06-24 - - remote_server: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 + local_filesystem: file:///Users/gamblin2/spack-mirror-2014-06-24 + remote_server: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 If you want to change the order in which mirrors are searched for packages, you can edit this file and reorder the sections. Spack will From 0244d794cd68efd68edd6a797dd1db158aca87b6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 31 Jan 2016 13:28:12 -0800 Subject: [PATCH 552/910] remove unnecessary import --- lib/spack/spack/stage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 4703a3aae6a..5354135e6a3 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -26,7 +26,6 @@ import errno import shutil import tempfile -import sys from urlparse import urljoin import llnl.util.tty as tty From b0572a546242ef1c570f5dd3c9c6336bc1d55607 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 7 Feb 2016 11:27:39 -0700 Subject: [PATCH 553/910] Minor tweaks to abi code. --- lib/spack/spack/abi.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index f0a997703cd..7e565bcbf9e 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -69,11 +69,11 @@ def _gcc_get_libstdcxx_version(self, version): if not libpath: return None return os.path.basename(libpath) - - + + @memoized def _gcc_compiler_compare(self, pversion, cversion): - """Returns true iff the gcc version pversion and cversion + """Returns true iff the gcc version pversion and cversion are ABI compatible.""" plib = self._gcc_get_libstdcxx_version(pversion) clib = self._gcc_get_libstdcxx_version(cversion) @@ -86,43 +86,43 @@ def _intel_compiler_compare(self, pversion, cversion): """Returns true iff the intel version pversion and cversion are ABI compatible""" - #Test major and minor versions. Ignore build version. + # Test major and minor versions. Ignore build version. if (len(pversion.version) < 2 or len(cversion.version) < 2): return False - return (pversion.version[0] == cversion.version[0]) and \ - (pversion.version[1] == cversion.version[1]) - - + return pversion.version[:2] == cversion.version[:2] + + def compiler_compatible(self, parent, child, **kwargs): """Returns true iff the compilers for parent and child specs are ABI compatible""" if not parent.compiler or not child.compiler: return True - + if parent.compiler.name != child.compiler.name: - #Different compiler families are assumed ABI incompatible + # Different compiler families are assumed ABI incompatible return False - + if kwargs.get('loose', False): return True + # TODO: Can we move the specialized ABI matching stuff + # TODO: into compiler classes? for pversion in parent.compiler.versions: for cversion in child.compiler.versions: - #For a few compilers use specialized comparisons. Otherwise + # For a few compilers use specialized comparisons. Otherwise # match on version match. if pversion.satisfies(cversion): return True - elif parent.compiler.name == "gcc" and \ - self._gcc_compiler_compare(pversion, cversion): + elif (parent.compiler.name == "gcc" and + self._gcc_compiler_compare(pversion, cversion)): return True - elif parent.compiler.name == "intel" and \ - self._intel_compiler_compare(pversion, cversion): + elif (parent.compiler.name == "intel" and + self._intel_compiler_compare(pversion, cversion)): return True return False - + def compatible(self, parent, child, **kwargs): """Returns true iff a parent and child spec are ABI compatible""" loosematch = kwargs.get('loose', False) return self.architecture_compatible(parent, child) and \ self.compiler_compatible(parent, child, loose=loosematch) - From 048c406f49a3c7a30008268590ab57b74ea60b6b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 7 Feb 2016 11:32:26 -0700 Subject: [PATCH 554/910] Remove vestigial variants in directory name. --- lib/spack/spack/directory_layout.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index b94468faf01..39ee4e203d6 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -198,14 +198,10 @@ def hidden_file_paths(self): def relative_path_for_spec(self, spec): _check_concrete(spec) - + if spec.external: return spec.external - enabled_variants = ( - '-' + v.name for v in spec.variants.values() - if v.enabled) - dir_name = "%s-%s-%s" % ( spec.name, spec.version, From 1fe196f95cc26cac73abe64752ff67b150f4d50a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 2 Mar 2016 22:38:21 -0800 Subject: [PATCH 555/910] whitespace and formatting --- lib/spack/spack/preferred_packages.py | 38 +++++++++---------- lib/spack/spack/spec.py | 8 ++-- .../repos/builtin/packages/python/package.py | 5 ++- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index eaea016a85a..4ff0f18b31d 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -33,8 +33,8 @@ def __init__(self): self.preferred = spack.config.get_config('packages') self._spec_for_pkgname_cache = {} - #Given a package name, sort component (e.g, version, compiler, ...), and - # a second_key (used by providers), return the list + # Given a package name, sort component (e.g, version, compiler, ...), and + # a second_key (used by providers), return the list def _order_for_package(self, pkgname, component, second_key, test_all=True): pkglist = [pkgname] if test_all: @@ -47,10 +47,10 @@ def _order_for_package(self, pkgname, component, second_key, test_all=True): continue return [str(s).strip() for s in order] return [] - - # A generic sorting function. Given a package name and sort - # component, return less-than-0, 0, or greater-than-0 if + + # A generic sorting function. Given a package name and sort + # component, return less-than-0, 0, or greater-than-0 if # a is respectively less-than, equal to, or greater than b. def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key): if a is None: @@ -76,7 +76,7 @@ def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, cmp_a = orderlist.index(str(a)) cmp_b = orderlist.index(str(b)) reverse = 1 - + if cmp_a < cmp_b: return -1 * reverse elif cmp_a > cmp_b: @@ -87,7 +87,7 @@ def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, # A sorting function for specs. Similar to component_compare, but # a and b are considered to match entries in the sorting list if they - # satisfy the list component. + # satisfy the list component. def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key): if not a or not a.concrete: return -1 @@ -121,7 +121,7 @@ def _spec_for_pkgname(self, pkgname, component, second_key): key = (pkgname, component, second_key) if not key in self._spec_for_pkgname_cache: pkglist = self._order_for_package(pkgname, component, second_key) - if not pkglist: + if not pkglist: if component in self._default_order: pkglist = self._default_order[component] if component == 'compiler': @@ -132,9 +132,9 @@ def _spec_for_pkgname(self, pkgname, component, second_key): self._spec_for_pkgname_cache[key] = [spack.spec.Spec(s) for s in pkglist] return self._spec_for_pkgname_cache[key] - + def provider_compare(self, pkgname, provider_str, a, b): - """Return less-than-0, 0, or greater than 0 if a is respecively less-than, equal-to, or + """Return less-than-0, 0, or greater than 0 if a is respecively less-than, equal-to, or greater-than b. A and b are possible implementations of provider_str. One provider is less-than another if it is preferred over the other. For example, provider_compare('scorep', 'mpi', 'mvapich', 'openmpi') would return -1 if @@ -148,28 +148,28 @@ def spec_has_preferred_provider(self, pkgname, provider_str): def version_compare(self, pkgname, a, b): - """Return less-than-0, 0, or greater than 0 if version a of pkgname is - respecively less-than, equal-to, or greater-than version b of pkgname. + """Return less-than-0, 0, or greater than 0 if version a of pkgname is + respecively less-than, equal-to, or greater-than version b of pkgname. One version is less-than another if it is preferred over the other.""" return self._spec_compare(pkgname, 'version', a, b, True, None) - + def variant_compare(self, pkgname, a, b): - """Return less-than-0, 0, or greater than 0 if variant a of pkgname is - respecively less-than, equal-to, or greater-than variant b of pkgname. + """Return less-than-0, 0, or greater than 0 if variant a of pkgname is + respecively less-than, equal-to, or greater-than variant b of pkgname. One variant is less-than another if it is preferred over the other.""" return self._component_compare(pkgname, 'variant', a, b, False, None) def architecture_compare(self, pkgname, a, b): - """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is - respecively less-than, equal-to, or greater-than architecture b of pkgname. + """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is + respecively less-than, equal-to, or greater-than architecture b of pkgname. One architecture is less-than another if it is preferred over the other.""" return self._component_compare(pkgname, 'architecture', a, b, False, None) def compiler_compare(self, pkgname, a, b): - """Return less-than-0, 0, or greater than 0 if compiler a of pkgname is - respecively less-than, equal-to, or greater-than compiler b of pkgname. + """Return less-than-0, 0, or greater than 0 if compiler a of pkgname is + respecively less-than, equal-to, or greater-than compiler b of pkgname. One compiler is less-than another if it is preferred over the other.""" return self._spec_compare(pkgname, 'compiler', a, b, False, None) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 6f55065f010..b8c0d0ef9c5 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -428,7 +428,7 @@ def __init__(self, spec_like, *dep_like, **kwargs): for dep in dep_like: spec = dep if isinstance(dep, Spec) else Spec(dep) self._add_dependency(spec) - + # # Private routines here are called by the parser when building a spec. @@ -1410,7 +1410,7 @@ def _dup(self, other, **kwargs): self.architecture != other.architecture and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ self.concrete != other.concrete and self.external != other.external) - + # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() @@ -1585,7 +1585,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): $@ Version with '@' prefix $% Compiler with '%' prefix $%@ Compiler with '%' prefix & compiler version with '@' prefix - $+ Options + $+ Options $= Architecture with '=' prefix $# 7-char prefix of DAG hash with '-' prefix $$ $ @@ -1738,7 +1738,7 @@ def __cmp__(self, other): #Package name sort order is not configurable, always goes alphabetical if self.name != other.name: return cmp(self.name, other.name) - + #Package version is second in compare order pkgname = self.name if self.versions != other.versions: diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 58d401244e3..dd240d1ea0a 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -34,8 +34,9 @@ def install(self, spec, prefix): env['PYTHONHOME'] = prefix env['MACOSX_DEPLOYMENT_TARGET'] = '10.6' - # Rest of install is pretty standard except setup.py needs to be able to read the CPPFLAGS - # and LDFLAGS as it scans for the library and headers to build + # Rest of install is pretty standard except setup.py needs to + # be able to read the CPPFLAGS and LDFLAGS as it scans for the + # library and headers to build configure_args= [ "--prefix=%s" % prefix, "--with-threads", From 82b7067fdfc3f2fb90cff9014ab5379e334b40fd Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 3 Mar 2016 00:44:00 -0800 Subject: [PATCH 556/910] Refactored external packages slightly. - Move `Spec.__cmp__` out of spec, into concretize as `cmp_specs`. - `Spec.__cmp__` was never called (except explicitly) due to rich comparison operators from `key_ordering` - Refactor `_find_other_spec` to free function `find_spec`. Add a test for it to make sure it works. --- lib/spack/spack/concretize.py | 142 +++++++++++++++++--------- lib/spack/spack/preferred_packages.py | 2 +- lib/spack/spack/spec.py | 34 ------ lib/spack/spack/test/concretize.py | 64 ++++++++++++ 4 files changed, 158 insertions(+), 84 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 8da7011b538..bad67c34e3f 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -50,34 +50,17 @@ class DefaultConcretizer(object): default concretization strategies, or you can override all of them. """ - def _find_other_spec(self, spec, condition): - """Searches the dag from spec in an intelligent order and looks - for a spec that matches a condition""" - dagiter = chain(spec.traverse(direction='parents'), spec.traverse(direction='children')) - found = next((x for x in dagiter if x is not spec and condition(x)), None) - if found: - return found - dagiter = chain(spec.traverse(direction='parents'), spec.traverse(direction='children')) - searched = list(dagiter) - found = next((x for x in spec.root.traverse() if x not in searched and x is not spec and condition(x)), None) - if found: - return found - if condition(spec): - return spec - return None - - def _valid_virtuals_and_externals(self, spec): """Returns a list of spec/external-path pairs for both virtuals and externals - that can concretize this spec.""" + that can concretize this spec.""" # Get a list of candidate packages that could satisfy this spec packages = [] if spec.virtual: providers = spack.repo.providers_for(spec) if not providers: raise UnsatisfiableProviderSpecError(providers[0], spec) - spec_w_preferred_providers = self._find_other_spec(spec, \ - lambda(x): spack.pkgsort.spec_has_preferred_provider(x.name, spec.name)) + spec_w_preferred_providers = find_spec( + spec, lambda(x): spack.pkgsort.spec_has_preferred_provider(x.name, spec.name)) if not spec_w_preferred_providers: spec_w_preferred_providers = spec provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name) @@ -101,15 +84,15 @@ def _valid_virtuals_and_externals(self, spec): raise NoBuildError(spec) def cmp_externals(a, b): - result = a[0].__cmp__(b[0]) - if result != 0: return result + result = cmp_specs(a[0], b[0]) + if result != 0: + return result if not a[1] and b[1]: return 1 if not b[1] and a[1]: return -1 - return a[1].__cmp__(b[1]) + return cmp_specs(a[1], b[1]) - #result = sorted(result, cmp=lambda a,b: a[0].__cmp__(b[0])) result = sorted(result, cmp=cmp_externals) return result @@ -121,27 +104,27 @@ def concretize_virtual_and_external(self, spec): if not candidates: return False - #Find the nearest spec in the dag that has a compiler. We'll use that + # Find the nearest spec in the dag that has a compiler. We'll use that # spec to test compiler compatibility. - other_spec = self._find_other_spec(spec, lambda(x): x.compiler) + other_spec = find_spec(spec, lambda(x): x.compiler) if not other_spec: other_spec = spec.root - #Choose an ABI-compatible candidate, or the first match otherwise. + # Choose an ABI-compatible candidate, or the first match otherwise. candidate = None if other_spec: candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec)), None) if not candidate: - #Try a looser ABI matching + # Try a looser ABI matching candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec, loose=True)), None) if not candidate: - #No ABI matches. Pick the top choice based on the orignal preferences. + # No ABI matches. Pick the top choice based on the orignal preferences. candidate = candidates[0] candidate_spec = candidate[0] external = candidate[1] changed = False - #If we're external then trim the dependencies + # If we're external then trim the dependencies if external: if (spec.dependencies): changed = True @@ -150,26 +133,26 @@ def concretize_virtual_and_external(self, spec): def fequal(candidate_field, spec_field): return (not candidate_field) or (candidate_field == spec_field) - if fequal(candidate_spec.name, spec.name) and \ - fequal(candidate_spec.versions, spec.versions) and \ - fequal(candidate_spec.compiler, spec.compiler) and \ - fequal(candidate_spec.architecture, spec.architecture) and \ - fequal(candidate_spec.dependencies, spec.dependencies) and \ - fequal(candidate_spec.variants, spec.variants) and \ - fequal(external, spec.external): + if (fequal(candidate_spec.name, spec.name) and + fequal(candidate_spec.versions, spec.versions) and + fequal(candidate_spec.compiler, spec.compiler) and + fequal(candidate_spec.architecture, spec.architecture) and + fequal(candidate_spec.dependencies, spec.dependencies) and + fequal(candidate_spec.variants, spec.variants) and + fequal(external, spec.external)): return changed - - #Refine this spec to the candidate. + + # Refine this spec to the candidate. if spec.virtual: spec._replace_with(candidate_spec) changed = True if spec._dup(candidate_spec, deps=False, cleardeps=False): changed = True - spec.external = external + spec.external = external return changed - - + + def concretize_version(self, spec): """If the spec is already concrete, return. Otherwise take the preferred version from spackconfig, and default to the package's @@ -263,7 +246,7 @@ def concretize_compiler(self, spec): """If the spec already has a compiler, we're done. If not, then take the compiler used for the nearest ancestor with a compiler spec and use that. If the ancestor's compiler is not - concrete, then used the preferred compiler as specified in + concrete, then used the preferred compiler as specified in spackconfig. Intuition: Use the spackconfig default if no package that depends on @@ -272,37 +255,99 @@ def concretize_compiler(self, spec): link to this one, to maximize compatibility. """ all_compilers = spack.compilers.all_compilers() - + if (spec.compiler and spec.compiler.concrete and spec.compiler in all_compilers): return False #Find the another spec that has a compiler, or the root if none do - other_spec = self._find_other_spec(spec, lambda(x) : x.compiler) + other_spec = find_spec(spec, lambda(x) : x.compiler) if not other_spec: other_spec = spec.root other_compiler = other_spec.compiler assert(other_spec) - + # Check if the compiler is already fully specified if other_compiler in all_compilers: spec.compiler = other_compiler.copy() return True - + # Filter the compilers into a sorted list based on the compiler_order from spackconfig compiler_list = all_compilers if not other_compiler else spack.compilers.find(other_compiler) cmp_compilers = partial(spack.pkgsort.compiler_compare, other_spec.name) matches = sorted(compiler_list, cmp=cmp_compilers) if not matches: raise UnavailableCompilerVersionError(other_compiler) - + # copy concrete version into other_compiler spec.compiler = matches[0].copy() assert(spec.compiler.concrete) return True # things changed. +def find_spec(spec, condition): + """Searches the dag from spec in an intelligent order and looks + for a spec that matches a condition""" + # First search parents, then search children + dagiter = chain(spec.traverse(direction='parents', root=False), + spec.traverse(direction='children', root=False)) + visited = set() + for relative in dagiter: + if condition(relative): + return relative + visited.add(id(relative)) + + # Then search all other relatives in the DAG *except* spec + for relative in spec.root.traverse(): + if relative is spec: continue + if id(relative) in visited: continue + if condition(relative): + return relative + + # Finally search spec itself. + if condition(spec): + return spec + + return None # Nohting matched the condition. + + +def cmp_specs(lhs, rhs): + # Package name sort order is not configurable, always goes alphabetical + if lhs.name != rhs.name: + return cmp(lhs.name, rhs.name) + + # Package version is second in compare order + pkgname = lhs.name + if lhs.versions != rhs.versions: + return spack.pkgsort.version_compare( + pkgname, lhs.versions, rhs.versions) + + # Compiler is third + if lhs.compiler != rhs.compiler: + return spack.pkgsort.compiler_compare( + pkgname, lhs.compiler, rhs.compiler) + + # Variants + if lhs.variants != rhs.variants: + return spack.pkgsort.variant_compare( + pkgname, lhs.variants, rhs.variants) + + # Architecture + if lhs.architecture != rhs.architecture: + return spack.pkgsort.architecture_compare( + pkgname, lhs.architecture, rhs.architecture) + + # Dependency is not configurable + lhash, rhash = hash(lhs), hash(rhs) + if lhash != rhash: + return -1 if lhash < rhash else 1 + + # Equal specs + return 0 + + + class UnavailableCompilerVersionError(spack.error.SpackError): """Raised when there is no available compiler that satisfies a compiler spec.""" @@ -326,4 +371,3 @@ class NoBuildError(spack.error.SpackError): def __init__(self, spec): super(NoBuildError, self).__init__( "The spec '%s' is configured as nobuild, and no matching external installs were found" % spec.name) - diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index 4ff0f18b31d..9d219a1a6ef 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -27,7 +27,7 @@ from spack.version import * class PreferredPackages(object): - _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] }, #Arbitrary, but consistent + _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] }, # Arbitrary, but consistent def __init__(self): self.preferred = spack.config.get_config('packages') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index b8c0d0ef9c5..c045e803655 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1734,40 +1734,6 @@ def dep_string(self): return ''.join("^" + dep.format() for dep in self.sorted_deps()) - def __cmp__(self, other): - #Package name sort order is not configurable, always goes alphabetical - if self.name != other.name: - return cmp(self.name, other.name) - - #Package version is second in compare order - pkgname = self.name - if self.versions != other.versions: - return spack.pkgsort.version_compare(pkgname, - self.versions, other.versions) - - #Compiler is third - if self.compiler != other.compiler: - return spack.pkgsort.compiler_compare(pkgname, - self.compiler, other.compiler) - - #Variants - if self.variants != other.variants: - return spack.pkgsort.variant_compare(pkgname, - self.variants, other.variants) - - #Architecture - if self.architecture != other.architecture: - return spack.pkgsort.architecture_compare(pkgname, - self.architecture, other.architecture) - - #Dependency is not configurable - if self.dag_hash() != other.dag_hash(): - return -1 if self.dag_hash() < other.dag_hash() else 1 - - #Equal specs - return 0 - - def __str__(self): return self.format() + self.dep_string() diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 820c5d84a89..07828d8ea6c 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -24,6 +24,7 @@ ############################################################################## import spack from spack.spec import Spec, CompilerSpec +from spack.concretize import find_spec from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): @@ -218,3 +219,66 @@ def test_external_and_virtual(self): self.assertEqual(spec['stuff'].external, '/path/to/external_virtual_gcc') self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) self.assertTrue(spec['stuff'].compiler.satisfies('gcc')) + + + def test_find_spec_parents(self): + """Tests the spec finding logic used by concretization. """ + s = Spec('a +foo', + Spec('b +foo', + Spec('c'), + Spec('d +foo')), + Spec('e +foo')) + + self.assertEqual('a', find_spec(s['b'], lambda s: '+foo' in s).name) + + + def test_find_spec_children(self): + s = Spec('a', + Spec('b +foo', + Spec('c'), + Spec('d +foo')), + Spec('e +foo')) + self.assertEqual('d', find_spec(s['b'], lambda s: '+foo' in s).name) + s = Spec('a', + Spec('b +foo', + Spec('c +foo'), + Spec('d')), + Spec('e +foo')) + self.assertEqual('c', find_spec(s['b'], lambda s: '+foo' in s).name) + + + def test_find_spec_sibling(self): + s = Spec('a', + Spec('b +foo', + Spec('c'), + Spec('d')), + Spec('e +foo')) + self.assertEqual('e', find_spec(s['b'], lambda s: '+foo' in s).name) + self.assertEqual('b', find_spec(s['e'], lambda s: '+foo' in s).name) + + s = Spec('a', + Spec('b +foo', + Spec('c'), + Spec('d')), + Spec('e', + Spec('f +foo'))) + self.assertEqual('f', find_spec(s['b'], lambda s: '+foo' in s).name) + + + def test_find_spec_self(self): + s = Spec('a', + Spec('b +foo', + Spec('c'), + Spec('d')), + Spec('e')) + self.assertEqual('b', find_spec(s['b'], lambda s: '+foo' in s).name) + + + def test_find_spec_none(self): + s = Spec('a', + Spec('b', + Spec('c'), + Spec('d')), + Spec('e')) + self.assertEqual(None, find_spec(s['b'], lambda s: '+foo' in s)) + From 4f14db8af214b6648da04c5c5121757ea171f18c Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 10 Mar 2016 10:34:51 +0100 Subject: [PATCH 557/910] netlib-lapack : optional dependency on external blas --- .../repos/builtin/packages/netlib-lapack/package.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py index 1bb77919d50..a208929d08b 100644 --- a/var/spack/repos/builtin/packages/netlib-lapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -19,19 +19,29 @@ class NetlibLapack(Package): variant('debug', default=False, description='Activates the Debug build type') variant('shared', default=True, description="Build shared library version") + variant('external-blas', default=False, description='Build lapack with an external blas') variant('lapacke', default=True, description='Activates the build of the LAPACKE C interface') # virtual dependency - provides('blas') + provides('blas', when='~external-blas') provides('lapack') depends_on('cmake') + depends_on('blas', when='+external-blas') def install(self, spec, prefix): cmake_args = ['-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), '-DLAPACKE:BOOL=%s' % ('ON' if '+lapacke' in spec else 'OFF')] + if '+external-blas' in spec: + # TODO : the mechanism to specify the library should be more general, + # TODO : but this allows to have an hook to an external blas + cmake_args.extend([ + '-DUSE_OPTIMIZED_BLAS:BOOL=ON', + '-DBLAS_LIBRARIES:PATH=%s' % join_path(spec['blas'].prefix.lib, 'libblas.a') + ]) + cmake_args.extend(std_cmake_args) with working_dir('spack-build', create=True): From c4fddcc6e977eb7040c48581eb90f1a441cf3e71 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 10 Mar 2016 03:15:49 -0800 Subject: [PATCH 558/910] Add 'provders' back into packages.yaml schema --- lib/spack/spack/config.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 95a988f7ffe..3a785fe6924 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -224,6 +224,15 @@ 'type': 'boolean', 'default': False, }, + 'providers': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*': { + 'type' : 'array', + 'default' : [], + 'items' : { 'type' : 'string' },},},}, 'paths': { 'type' : 'object', 'default' : {}, @@ -534,11 +543,11 @@ def spec_externals(spec): allpkgs = get_config('packages') name = spec.name spec_locations = [] - + pkg_paths = allpkgs.get(name, {}).get('paths', None) if not pkg_paths: return [] - + for pkg,path in pkg_paths.iteritems(): if not spec.satisfies(pkg): continue From 7ae6c62ddb0271d9e2364cf4249adbc7abbc0b31 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 10 Mar 2016 14:08:43 +0100 Subject: [PATCH 559/910] Fixed shell quoting error --- lib/spack/env/cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 6b6073db430..2888c28c489 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -154,7 +154,7 @@ fi input_command="$@" if [ "$mode" == vcheck ] ; then - exec "${input_command}" + exec ${command} ${input_command} fi # From 78ef0618bc6d60f50edc6bb83b5cb1a5a48ac22b Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 10 Mar 2016 14:35:09 +0100 Subject: [PATCH 560/910] cc : converted indents to spaces --- lib/spack/env/cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 2888c28c489..3e429fdf220 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -119,7 +119,7 @@ if [ -z "$mode" ]; then if [ "$arg" = -v -o "$arg" = -V -o "$arg" = --version -o "$arg" = -dumpversion ]; then mode=vcheck break - fi + fi done fi From 30adc4c9b8cdbe099ebe4ef9bec4b2594cda7b61 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 10 Mar 2016 14:59:31 +0100 Subject: [PATCH 561/910] cc : handle spaces in folders? --- lib/spack/env/cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 3e429fdf220..4a3e6eddc94 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -154,7 +154,7 @@ fi input_command="$@" if [ "$mode" == vcheck ] ; then - exec ${command} ${input_command} + exec ${command} "$@" fi # From 2bbf42b49d874adad2d9f53e1c8f6225df80415b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 10 Mar 2016 16:18:11 -0800 Subject: [PATCH 562/910] Indentation change. --- lib/spack/docs/site_configuration.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index 76fe8fdd7d2..7ae541d4a39 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -56,7 +56,7 @@ directory is. External Packages ~~~~~~~~~~~~~~~~~~~~~ -Spack can be configured to use externally-installed +Spack can be configured to use externally-installed packages rather than building its own packages. This may be desirable if machines ship with system packages, such as a customized MPI that should be used instead of Spack building its own MPI. @@ -76,7 +76,7 @@ directory. Here's an example of an external configuration: This example lists three installations of OpenMPI, one built with gcc, one built with gcc and debug information, and another built with Intel. -If Spack is asked to build a package that uses one of these MPIs as a +If Spack is asked to build a package that uses one of these MPIs as a dependency, it will use the the pre-installed OpenMPI in the given directory. This example also specifies that Spack should never build its own OpenMPI via the ``nobuild: True`` option. @@ -85,12 +85,12 @@ Each ``packages.yaml`` begins with a ``packages:`` token, followed by a list of package names. To specify externals, add a ``paths`` token under the package name, which lists externals in a ``spec : /path`` format. Each spec should be as -well-defined as reasonably possible. If a -package lacks a spec component, such as missing a compiler or -package version, then Spack will guess the missing component based +well-defined as reasonably possible. If a +package lacks a spec component, such as missing a compiler or +package version, then Spack will guess the missing component based on its most-favored packages, and it may guess incorrectly. -Each package version and compilers listed in an external should +Each package version and compilers listed in an external should have entries in Spack's packages and compiler configuration, even though the package and compiler may not every be built. @@ -111,18 +111,18 @@ be: paths: openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel - nobuild: True + openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel + nobuild: True The addition of the ``nobuild`` flag tells Spack that it should never build its own version of OpenMPI, and it will instead always rely on a pre-built OpenMPI. Similar to ``paths``, ``nobuild`` is specified as a property under a package name. -The ``nobuild`` does not need to be paired with external packages. -It could also be used alone to forbid packages that may be +The ``nobuild`` does not need to be paired with external packages. +It could also be used alone to forbid packages that may be buggy or otherwise undesirable. - + Profiling ~~~~~~~~~~~~~~~~~~~~~ From 4693af0736910244abcb193a65041314447f0da6 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Thu, 10 Mar 2016 16:23:35 -0800 Subject: [PATCH 563/910] Fix type error that was causing mis-ordering of compiler versions --- lib/spack/spack/preferred_packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index 9d219a1a6ef..4d8526c75fa 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -27,7 +27,7 @@ from spack.version import * class PreferredPackages(object): - _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] }, # Arbitrary, but consistent + _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] } # Arbitrary, but consistent def __init__(self): self.preferred = spack.config.get_config('packages') From ac88cab68ffa16d4ff448de5f186d746cb36b40a Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Thu, 10 Mar 2016 17:00:27 -0800 Subject: [PATCH 564/910] Fix issue with preferred satisfies not being respected --- lib/spack/spack/concretize.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index bad67c34e3f..2268084e568 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -70,20 +70,23 @@ def _valid_virtuals_and_externals(self, spec): # For each candidate package, if it has externals add those to the candidates # if it's a nobuild, then only add the externals. - result = [] + candidates = [] all_compilers = spack.compilers.all_compilers() for pkg in packages: externals = spec_externals(pkg) buildable = not is_spec_nobuild(pkg) if buildable: - result.append((pkg, None)) + candidates.append((pkg, None)) for ext in externals: if ext[0].satisfies(spec): - result.append(ext) - if not result: + candidates.append(ext) + if not candidates: raise NoBuildError(spec) def cmp_externals(a, b): + if a[0].name != b[0].name: + #We're choosing between different providers. Maintain order from above sort + return candidates.index(a) - candidates.index(b) result = cmp_specs(a[0], b[0]) if result != 0: return result @@ -91,10 +94,10 @@ def cmp_externals(a, b): return 1 if not b[1] and a[1]: return -1 - return cmp_specs(a[1], b[1]) + return cmp(a[1], b[1]) - result = sorted(result, cmp=cmp_externals) - return result + candidates = sorted(candidates, cmp=cmp_externals) + return candidates def concretize_virtual_and_external(self, spec): From 1d70b590fc4579c4e6b4bd592bc58eabf330fd9b Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 11 Mar 2016 13:20:57 +0100 Subject: [PATCH 565/910] build_environment : fixed minor spelling errors and a few style issues --- lib/spack/spack/build_environment.py | 37 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 87fc310b5a3..392ba7ea4d5 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -3,7 +3,7 @@ build environment. All of this is set up by package.py just before install() is called. -There are two parts to the bulid environment: +There are two parts to the build environment: 1. Python build environment (i.e. install() method) @@ -13,7 +13,7 @@ the package's module scope. Ths allows package writers to call them all directly in Package.install() without writing 'self.' everywhere. No, this isn't Pythonic. Yes, it makes the code more - readable and more like the shell script from whcih someone is + readable and more like the shell script from which someone is likely porting. 2. Build execution environment @@ -27,17 +27,16 @@ Skimming this module is a nice way to get acquainted with the types of calls you can make from within the install() function. """ -import os -import sys -import shutil import multiprocessing +import os import platform -from llnl.util.filesystem import * +import shutil +import sys import spack -import spack.compilers as compilers -from spack.util.executable import Executable, which +from llnl.util.filesystem import * from spack.util.environment import * +from spack.util.executable import Executable, which # # This can be set by the user to globally disable parallel builds. @@ -107,18 +106,19 @@ def set_compiler_environment_variables(pkg): if compiler.fc: os.environ['SPACK_FC'] = compiler.fc - os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) + os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) def set_build_environment_variables(pkg): - """This ensures a clean install environment when we build packages. + """ + This ensures a clean install environment when we build packages """ # Add spack build environment path with compiler wrappers first in # the path. We add both spack.env_path, which includes default # wrappers (cc, c++, f77, f90), AND a subdirectory containing # compiler-specific symlinks. The latter ensures that builds that # are sensitive to the *name* of the compiler see the right name - # when we're building wtih the wrappers. + # when we're building with the wrappers. # # Conflicts on case-insensitive systems (like "CC" and "cc") are # handled by putting one in the /case-insensitive @@ -296,23 +296,23 @@ def child_fun(): # do stuff build_env.fork(pkg, child_fun) - Forked processes are run with the build environemnt set up by + Forked processes are run with the build environment set up by spack.build_environment. This allows package authors to have - full control over the environment, etc. without offecting + full control over the environment, etc. without affecting other builds that might be executed in the same spack call. - If something goes wrong, the child process is expected toprint + If something goes wrong, the child process is expected to print the error and the parent process will exit with error as well. If things go well, the child exits and the parent carries on. """ try: pid = os.fork() - except OSError, e: + except OSError as e: raise InstallError("Unable to fork build process: %s" % e) if pid == 0: - # Give the child process the package's build environemnt. + # Give the child process the package's build environment. setup_package(pkg) try: @@ -323,7 +323,7 @@ def child_fun(): # which interferes with unit tests. os._exit(0) - except spack.error.SpackError, e: + except spack.error.SpackError as e: e.die() except: @@ -338,8 +338,7 @@ def child_fun(): # message. Just make the parent exit with an error code. pid, returncode = os.waitpid(pid, 0) if returncode != 0: - raise InstallError("Installation process had nonzero exit code." - .format(str(returncode))) + raise InstallError("Installation process had nonzero exit code.".format(str(returncode))) class InstallError(spack.error.SpackError): From f5e8857c5e44719778e531bcc149c9fe228240b3 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Fri, 11 Mar 2016 09:51:12 -0700 Subject: [PATCH 566/910] + Rename variant 'sphinxbuild' to 'doc' as recommended in the discussion of PR#526. Also, remove a debug print statement that was accidentally committed. --- var/spack/repos/builtin/packages/cmake/package.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index 806c37a68cc..cc93c7067c3 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -39,12 +39,12 @@ class Cmake(Package): variant('ncurses', default=True, description='Enables the build of the ncurses gui') variant('qt', default=False, description='Enables the build of cmake-gui') - variant('sphinxbuild', default=False, description='Enables the generation of html and man page documentation') + variant('doc', default=False, description='Enables the generation of html and man page documentation') depends_on('ncurses', when='+ncurses') depends_on('qt', when='+qt') - depends_on('python@2.7.11:', when='+sphinxbuild') - depends_on('py-sphinx', when='+sphinxbuild') + depends_on('python@2.7.11:', when='+doc') + depends_on('py-sphinx', when='+doc') def url_for_version(self, version): """Handle CMake's version-based custom URLs.""" @@ -58,8 +58,6 @@ def validate(self, spec): :raises RuntimeError: in case of inconsistencies """ - print spec - if '+qt' in spec and spec.satisfies('^qt@5.4.0'): msg = 'qt-5.4.0 has broken CMake modules.' raise RuntimeError(msg) @@ -75,7 +73,7 @@ def install(self, spec, prefix): if '+qt' in spec: options.append('--qt-gui') - if '+sphinxbuild' in spec: + if '+doc' in spec: options.append('--sphinx-html') options.append('--sphinx-man') From 1c7f754e5b8cd7740f3c4a91ee22a0354b40844a Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Fri, 11 Mar 2016 10:00:00 -0800 Subject: [PATCH 567/910] Invert and rename the `nobuild` option in package.yaml configs to `buildable`. --- lib/spack/docs/site_configuration.rst | 15 +++++++-------- lib/spack/spack/concretize.py | 8 ++++---- lib/spack/spack/config.py | 16 ++++++++-------- lib/spack/spack/test/mock_packages_test.py | 4 ++-- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index 7ae541d4a39..3abfa21a9da 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -78,8 +78,7 @@ This example lists three installations of OpenMPI, one built with gcc, one built with gcc and debug information, and another built with Intel. If Spack is asked to build a package that uses one of these MPIs as a dependency, it will use the the pre-installed OpenMPI in -the given directory. This example also specifies that Spack should never -build its own OpenMPI via the ``nobuild: True`` option. +the given directory. Each ``packages.yaml`` begins with a ``packages:`` token, followed by a list of package names. To specify externals, add a ``paths`` @@ -111,16 +110,16 @@ be: paths: openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel - nobuild: True + openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel + buildable: False -The addition of the ``nobuild`` flag tells Spack that it should never build +The addition of the ``buildable`` flag tells Spack that it should never build its own version of OpenMPI, and it will instead always rely on a pre-built -OpenMPI. Similar to ``paths``, ``nobuild`` is specified as a property under +OpenMPI. Similar to ``paths``, ``buildable`` is specified as a property under a package name. -The ``nobuild`` does not need to be paired with external packages. -It could also be used alone to forbid packages that may be +The ``buildable`` does not need to be paired with external packages. +It could also be used alone to forbid packages that may be buggy or otherwise undesirable. diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 2268084e568..8d29a03f932 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -69,12 +69,12 @@ def _valid_virtuals_and_externals(self, spec): packages = [spec] # For each candidate package, if it has externals add those to the candidates - # if it's a nobuild, then only add the externals. + # if it's not buildable, then only add the externals. candidates = [] all_compilers = spack.compilers.all_compilers() for pkg in packages: externals = spec_externals(pkg) - buildable = not is_spec_nobuild(pkg) + buildable = is_spec_buildable(pkg) if buildable: candidates.append((pkg, None)) for ext in externals: @@ -369,8 +369,8 @@ def __init__(self, spec): class NoBuildError(spack.error.SpackError): - """Raised when a package is configured with the nobuild option, but + """Raised when a package is configured with the buildable option False, but no satisfactory external versions can be found""" def __init__(self, spec): super(NoBuildError, self).__init__( - "The spec '%s' is configured as nobuild, and no matching external installs were found" % spec.name) + "The spec '%s' is configured as not buildable, and no matching external installs were found" % spec.name) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3a785fe6924..a21dd6dbe1e 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -220,9 +220,9 @@ 'type' : 'array', 'default' : [], 'items' : { 'type' : 'string' } }, #compiler specs - 'nobuild': { + 'buildable': { 'type': 'boolean', - 'default': False, + 'default': True, }, 'providers': { 'type': 'object', @@ -557,15 +557,15 @@ def spec_externals(spec): return spec_locations -def is_spec_nobuild(spec): - """Return true if the spec pkgspec is configured as nobuild""" +def is_spec_buildable(spec): + """Return true if the spec pkgspec is configured as buildable""" allpkgs = get_config('packages') name = spec.name if not spec.name in allpkgs: - return False - if not 'nobuild' in allpkgs[spec.name]: - return False - return allpkgs[spec.name]['nobuild'] + return True + if not 'buildable' in allpkgs[spec.name]: + return True + return allpkgs[spec.name]['buildable'] class ConfigError(SpackError): pass diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 079cbcc1365..6d24a84150b 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -52,11 +52,11 @@ mock_packages_config = """\ packages: externaltool: - nobuild: True + buildable: False paths: externaltool@1.0%gcc@4.5.0: /path/to/external_tool externalvirtual: - nobuild: True + buildable: False paths: externalvirtual@2.0%clang@3.3: /path/to/external_virtual_clang externalvirtual@1.0%gcc@4.5.0: /path/to/external_virtual_gcc From bae03404f44ac60439e85e31a85bb6848db2a5e4 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 11 Mar 2016 12:51:45 -0600 Subject: [PATCH 568/910] Documentation typo fixes --- lib/spack/docs/index.rst | 2 +- lib/spack/docs/packaging_guide.rst | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 79757208c9a..d6ce52b747c 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -18,7 +18,7 @@ configurations can coexist on the same system. Most importantly, Spack is *simple*. It offers a simple *spec* syntax so that users can specify versions and configuration options concisely. Spack is also simple for package authors: package files -are writtin in pure Python, and specs allow package authors to +are written in pure Python, and specs allow package authors to maintain a single file for many different builds of the same package. See the :doc:`features` for examples and highlights. diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index ef9fd89b62f..169899212db 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -419,7 +419,7 @@ directory to the directory containing the downloaded archive before it calls your ``install`` method. Within ``install``, the path to the downloaded archive is available as ``self.stage.archive_file``. -Here is an example snippet for packages distribuetd as self-extracting +Here is an example snippet for packages distributed as self-extracting archives. The example sets permissions on the downloaded file to make it executable, then runs it with some arguments. @@ -1556,12 +1556,12 @@ you ask for a particular spec. ``Concretization Policies`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A user may have certain perferrences for how packages should +A user may have certain preferences for how packages should be concretized on their system. For example, one user may prefer packages built with OpenMPI and the Intel compiler. Another user may prefer packages be built with MVAPICH and GCC. -Spack can be configurated to prefer certain compilers, package +Spack can be configured to prefer certain compilers, package versions, depends_on, and variants during concretization. The preferred configuration can be controlled via the ``~/.spack/packages.yaml`` file for user configuations, or the @@ -1588,16 +1588,16 @@ At a high level, this example is specifying how packages should be concretized. The dyninst package should prefer using gcc 4.9 and be built with debug options. The gperftools package should prefer version 2.2 over 2.4. Every package on the system should prefer mvapich for -its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by perfering gcc 4.9). +its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). These options are used to fill in implicit defaults. Any of them can be overwritten on the command line if explicitly requested. -Each packages.yaml file begin with the string ``packages:`` and +Each packages.yaml file begins with the string ``packages:`` and package names are specified on the next level. The special string ``all`` applies settings to each package. Underneath each package name is one or more components: ``compiler``, ``variants``, ``version``, or ``providers``. Each component has an ordered list of spec -``constraints``, with earlier entries in the list being prefered over +``constraints``, with earlier entries in the list being preferred over later entries. Sometimes a package installation may have constraints that forbid From 6ec65cd4ca1e6c2d346c949e21902680f27364c5 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 11 Mar 2016 15:03:37 -0600 Subject: [PATCH 569/910] Add GNU Octave package --- .../repos/builtin/packages/octave/package.py | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 var/spack/repos/builtin/packages/octave/package.py diff --git a/var/spack/repos/builtin/packages/octave/package.py b/var/spack/repos/builtin/packages/octave/package.py new file mode 100644 index 00000000000..6810da6d98c --- /dev/null +++ b/var/spack/repos/builtin/packages/octave/package.py @@ -0,0 +1,182 @@ +from spack import * + +class Octave(Package): + """GNU Octave is a high-level language, primarily intended for numerical + computations. It provides a convenient command line interface for solving + linear and nonlinear problems numerically, and for performing other + numerical experiments using a language that is mostly compatible with + Matlab. It may also be used as a batch-oriented language.""" + + homepage = "https://www.gnu.org/software/octave/" + url = "ftp://ftp.gnu.org/gnu/octave/octave-4.0.0.tar.gz" + + version('4.0.0' , 'a69f8320a4f20a8480c1b278b1adb799') + + variant('readline', default=True) + variant('arpack', default=False) + variant('curl', default=False) + variant('fftw', default=False) + variant('fltk', default=False) + variant('fontconfig', default=False) + variant('freetype', default=False) + variant('glpk', default=False) + variant('gl2ps', default=False) + variant('gnuplot', default=False) + variant('magick', default=False) + variant('hdf5', default=False) + variant('jdk', default=False) + variant('llvm', default=False) + variant('opengl', default=False) + variant('qhull', default=False) + variant('qrupdate', default=False) + variant('qscintilla', default=False) + variant('qt', default=False) + variant('suiteparse', default=False) + variant('zlib', default=False) + + # Required dependencies + depends_on('blas') + depends_on('lapack') + depends_on('pcre') + + # Strongly recommended dependencies + depends_on('readline', when='+readline') + + # Optional dependencies + depends_on('arpack', when='+arpack') + depends_on('curl', when='+curl') + depends_on('fftw@3', when='+fftw') + depends_on('fltk', when='+fltk') + depends_on('fontconfig', when='+fontconfig') + depends_on('freetype', when='+freetype') + depends_on('glpk', when='+glpk') + #depends_on('gl2ps', when='+gl2ps') + depends_on('gnuplot', when='+gnuplot') + depends_on('ImageMagick', when='+magick') + depends_on('hdf5', when='+hdf5') + depends_on('jdk', when='+jdk') + depends_on('llvm', when='+llvm') + #depends_on('opengl', when='+opengl') + depends_on('qhull', when='+qhull') + #depends_on('qrupdate', when='+qrupdate') + #depends_on('qscintilla', when='+qscintilla) + depends_on('qt', when='+qt') + depends_on('SuiteSparse', when='suitesparse') + depends_on('zlib', when='+zlib') + + + def install(self, spec, prefix): + config_args = [ + "--prefix=%s" % prefix + ] + + # Required dependencies + config_args.extend([ + "--with-blas=%s" % spec['blas'].prefix.lib, + "--with-lapack=%s" % spec['lapack'].prefix.lib + ]) + + # Strongly recommended dependencies + if '+readline' in spec: + config_args.append('--enable-readline') + else: + config_args.append('--disable-readline') + + # Optional dependencies + if '+arpack' in spec: + config_args.extend([ + "--with-arpack-includedir=%s" % spec['arpack'].prefix.include, + "--with-arpack-libdir=%s" % spec['arpack'].prefix.lib + ]) + else: + config_args.append("--without-arpack") + + if '+curl' in spec: + config_args.extend([ + "--with-curl-includedir=%s" % spec['curl'].prefix.include, + "--with-curl-libdir=%s" % spec['curl'].prefix.lib + ]) + else: + config_args.append("--without-curl") + + if '+fftw' in spec: + config_args.extend([ + "--with-fftw3-includedir=%s" % spec['fftw'].prefix.include, + "--with-fftw3-libdir=%s" % spec['fftw'].prefix.lib, + "--with-fftw3f-includedir=%s" % spec['fftw'].prefix.include, + "--with-fftw3f-libdir=%s" % spec['fftw'].prefix.lib + ]) + else: + config_args.extend([ + "--without-fftw3", + "--without-fftw3f" + ]) + + if '+fltk' in spec: + config_args.extend([ + "--with-fltk-prefix=%s" % spec['fltk'].prefix, + "--with-fltk-exec-prefix=%s" % spec['fltk'].prefix + ]) + else: + config_args.append("--without-fltk") + + if '+glpk' in spec: + config_args.extend([ + "--with-glpk-includedir=%s" % spec['glpk'].prefix.include + "--with-glpk-libdir=%s" % spec['glpk'].prefix.lib + ]) + else: + config_args.append("--without-glpk") + + if '+magick' in spec: + config_args.append("--with-magick=%s" % spec['ImageMagick'].prefix.lib) + + if '+hdf5' in spec: + config_args.extend([ + "--with-hdf5-includedir=%s" % spec['hdf5'].prefix.include, + "--with-hdf5-libdir=%s" % spec['hdf5'].prefix.lib + ]) + else: + config_args.append("--without-hdf5") + + if '+jdk' in spec: + config_args.extend([ + "--with-java-homedir=%s" % spec['jdk'].prefix, + "--with-java-includedir=%s" % spec['jdk'].prefix.include, + "--with-java-libdir=%s" % spec['jdk'].prefix.lib + ]) + + #if '~opengl' in spec: + # config_args.extend([ + # "--without-opengl", + # "--without-framework-opengl" + # ]) + + if '+qhull' in spec: + config_args.extend([ + "--with-qhull-includedir=%s" % spec['qhull'].prefix.include, + "--with-qhull-libdir=%s" % spec['qhull'].prefix.lib + ]) + else: + config_args.append("--without-qhull") + + #if '+qrupdate' in spec: + # config_args.extend([ + # "--with-qrupdate-includedir=%s" % spec['qrupdate'].prefix.include, + # "--with-qrupdate-libdir=%s" % spec['qrupdate'].prefix.lib + # ]) + #else: + # config_args.append("--without-qrupdate") + + if '+zlib' in spec: + config_args.extend([ + "--with-z-includedir=%s" % spec['zlib'].prefix.include, + "--with-z-libdir=%s" % spec['zlib'].prefix.lib + ]) + else: + config_args.append("--without-z") + + configure(*config_args) + + make() + make("install") From 1179217334386984211645d508d8e02692297183 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 11 Mar 2016 13:33:56 -0800 Subject: [PATCH 570/910] Add compiler info and a simple libdwarf build to the checks. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index ab379be4867..1bed6b08740 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,10 @@ before_install: script: - . share/spack/setup-env.sh + - spack compilers + - spack config get compilers - spack test + - spack install -v libdwarf notifications: email: From 145390c7f3a9de6e679f56dc9b275973a459de91 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 11 Mar 2016 16:57:37 -0600 Subject: [PATCH 571/910] Add gl2ps and qrupdate packages --- .../repos/builtin/packages/gl2ps/package.py | 18 ++++++++++++++ .../repos/builtin/packages/octave/package.py | 24 +++++++++---------- .../builtin/packages/qrupdate/package.py | 18 ++++++++++++++ 3 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 var/spack/repos/builtin/packages/gl2ps/package.py create mode 100644 var/spack/repos/builtin/packages/qrupdate/package.py diff --git a/var/spack/repos/builtin/packages/gl2ps/package.py b/var/spack/repos/builtin/packages/gl2ps/package.py new file mode 100644 index 00000000000..cb376b3f03b --- /dev/null +++ b/var/spack/repos/builtin/packages/gl2ps/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Gl2ps(Package): + """GL2PS is a C library providing high quality vector output for any + OpenGL application.""" + + homepage = "http://www.geuz.org/gl2ps/" + url = "http://geuz.org/gl2ps/src/gl2ps-1.3.9.tgz" + + version('1.3.9', '377b2bcad62d528e7096e76358f41140') + + depends_on("libpng") + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/octave/package.py b/var/spack/repos/builtin/packages/octave/package.py index 6810da6d98c..99847e1dbee 100644 --- a/var/spack/repos/builtin/packages/octave/package.py +++ b/var/spack/repos/builtin/packages/octave/package.py @@ -50,7 +50,7 @@ class Octave(Package): depends_on('fontconfig', when='+fontconfig') depends_on('freetype', when='+freetype') depends_on('glpk', when='+glpk') - #depends_on('gl2ps', when='+gl2ps') + depends_on('gl2ps', when='+gl2ps') depends_on('gnuplot', when='+gnuplot') depends_on('ImageMagick', when='+magick') depends_on('hdf5', when='+hdf5') @@ -58,10 +58,10 @@ class Octave(Package): depends_on('llvm', when='+llvm') #depends_on('opengl', when='+opengl') depends_on('qhull', when='+qhull') - #depends_on('qrupdate', when='+qrupdate') + depends_on('qrupdate', when='+qrupdate') #depends_on('qscintilla', when='+qscintilla) depends_on('qt', when='+qt') - depends_on('SuiteSparse', when='suitesparse') + depends_on('SuiteSparse', when='+suitesparse') depends_on('zlib', when='+zlib') @@ -72,7 +72,7 @@ def install(self, spec, prefix): # Required dependencies config_args.extend([ - "--with-blas=%s" % spec['blas'].prefix.lib, + "--with-blas=%s" % spec['blas'].prefix.lib, "--with-lapack=%s" % spec['lapack'].prefix.lib ]) @@ -122,7 +122,7 @@ def install(self, spec, prefix): if '+glpk' in spec: config_args.extend([ - "--with-glpk-includedir=%s" % spec['glpk'].prefix.include + "--with-glpk-includedir=%s" % spec['glpk'].prefix.include, "--with-glpk-libdir=%s" % spec['glpk'].prefix.lib ]) else: @@ -160,13 +160,13 @@ def install(self, spec, prefix): else: config_args.append("--without-qhull") - #if '+qrupdate' in spec: - # config_args.extend([ - # "--with-qrupdate-includedir=%s" % spec['qrupdate'].prefix.include, - # "--with-qrupdate-libdir=%s" % spec['qrupdate'].prefix.lib - # ]) - #else: - # config_args.append("--without-qrupdate") + if '+qrupdate' in spec: + config_args.extend([ + "--with-qrupdate-includedir=%s" % spec['qrupdate'].prefix.include, + "--with-qrupdate-libdir=%s" % spec['qrupdate'].prefix.lib + ]) + else: + config_args.append("--without-qrupdate") if '+zlib' in spec: config_args.extend([ diff --git a/var/spack/repos/builtin/packages/qrupdate/package.py b/var/spack/repos/builtin/packages/qrupdate/package.py new file mode 100644 index 00000000000..aff44bb2d80 --- /dev/null +++ b/var/spack/repos/builtin/packages/qrupdate/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Qrupdate(Package): + """qrupdate is a Fortran library for fast updates of QR and + Cholesky decompositions.""" + + homepage = "http://sourceforge.net/projects/qrupdate/" + url = "https://downloads.sourceforge.net/qrupdate/qrupdate-1.1.2.tar.gz" + + version('1.1.2', '6d073887c6e858c24aeda5b54c57a8c4') + + depends_on("openblas") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") From 90f2e40ff92077da1f1b9b4d9aa317b89aa07960 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 11 Mar 2016 23:28:16 -0500 Subject: [PATCH 572/910] Added comment to Emacs. --- var/spack/repos/builtin/packages/emacs/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/emacs/package.py b/var/spack/repos/builtin/packages/emacs/package.py index 09eb05d5a76..caa264857e0 100644 --- a/var/spack/repos/builtin/packages/emacs/package.py +++ b/var/spack/repos/builtin/packages/emacs/package.py @@ -12,6 +12,8 @@ class Emacs(Package): # GTK or other widget library # libtiff, png, etc. # For now, we assume the system provides all that stuff. + # For Ubuntu 14.04 LTS: + # sudo apt-get install libgtk-3-dev libxpm-dev libtiff5-dev libjpeg8-dev libgif-dev libpng12-dev def install(self, spec, prefix): configure('--prefix=%s' % prefix) From 7cd478418d5b8a204a8e6b9a5307ecf71ea83939 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 11 Mar 2016 23:28:36 -0500 Subject: [PATCH 573/910] New version of LAPACK --- var/spack/repos/builtin/packages/netlib-lapack/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py index fb6b99e27c3..741f4af4215 100644 --- a/var/spack/repos/builtin/packages/netlib-lapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -12,6 +12,7 @@ class NetlibLapack(Package): homepage = "http://www.netlib.org/lapack/" url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" + version('3.6.0', 'f2f6c67134e851fe189bb3ca1fbb5101') version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') version('3.4.2', '61bf1a8a4469d4bdb7604f5897179478') version('3.4.1', '44c3869c38c8335c2b9c2a8bb276eb55') From 3383486adc86ba03456aae3703c0176620888e77 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 13 Mar 2016 19:38:05 -0400 Subject: [PATCH 574/910] Fixed typo bug. Made error comment more explicit --- lib/spack/spack/cmd/diy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 2c3a8761ab6..199362d915c 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -75,8 +75,8 @@ def diy(self, args): edit_package(spec.name, spack.repo.first_repo(), None, True) return - if not spec.version.concrete: - tty.die("spack diy spec must have a single, concrete version.") + if not spec.versions.concrete: + tty.die("spack spconfig spec must have a single, concrete version. Did you forget a package version number?") spec.concretize() package = spack.repo.get(spec) From 5c865b9b702695797ab29263fb629b53e56983fb Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 13 Mar 2016 19:42:15 -0400 Subject: [PATCH 575/910] Fixed typo in typo fix. --- lib/spack/spack/cmd/diy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 199362d915c..45f13e44639 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -76,7 +76,7 @@ def diy(self, args): return if not spec.versions.concrete: - tty.die("spack spconfig spec must have a single, concrete version. Did you forget a package version number?") + tty.die("spack diy spec must have a single, concrete version. Did you forget a package version number?") spec.concretize() package = spack.repo.get(spec) From 8b715d9c3fabdf4658ff4c0c500d01f74e7ed64e Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 13 Mar 2016 21:14:41 -0400 Subject: [PATCH 576/910] Update tmux --- var/spack/repos/builtin/packages/tmux/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/tmux/package.py b/var/spack/repos/builtin/packages/tmux/package.py index 23d36db4275..f2067d13665 100644 --- a/var/spack/repos/builtin/packages/tmux/package.py +++ b/var/spack/repos/builtin/packages/tmux/package.py @@ -7,10 +7,11 @@ class Tmux(Package): do a lot more. """ - homepage = "http://tmux.sourceforge.net" - url = "http://downloads.sourceforge.net/project/tmux/tmux/tmux-1.9/tmux-1.9a.tar.gz" + homepage = "http://tmux.github.io" + url = "https://github.com/tmux/tmux/releases/download/2.1/tmux-2.1.tar.gz" version('1.9a', 'b07601711f96f1d260b390513b509a2d') + version('2.1', '74a2855695bccb51b6e301383ad4818c') depends_on('libevent') depends_on('ncurses') From 003fd4d834d3e06ea89c7f3c2fa13241b0730f06 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 14 Mar 2016 04:55:30 -0700 Subject: [PATCH 577/910] Optimize __eq__ and __ne__ in key_ordering - use `is` when possible before calling `_cmp_key()` --- lib/spack/llnl/util/lang.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 1c4d1ed623e..13d301f84e6 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -235,11 +235,11 @@ def setter(name, value): if not has_method(cls, '_cmp_key'): raise TypeError("'%s' doesn't define _cmp_key()." % cls.__name__) - setter('__eq__', lambda s,o: o is not None and s._cmp_key() == o._cmp_key()) + setter('__eq__', lambda s,o: (s is o) or (o is not None and s._cmp_key() == o._cmp_key())) setter('__lt__', lambda s,o: o is not None and s._cmp_key() < o._cmp_key()) setter('__le__', lambda s,o: o is not None and s._cmp_key() <= o._cmp_key()) - setter('__ne__', lambda s,o: o is None or s._cmp_key() != o._cmp_key()) + setter('__ne__', lambda s,o: (s is not o) and (o is None or s._cmp_key() != o._cmp_key())) setter('__gt__', lambda s,o: o is None or s._cmp_key() > o._cmp_key()) setter('__ge__', lambda s,o: o is None or s._cmp_key() >= o._cmp_key()) From 05c761dee9c86faf9ce6d5b98ae57c8737694898 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 14 Mar 2016 04:59:29 -0700 Subject: [PATCH 578/910] Add `package_class` method to spec. - Shouldn't call .package from within things like normalize() and concretize() beacuse spec may be inconsistent. - Add `.package_class` property so that we can get at package metadata without constructing a Package with a Spec. - should be faster than `.package` was, anyway. Use where possible. --- lib/spack/spack/concretize.py | 2 +- lib/spack/spack/repository.py | 9 +++++++-- lib/spack/spack/spec.py | 12 ++++++++++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 8d29a03f932..445ecd8896d 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -238,7 +238,7 @@ def concretize_variants(self, spec): the default variants from the package specification. """ changed = False - for name, variant in spec.package.variants.items(): + for name, variant in spec.package_class.variants.items(): if name not in spec.variants: spec.variants[name] = spack.spec.VariantSpec(name, variant.default) changed = True diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 3c3ba08bcc2..d2fdc937f7a 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -316,6 +316,11 @@ def get(self, spec, new=False): return self.repo_for_pkg(spec).get(spec) + def get_pkg_class(self, pkg_name): + """Find a class for the spec's package and return the class object.""" + return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name) + + @_autospec def dump_provenance(self, spec, path): """Dump provenance information for a spec to a particular path. @@ -550,7 +555,7 @@ def get(self, spec, new=False): key = hash(spec) if new or key not in self._instances: - package_class = self._get_pkg_class(spec.name) + package_class = self.get_pkg_class(spec.name) try: copy = spec.copy() # defensive copy. Package owns its spec. self._instances[key] = package_class(copy) @@ -715,7 +720,7 @@ def _get_pkg_module(self, pkg_name): return self._modules[pkg_name] - def _get_pkg_class(self, pkg_name): + def get_pkg_class(self, pkg_name): """Get the class for the package out of its module. First loads (or fetches from cache) a module for the diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index c045e803655..573e288d174 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -353,7 +353,7 @@ def constrain(self, other): @property def concrete(self): return self.spec._concrete or all( - v in self for v in self.spec.package.variants) + v in self for v in self.spec.package_class.variants) def copy(self): @@ -498,6 +498,14 @@ def package(self): return spack.repo.get(self) + @property + def package_class(self): + """Internal package call gets only the class object for a package. + Use this to just get package metadata. + """ + return spack.repo.get_pkg_class(self.name) + + @property def virtual(self): """Right now, a spec is virtual if no package exists with its name. @@ -1161,7 +1169,7 @@ def validate_names(self): # Ensure that variants all exist. for vname, variant in spec.variants.items(): - if vname not in spec.package.variants: + if vname not in spec.package_class.variants: raise UnknownVariantError(spec.name, vname) From f45b8b1083e5f628dd31fa4b7b873b6df7119d0e Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 14 Mar 2016 05:02:50 -0700 Subject: [PATCH 579/910] Add some tests for packages with multiple virtual dependencies. - Added mock `hypre` package, depends on `lapack` and `blas`. - test cases where some packages provide both `lapack` and `blas`, but others do not. --- lib/spack/spack/test/concretize.py | 29 +++++++++++++- .../builtin.mock/packages/hypre/package.py | 39 +++++++++++++++++++ .../packages/openblas-with-lapack/package.py | 38 ++++++++++++++++++ 3 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 var/spack/repos/builtin.mock/packages/hypre/package.py create mode 100644 var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 07828d8ea6c..f264faf17a2 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -142,6 +142,34 @@ def test_concretize_with_provides_when(self): for spec in spack.repo.providers_for('mpi@3'))) + def test_concretize_two_virtuals(self): + """Test a package with multiple virtual dependencies.""" + s = Spec('hypre').concretize() + + + def test_concretize_two_virtuals_with_one_bound(self): + """Test a package with multiple virtual dependencies and one preset.""" + s = Spec('hypre ^openblas').concretize() + + + def test_concretize_two_virtuals_with_two_bound(self): + """Test a package with multiple virtual dependencies and two of them preset.""" + s = Spec('hypre ^openblas ^netlib-lapack').concretize() + + + def test_concretize_two_virtuals_with_dual_provider(self): + """Test a package with multiple virtual dependencies and force a provider + that provides both.""" + s = Spec('hypre ^openblas-with-lapack').concretize() + + + def test_concretize_two_virtuals_with_dual_provider_and_a_conflict(self): + """Test a package with multiple virtual dependencies and force a provider + that provides both, and another conflicting package that provides one.""" + s = Spec('hypre ^openblas-with-lapack ^netlib-lapack') + self.assertRaises(spack.spec.MultipleProviderError, s.concretize) + + def test_virtual_is_fully_expanded_for_callpath(self): # force dependence on fake "zmpi" by asking for MPI 10.0 spec = Spec('callpath ^mpi@10.0') @@ -281,4 +309,3 @@ def test_find_spec_none(self): Spec('d')), Spec('e')) self.assertEqual(None, find_spec(s['b'], lambda s: '+foo' in s)) - diff --git a/var/spack/repos/builtin.mock/packages/hypre/package.py b/var/spack/repos/builtin.mock/packages/hypre/package.py new file mode 100644 index 00000000000..f69f16d2cc3 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/hypre/package.py @@ -0,0 +1,39 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Hypre(Package): + """Hypre is included here as an example of a package that depends on + both LAPACK and BLAS.""" + homepage = "http://www.openblas.net" + url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + + version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') + + depends_on('lapack') + depends_on('blas') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py b/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py new file mode 100644 index 00000000000..509bfb71e56 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class OpenblasWithLapack(Package): + """Dummy version of OpenBLAS that also provides LAPACK, for testing.""" + homepage = "http://www.openblas.net" + url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + + version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') + + provides('lapack') + provides('blas') + + def install(self, spec, prefix): + pass From f2761270f3c0506a689b484b0e12d7d6e9f4300d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 14 Mar 2016 05:04:01 -0700 Subject: [PATCH 580/910] Make concretization less greedy: add backtracking for virtuals. - `_expand_virtual_packages` now gets a candidate list and will try all the candidates. - Good news: If the first virtual in the list conflicts with something else in the spec, we'll keep trying until we find a good one. - Bad news: Only looks as far as the next normalize(); can't see conflicts further ahead than that if they're inevitable some other virtual expansion. - Refactor `concretize.py` to keep all the nasty spec graph stitching in `spec.py`. This is more similar to before externals support. - `concretize.py` now just returns a list of candidates sorted by ABI compatibility to `_expand_virtual_packages`, and `spec.py` handles testing the candidates. - Refactor the way external paths are handled in `config.py` and `concretize.py`: - previously, `spec_externals` returned spec/path pairs. Now it returns specs with `external` set. Makes code in `concretize.py` more natural. --- lib/spack/spack/concretize.py | 130 ++++++++++++++-------------------- lib/spack/spack/config.py | 17 +++-- lib/spack/spack/spec.py | 129 +++++++++++++++++++++++++++------ 3 files changed, 169 insertions(+), 107 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 445ecd8896d..8083f919828 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -51,10 +51,10 @@ class DefaultConcretizer(object): """ def _valid_virtuals_and_externals(self, spec): - """Returns a list of spec/external-path pairs for both virtuals and externals - that can concretize this spec.""" - # Get a list of candidate packages that could satisfy this spec - packages = [] + """Returns a list of candidate virtual dep providers and external + packages that coiuld be used to concretize a spec.""" + # First construct a list of concrete candidates to replace spec with. + candidates = [spec] if spec.virtual: providers = spack.repo.providers_for(spec) if not providers: @@ -64,96 +64,72 @@ def _valid_virtuals_and_externals(self, spec): if not spec_w_preferred_providers: spec_w_preferred_providers = spec provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name) - packages = sorted(providers, cmp=provider_cmp) - else: - packages = [spec] + candidates = sorted(providers, cmp=provider_cmp) - # For each candidate package, if it has externals add those to the candidates - # if it's not buildable, then only add the externals. - candidates = [] - all_compilers = spack.compilers.all_compilers() - for pkg in packages: - externals = spec_externals(pkg) - buildable = is_spec_buildable(pkg) - if buildable: - candidates.append((pkg, None)) + # For each candidate package, if it has externals, add those to the usable list. + # if it's not buildable, then *only* add the externals. + usable = [] + for cspec in candidates: + if is_spec_buildable(cspec): + usable.append(cspec) + externals = spec_externals(cspec) for ext in externals: - if ext[0].satisfies(spec): - candidates.append(ext) - if not candidates: + if ext.satisfies(spec): + usable.append(ext) + + # If nothing is in the usable list now, it's because we aren't + # allowed to build anything. + if not usable: raise NoBuildError(spec) def cmp_externals(a, b): - if a[0].name != b[0].name: - #We're choosing between different providers. Maintain order from above sort + if a.name != b.name: + # We're choosing between different providers, so + # maintain order from provider sort return candidates.index(a) - candidates.index(b) - result = cmp_specs(a[0], b[0]) + + result = cmp_specs(a, b) if result != 0: return result - if not a[1] and b[1]: - return 1 - if not b[1] and a[1]: - return -1 - return cmp(a[1], b[1]) - candidates = sorted(candidates, cmp=cmp_externals) - return candidates + # prefer external packages to internal packages. + if a.external is None or b.external is None: + return -cmp(a.external, b.external) + else: + return cmp(a.external, b.external) + + usable.sort(cmp=cmp_externals) + return usable - def concretize_virtual_and_external(self, spec): - """From a list of candidate virtual and external packages, concretize to one that - is ABI compatible with the rest of the DAG.""" + def choose_virtual_or_external(self, spec): + """Given a list of candidate virtual and external packages, try to + find one that is most ABI compatible. + """ candidates = self._valid_virtuals_and_externals(spec) if not candidates: - return False + return candidates - # Find the nearest spec in the dag that has a compiler. We'll use that - # spec to test compiler compatibility. - other_spec = find_spec(spec, lambda(x): x.compiler) - if not other_spec: - other_spec = spec.root + # Find the nearest spec in the dag that has a compiler. We'll + # use that spec to calibrate compiler compatibility. + abi_exemplar = find_spec(spec, lambda(x): x.compiler) + if not abi_exemplar: + abi_exemplar = spec.root - # Choose an ABI-compatible candidate, or the first match otherwise. - candidate = None - if other_spec: - candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec)), None) - if not candidate: - # Try a looser ABI matching - candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec, loose=True)), None) - if not candidate: - # No ABI matches. Pick the top choice based on the orignal preferences. - candidate = candidates[0] - candidate_spec = candidate[0] - external = candidate[1] - changed = False + # Make a list including ABI compatibility of specs with the exemplar. + strict = [spack.abi.compatible(c, abi_exemplar) for c in candidates] + loose = [spack.abi.compatible(c, abi_exemplar, loose=True) for c in candidates] + keys = zip(strict, loose, candidates) - # If we're external then trim the dependencies - if external: - if (spec.dependencies): - changed = True - spec.dependencies = DependencyMap() - candidate_spec.dependencies = DependencyMap() + # Sort candidates from most to least compatibility. + # Note: + # 1. We reverse because True > False. + # 2. Sort is stable, so c's keep their order. + keys.sort(key=lambda k:k[:2], reverse=True) - def fequal(candidate_field, spec_field): - return (not candidate_field) or (candidate_field == spec_field) - if (fequal(candidate_spec.name, spec.name) and - fequal(candidate_spec.versions, spec.versions) and - fequal(candidate_spec.compiler, spec.compiler) and - fequal(candidate_spec.architecture, spec.architecture) and - fequal(candidate_spec.dependencies, spec.dependencies) and - fequal(candidate_spec.variants, spec.variants) and - fequal(external, spec.external)): - return changed - - # Refine this spec to the candidate. - if spec.virtual: - spec._replace_with(candidate_spec) - changed = True - if spec._dup(candidate_spec, deps=False, cleardeps=False): - changed = True - spec.external = external - - return changed + # Pull the candidates back out and return them in order + candidates = [c for s,l,c in keys] + return candidates def concretize_version(self, spec): diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index a21dd6dbe1e..6afd69b3ac7 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -539,22 +539,25 @@ def print_section(section): def spec_externals(spec): - """Return a list of spec, directory pairs for each external location for spec""" + """Return a list of external specs (with external directory path filled in), + one for each known external installation.""" allpkgs = get_config('packages') name = spec.name - spec_locations = [] + external_specs = [] pkg_paths = allpkgs.get(name, {}).get('paths', None) if not pkg_paths: return [] - for pkg,path in pkg_paths.iteritems(): - if not spec.satisfies(pkg): - continue + for external_spec, path in pkg_paths.iteritems(): if not path: + # skip entries without paths (avoid creating extra Specs) continue - spec_locations.append( (spack.spec.Spec(pkg), path) ) - return spec_locations + + external_spec = spack.spec.Spec(external_spec, external=path) + if external_spec.satisfies(spec): + external_specs.append(external_spec) + return external_specs def is_spec_buildable(spec): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 573e288d174..d04135860e7 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -418,9 +418,11 @@ def __init__(self, spec_like, *dep_like, **kwargs): # cases we've read them from a file want to assume normal. # This allows us to manipulate specs that Spack doesn't have # package.py files for. - self._normal = kwargs.get('normal', False) + self._normal = kwargs.get('normal', False) self._concrete = kwargs.get('concrete', False) - self.external = None + + # Allow a spec to be constructed with an external path. + self.external = kwargs.get('external', None) # This allows users to construct a spec DAG with literals. # Note that given two specs a and b, Spec(a) copies a, but @@ -794,8 +796,30 @@ def _replace_with(self, concrete): """Replace this virtual spec with a concrete spec.""" assert(self.virtual) for name, dependent in self.dependents.items(): + # remove self from all dependents. del dependent.dependencies[self.name] - dependent._add_dependency(concrete) + + # add the replacement, unless it is already a dep of dependent. + if concrete.name not in dependent.dependencies: + dependent._add_dependency(concrete) + + + def _replace_node(self, replacement): + """Replace this spec with another. + + Connects all dependents of this spec to its replacement, and + disconnects this spec from any dependencies it has. New spec + will have any dependencies the replacement had, and may need + to be normalized. + + """ + for name, dependent in self.dependents.items(): + del dependent.dependencies[self.name] + dependent._add_dependency(replacement) + + for name, dep in self.dependencies.items(): + del dep.dependents[self.name] + del self.dependencies[dep.name] def _expand_virtual_packages(self): @@ -815,18 +839,80 @@ def _expand_virtual_packages(self): this are infrequent, but should implement this before it is a problem. """ + # Make an index of stuff this spec already provides + self_index = ProviderIndex(self.traverse(), restrict=True) + changed = False done = False while not done: done = True for spec in list(self.traverse()): - if spack.concretizer.concretize_virtual_and_external(spec): - done = False + replacement = None + if spec.virtual: + replacement = self._find_provider(spec, self_index) + if replacement: + # TODO: may break if in-place on self but + # shouldn't happen if root is traversed first. + spec._replace_with(replacement) + done=False + break + + if not replacement: + # Get a list of possible replacements in order of preference. + candidates = spack.concretizer.choose_virtual_or_external(spec) + + # Try the replacements in order, skipping any that cause + # satisfiability problems. + for replacement in candidates: + if replacement is spec: + break + + # Replace spec with the candidate and normalize + copy = self.copy() + copy[spec.name]._dup(replacement.copy(deps=False)) + + try: + # If there are duplicate providers or duplicate provider + # deps, consolidate them and merge constraints. + copy.normalize(force=True) + break + except SpecError as e: + # On error, we'll try the next replacement. + continue + + # If replacement is external then trim the dependencies + if replacement.external: + if (spec.dependencies): + changed = True + spec.dependencies = DependencyMap() + replacement.dependencies = DependencyMap() + + # TODO: could this and the stuff in _dup be cleaned up? + def feq(cfield, sfield): + return (not cfield) or (cfield == sfield) + + if replacement is spec or (feq(replacement.name, spec.name) and + feq(replacement.versions, spec.versions) and + feq(replacement.compiler, spec.compiler) and + feq(replacement.architecture, spec.architecture) and + feq(replacement.dependencies, spec.dependencies) and + feq(replacement.variants, spec.variants) and + feq(replacement.external, spec.external)): + continue + + # Refine this spec to the candidate. This uses + # replace_with AND dup so that it can work in + # place. TODO: make this more efficient. + if spec.virtual: + spec._replace_with(replacement) + changed = True + if spec._dup(replacement, deps=False, cleardeps=False): changed = True - # If there are duplicate providers or duplicate provider deps, this - # consolidates them and merge constraints. - changed |= self.normalize(force=True) + self_index.update(spec) + done=False + break + return changed @@ -850,7 +936,7 @@ def concretize(self): force = False while changed: - changes = (self.normalize(force=force), + changes = (self.normalize(force), self._expand_virtual_packages(), self._concretize_helper()) changed = any(changes) @@ -976,8 +1062,8 @@ def _evaluate_dependency_conditions(self, name): def _find_provider(self, vdep, provider_index): """Find provider for a virtual spec in the provider index. - Raise an exception if there is a conflicting virtual - dependency already in this spec. + Raise an exception if there is a conflicting virtual + dependency already in this spec. """ assert(vdep.virtual) providers = provider_index.providers_for(vdep) @@ -1018,17 +1104,14 @@ def _merge_dependency(self, dep, visited, spec_deps, provider_index): """ changed = False - # If it's a virtual dependency, try to find a provider and - # merge that. + # If it's a virtual dependency, try to find an existing + # provider in the spec, and merge that. if dep.virtual: visited.add(dep.name) provider = self._find_provider(dep, provider_index) if provider: dep = provider - else: - # if it's a real dependency, check whether it provides - # something already required in the spec. index = ProviderIndex([dep], restrict=True) for vspec in (v for v in spec_deps.values() if v.virtual): if index.providers_for(vspec): @@ -1125,13 +1208,14 @@ def normalize(self, force=False): # Get all the dependencies into one DependencyMap spec_deps = self.flat_dependencies(copy=False) - # Initialize index of virtual dependency providers - index = ProviderIndex(spec_deps.values(), restrict=True) + # Initialize index of virtual dependency providers if + # concretize didn't pass us one already + provider_index = ProviderIndex(spec_deps.values(), restrict=True) # traverse the package DAG and fill out dependencies according # to package files & their 'when' specs visited = set() - any_change = self._normalize_helper(visited, spec_deps, index) + any_change = self._normalize_helper(visited, spec_deps, provider_index) # If there are deps specified but not visited, they're not # actually deps of this package. Raise an error. @@ -1410,13 +1494,12 @@ def _dup(self, other, **kwargs): Whether deps should be copied too. Set to false to copy a spec but not its dependencies. """ - # We don't count dependencies as changes here changed = True if hasattr(self, 'name'): - changed = (self.name != other.name and self.versions != other.versions and \ - self.architecture != other.architecture and self.compiler != other.compiler and \ - self.variants != other.variants and self._normal != other._normal and \ + changed = (self.name != other.name and self.versions != other.versions and + self.architecture != other.architecture and self.compiler != other.compiler and + self.variants != other.variants and self._normal != other._normal and self.concrete != other.concrete and self.external != other.external) # Local node attributes get copied first. From f9923452b3365e1472a4f1bd63712c57fea0ee0d Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 14 Mar 2016 14:35:48 +0100 Subject: [PATCH 581/910] environment : added machinery to collect modifications to the environment and apply them later --- lib/spack/spack/environment.py | 157 ++++++++++++++++++++++++++++ lib/spack/spack/test/__init__.py | 3 +- lib/spack/spack/test/environment.py | 50 +++++++++ 3 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 lib/spack/spack/environment.py create mode 100644 lib/spack/spack/test/environment.py diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py new file mode 100644 index 00000000000..7d3d7af0de1 --- /dev/null +++ b/lib/spack/spack/environment.py @@ -0,0 +1,157 @@ +import os +import os.path +import collections + + +class SetEnv(object): + def __init__(self, name, value, **kwargs): + self.name = name + self.value = value + for key, value in kwargs.items(): + setattr(self, key, value) + + def execute(self): + os.environ[self.name] = str(self.value) + + +class UnsetEnv(object): + def __init__(self, name, **kwargs): + self.name = name + for key, value in kwargs.items(): + setattr(self, key, value) + + def execute(self): + os.environ.pop(self.name, None) # Avoid throwing if the variable was not set + + +class AppendPath(object): + def __init__(self, name, path, **kwargs): + self.name = name + self.path = path + for key, value in kwargs.items(): + setattr(self, key, value) + + def execute(self): + environment_value = os.environ.get(self.name, '') + directories = environment_value.split(':') if environment_value else [] + # TODO : Check if this is a valid directory name + directories.append(os.path.normpath(self.path)) + os.environ[self.name] = ':'.join(directories) + + +class PrependPath(object): + def __init__(self, name, path, **kwargs): + self.name = name + self.path = path + for key, value in kwargs.items(): + setattr(self, key, value) + + def execute(self): + environment_value = os.environ.get(self.name, '') + directories = environment_value.split(':') if environment_value else [] + # TODO : Check if this is a valid directory name + directories = [os.path.normpath(self.path)] + directories + os.environ[self.name] = ':'.join(directories) + + +class RemovePath(object): + def __init__(self, name, path, **kwargs): + self.name = name + self.path = path + for key, value in kwargs.items(): + setattr(self, key, value) + + def execute(self): + environment_value = os.environ.get(self.name, '') + directories = environment_value.split(':') if environment_value else [] + directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.path)] + os.environ[self.name] = ':'.join(directories) + + +class EnvironmentModifications(object): + """ + Keeps track of requests to modify the current environment + """ + + def __init__(self): + self.env_modifications = [] + + def __iter__(self): + return iter(self.env_modifications) + + def set_env(self, name, value, **kwargs): + """ + Stores in the current object a request to set an environment variable + + Args: + name: name of the environment variable to be set + value: value of the environment variable + """ + item = SetEnv(name, value, **kwargs) + self.env_modifications.append(item) + + def unset_env(self, name, **kwargs): + """ + Stores in the current object a request to unset an environment variable + + Args: + name: name of the environment variable to be set + """ + item = UnsetEnv(name, **kwargs) + self.env_modifications.append(item) + + def append_path(self, name, path, **kwargs): + """ + Stores in the current object a request to append a path to a path list + + Args: + name: name of the path list in the environment + path: path to be appended + """ + item = AppendPath(name, path, **kwargs) + self.env_modifications.append(item) + + def prepend_path(self, name, path, **kwargs): + """ + Same as `append_path`, but the path is pre-pended + + Args: + name: name of the path list in the environment + path: path to be pre-pended + """ + item = PrependPath(name, path, **kwargs) + self.env_modifications.append(item) + + def remove_path(self, name, path, **kwargs): + """ + Stores in the current object a request to remove a path from a path list + + Args: + name: name of the path list in the environment + path: path to be removed + """ + item = RemovePath(name, path, **kwargs) + self.env_modifications.append(item) + + +def validate_environment_modifications(env): + modifications = collections.defaultdict(list) + for item in env: + modifications[item.name].append(item) + return modifications + + +def apply_environment_modifications(env): + """ + Modifies the current environment according to the request in env + + Args: + env: object storing modifications to the environment + """ + modifications = validate_environment_modifications(env) + + # Cycle over the environment variables that will be modified + for variable, actions in modifications.items(): + # Execute all the actions in the order they were issued + for x in actions: + x.execute() diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index d5d8b647650..cd842561e6c 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -66,7 +66,8 @@ 'database', 'namespace_trie', 'yaml', - 'sbang'] + 'sbang', + 'environment'] def list_tests(): diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py new file mode 100644 index 00000000000..dff3863d322 --- /dev/null +++ b/lib/spack/spack/test/environment.py @@ -0,0 +1,50 @@ +import unittest +import os +from spack.environment import EnvironmentModifications, apply_environment_modifications + + +class EnvironmentTest(unittest.TestCase): + def setUp(self): + os.environ.clear() + os.environ['UNSET_ME'] = 'foo' + os.environ['EMPTY_PATH_LIST'] = '' + os.environ['PATH_LIST'] = '/path/second:/path/third' + os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' + + def test_set_env(self): + env = EnvironmentModifications() + env.set_env('A', 'dummy value') + env.set_env('B', 3) + apply_environment_modifications(env) + self.assertEqual('dummy value', os.environ['A']) + self.assertEqual(str(3), os.environ['B']) + + def test_unset_env(self): + env = EnvironmentModifications() + self.assertEqual('foo', os.environ['UNSET_ME']) + env.unset_env('UNSET_ME') + apply_environment_modifications(env) + self.assertRaises(KeyError, os.environ.__getitem__, 'UNSET_ME') + + def test_path_manipulation(self): + env = EnvironmentModifications() + + env.append_path('PATH_LIST', '/path/last') + env.prepend_path('PATH_LIST', '/path/first') + + env.append_path('EMPTY_PATH_LIST', '/path/middle') + env.append_path('EMPTY_PATH_LIST', '/path/last') + env.prepend_path('EMPTY_PATH_LIST', '/path/first') + + env.append_path('NEWLY_CREATED_PATH_LIST', '/path/middle') + env.append_path('NEWLY_CREATED_PATH_LIST', '/path/last') + env.prepend_path('NEWLY_CREATED_PATH_LIST', '/path/first') + + env.remove_path('REMOVE_PATH_LIST', '/remove/this') + env.remove_path('REMOVE_PATH_LIST', '/duplicate/') + + apply_environment_modifications(env) + self.assertEqual('/path/first:/path/second:/path/third:/path/last', os.environ['PATH_LIST']) + self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST']) + self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST']) + self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST']) From 0d9a6d3c25de2b8490622c49362b09f706459bc0 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 14 Mar 2016 14:19:30 -0500 Subject: [PATCH 582/910] Updates to qrupdate --- .../repos/builtin/packages/octave/package.py | 17 +++++++++-------- .../repos/builtin/packages/qrupdate/package.py | 10 +++++----- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/var/spack/repos/builtin/packages/octave/package.py b/var/spack/repos/builtin/packages/octave/package.py index 99847e1dbee..38b355159dd 100644 --- a/var/spack/repos/builtin/packages/octave/package.py +++ b/var/spack/repos/builtin/packages/octave/package.py @@ -12,6 +12,7 @@ class Octave(Package): version('4.0.0' , 'a69f8320a4f20a8480c1b278b1adb799') + # Variants variant('readline', default=True) variant('arpack', default=False) variant('curl', default=False) @@ -45,7 +46,7 @@ class Octave(Package): # Optional dependencies depends_on('arpack', when='+arpack') depends_on('curl', when='+curl') - depends_on('fftw@3', when='+fftw') + depends_on('fftw', when='+fftw') depends_on('fltk', when='+fltk') depends_on('fontconfig', when='+fontconfig') depends_on('freetype', when='+freetype') @@ -56,10 +57,10 @@ class Octave(Package): depends_on('hdf5', when='+hdf5') depends_on('jdk', when='+jdk') depends_on('llvm', when='+llvm') - #depends_on('opengl', when='+opengl') + #depends_on('opengl', when='+opengl') # TODO: add package depends_on('qhull', when='+qhull') depends_on('qrupdate', when='+qrupdate') - #depends_on('qscintilla', when='+qscintilla) + #depends_on('qscintilla', when='+qscintilla) # TODO: add package depends_on('qt', when='+qt') depends_on('SuiteSparse', when='+suitesparse') depends_on('zlib', when='+zlib') @@ -146,11 +147,11 @@ def install(self, spec, prefix): "--with-java-libdir=%s" % spec['jdk'].prefix.lib ]) - #if '~opengl' in spec: - # config_args.extend([ - # "--without-opengl", - # "--without-framework-opengl" - # ]) + if '~opengl' in spec: + config_args.extend([ + "--without-opengl", + "--without-framework-opengl" + ]) if '+qhull' in spec: config_args.extend([ diff --git a/var/spack/repos/builtin/packages/qrupdate/package.py b/var/spack/repos/builtin/packages/qrupdate/package.py index aff44bb2d80..5374d02c971 100644 --- a/var/spack/repos/builtin/packages/qrupdate/package.py +++ b/var/spack/repos/builtin/packages/qrupdate/package.py @@ -9,10 +9,10 @@ class Qrupdate(Package): version('1.1.2', '6d073887c6e858c24aeda5b54c57a8c4') - depends_on("openblas") + depends_on("blas") + depends_on("lapack") def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") + # Build static and dynamic libraries + make("lib", "solib") + make("install", "PREFIX=%s" % prefix) From f20247ae55362424d1d5543840a21142017661af Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Mar 2016 10:08:54 +0100 Subject: [PATCH 583/910] environment : refactoreded set_compiler_environment_variables --- lib/spack/spack/build_environment.py | 38 +++++++++++++++------------- lib/spack/spack/environment.py | 17 ++++++++++++- lib/spack/spack/test/database.py | 1 + lib/spack/spack/test/environment.py | 9 +++++++ 4 files changed, 47 insertions(+), 18 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 392ba7ea4d5..e91a8a19977 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -35,6 +35,7 @@ import spack from llnl.util.filesystem import * +from spack.environment import EnvironmentModifications, apply_environment_modifications from spack.util.environment import * from spack.util.executable import Executable, which @@ -83,30 +84,32 @@ def __call__(self, *args, **kwargs): def set_compiler_environment_variables(pkg): - assert(pkg.spec.concrete) - compiler = pkg.compiler - + assert pkg.spec.concrete # Set compiler variables used by CMake and autotools - assert all(key in pkg.compiler.link_paths - for key in ('cc', 'cxx', 'f77', 'fc')) + assert all(key in pkg.compiler.link_paths for key in ('cc', 'cxx', 'f77', 'fc')) + # Populate an object with the list of environment modifications + # and return it + # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc. + env = EnvironmentModifications() link_dir = spack.build_env_path - os.environ['CC'] = join_path(link_dir, pkg.compiler.link_paths['cc']) - os.environ['CXX'] = join_path(link_dir, pkg.compiler.link_paths['cxx']) - os.environ['F77'] = join_path(link_dir, pkg.compiler.link_paths['f77']) - os.environ['FC'] = join_path(link_dir, pkg.compiler.link_paths['fc']) - + env.set_env('CC', join_path(link_dir, pkg.compiler.link_paths['cc'])) + env.set_env('CXX', join_path(link_dir, pkg.compiler.link_paths['cxx'])) + env.set_env('F77', join_path(link_dir, pkg.compiler.link_paths['f77'])) + env.set_env('FC', join_path(link_dir, pkg.compiler.link_paths['fc'])) # Set SPACK compiler variables so that our wrapper knows what to call + compiler = pkg.compiler if compiler.cc: - os.environ['SPACK_CC'] = compiler.cc + env.set_env('SPACK_CC', compiler.cc) if compiler.cxx: - os.environ['SPACK_CXX'] = compiler.cxx + env.set_env('SPACK_CXX', compiler.cxx) if compiler.f77: - os.environ['SPACK_F77'] = compiler.f77 + env.set_env('SPACK_F77', compiler.f77) if compiler.fc: - os.environ['SPACK_FC'] = compiler.fc + env.set_env('SPACK_FC', compiler.fc) - os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) + env.set_env('SPACK_COMPILER_SPEC', str(pkg.spec.compiler)) + return env def set_build_environment_variables(pkg): @@ -264,9 +267,10 @@ def parent_class_modules(cls): def setup_package(pkg): """Execute all environment setup routines.""" - set_compiler_environment_variables(pkg) + env = EnvironmentModifications() + env.extend(set_compiler_environment_variables(pkg)) + apply_environment_modifications(env) set_build_environment_variables(pkg) - # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from # an existing class like spack.repos.original.libelf.Libelf, diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 7d3d7af0de1..3b73f1c7a2a 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -73,12 +73,24 @@ class EnvironmentModifications(object): Keeps track of requests to modify the current environment """ - def __init__(self): + def __init__(self, other=None): self.env_modifications = [] + if other is not None: + self._check_other(other) + self.env_modifications.extend(other.env_modifications) def __iter__(self): return iter(self.env_modifications) + def extend(self, other): + self._check_other(other) + self.env_modifications.extend(other.env_modifications) + + @staticmethod + def _check_other(other): + if not isinstance(other, EnvironmentModifications): + raise TypeError('other must be an instance of EnvironmentModifications') + def set_env(self, name, value, **kwargs): """ Stores in the current object a request to set an environment variable @@ -138,6 +150,9 @@ def validate_environment_modifications(env): modifications = collections.defaultdict(list) for item in env: modifications[item.name].append(item) + # TODO : once we organized the modifications into a dictionary that maps an environment variable + # TODO : to a list of action to be done on it, we may easily spot inconsistencies and warn the user if + # TODO : something suspicious is happening return modifications diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 9a57e1f03ea..ce6e8a0552b 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -26,6 +26,7 @@ These tests check the database is functioning properly, both in memory and in its file """ +import os.path import multiprocessing import shutil import tempfile diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index dff3863d322..17061c8fd0e 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -48,3 +48,12 @@ def test_path_manipulation(self): self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST']) self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST']) self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST']) + + def test_extra_arguments(self): + env = EnvironmentModifications() + env.set_env('A', 'dummy value', who='Pkg1') + apply_environment_modifications(env) + self.assertEqual('dummy value', os.environ['A']) + + def test_copy(self): + pass From bcea1df01ce145aea5e49eaaceb7a063b44ddf80 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Mar 2016 10:49:33 +0100 Subject: [PATCH 584/910] environment : refactoreded set_build_environment_variables --- lib/spack/spack/build_environment.py | 52 ++++++++++--------- lib/spack/spack/environment.py | 3 ++ .../builtin/packages/mvapich2/package.py | 2 +- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index e91a8a19977..770e191ac90 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -35,7 +35,7 @@ import spack from llnl.util.filesystem import * -from spack.environment import EnvironmentModifications, apply_environment_modifications +from spack.environment import EnvironmentModifications, apply_environment_modifications, concatenate_paths from spack.util.environment import * from spack.util.executable import Executable, which @@ -127,44 +127,45 @@ def set_build_environment_variables(pkg): # handled by putting one in the /case-insensitive # directory. Add that to the path too. env_paths = [] - def add_env_path(path): - env_paths.append(path) - ci = join_path(path, 'case-insensitive') - if os.path.isdir(ci): env_paths.append(ci) - add_env_path(spack.build_env_path) - add_env_path(join_path(spack.build_env_path, pkg.compiler.name)) + for item in [spack.build_env_path, join_path(spack.build_env_path, pkg.compiler.name)]: + env_paths.append(item) + ci = join_path(item, 'case-insensitive') + if os.path.isdir(ci): + env_paths.append(ci) - path_put_first("PATH", env_paths) - path_set(SPACK_ENV_PATH, env_paths) + env = EnvironmentModifications() + for item in reversed(env_paths): + env.prepend_path('PATH', item) + env.set_env(SPACK_ENV_PATH, concatenate_paths(env_paths)) - # Prefixes of all of the package's dependencies go in - # SPACK_DEPENDENCIES + # Prefixes of all of the package's dependencies go in SPACK_DEPENDENCIES dep_prefixes = [d.prefix for d in pkg.spec.traverse(root=False)] - path_set(SPACK_DEPENDENCIES, dep_prefixes) + env.set_env(SPACK_DEPENDENCIES, concatenate_paths(dep_prefixes)) + env.set_env('CMAKE_PREFIX_PATH', concatenate_paths(dep_prefixes)) # Add dependencies to CMAKE_PREFIX_PATH # Install prefix - os.environ[SPACK_PREFIX] = pkg.prefix + env.set_env(SPACK_PREFIX, pkg.prefix) # Install root prefix - os.environ[SPACK_INSTALL] = spack.install_path + env.set_env(SPACK_INSTALL, spack.install_path) # Remove these vars from the environment during build because they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. - pop_keys(os.environ, "LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH") + env.unset_env('LD_LIBRARY_PATH') + env.unset_env('LD_RUN_PATH') + env.unset_env('DYLD_LIBRARY_PATH') # Add bin directories from dependencies to the PATH for the build. - bin_dirs = ['%s/bin' % prefix for prefix in dep_prefixes] - path_put_first('PATH', [bin for bin in bin_dirs if os.path.isdir(bin)]) + bin_dirs = reversed(filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes])) + for item in bin_dirs: + env.prepend_path('PATH', item) # Working directory for the spack command itself, for debug logs. if spack.debug: - os.environ[SPACK_DEBUG] = "TRUE" - os.environ[SPACK_SHORT_SPEC] = pkg.spec.short_spec - os.environ[SPACK_DEBUG_LOG_DIR] = spack.spack_working_dir - - # Add dependencies to CMAKE_PREFIX_PATH - path_set("CMAKE_PREFIX_PATH", dep_prefixes) + env.set_env(SPACK_DEBUG, 'TRUE') + env.set_env(SPACK_SHORT_SPEC, pkg.spec.short_spec) + env.set_env(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir) # Add any pkgconfig directories to PKG_CONFIG_PATH pkg_config_dirs = [] @@ -173,8 +174,9 @@ def add_env_path(path): pcdir = join_path(p, libdir, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) - path_set("PKG_CONFIG_PATH", pkg_config_dirs) + env.set_env('PKG_CONFIG_PATH', concatenate_paths(pkg_config_dirs)) + return env def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. @@ -269,8 +271,8 @@ def setup_package(pkg): """Execute all environment setup routines.""" env = EnvironmentModifications() env.extend(set_compiler_environment_variables(pkg)) + env.extend(set_build_environment_variables(pkg)) apply_environment_modifications(env) - set_build_environment_variables(pkg) # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from # an existing class like spack.repos.original.libelf.Libelf, diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 3b73f1c7a2a..18fef1ef12c 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -146,6 +146,9 @@ def remove_path(self, name, path, **kwargs): self.env_modifications.append(item) +def concatenate_paths(paths): + return ':'.join(str(item) for item in paths) + def validate_environment_modifications(env): modifications = collections.defaultdict(list) for item in env: diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index af5ed1b0888..e4e95f92af7 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -123,7 +123,7 @@ def set_network_type(self, spec, configure_args): count += 1 if count > 1: raise RuntimeError('network variants are mutually exclusive (only one can be selected at a time)') - + network_options = [] # From here on I can suppose that only one variant has been selected if self.enabled(Mvapich2.PSM) in spec: network_options = ["--with-device=ch3:psm"] From c85888eb5763d453af7b7255b6d8c3461082362f Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Mar 2016 13:36:41 +0100 Subject: [PATCH 585/910] package : added `environment_modifications` --- lib/spack/spack/build_environment.py | 7 +++--- lib/spack/spack/package.py | 4 ++++ .../repos/builtin/packages/mpich/package.py | 16 +++++++++----- .../packages/netlib-scalapack/package.py | 3 +-- .../repos/builtin/packages/openmpi/package.py | 14 +++++++----- .../repos/builtin/packages/python/package.py | 20 +++++++++-------- .../repos/builtin/packages/qt/package.py | 12 +++++----- .../repos/builtin/packages/ruby/package.py | 22 +++++++++++-------- 8 files changed, 58 insertions(+), 40 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 770e191ac90..d321a0e495e 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -178,6 +178,7 @@ def set_build_environment_variables(pkg): return env + def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. @@ -272,7 +273,6 @@ def setup_package(pkg): env = EnvironmentModifications() env.extend(set_compiler_environment_variables(pkg)) env.extend(set_build_environment_variables(pkg)) - apply_environment_modifications(env) # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from # an existing class like spack.repos.original.libelf.Libelf, @@ -284,8 +284,9 @@ def setup_package(pkg): # Allow dependencies to set up environment as well. for dep_spec in pkg.spec.traverse(root=False): - dep_spec.package.setup_dependent_environment( - pkg.module, dep_spec, pkg.spec) + env.extend(dep_spec.package.environment_modifications(pkg.module, dep_spec, pkg.spec)) + dep_spec.package.setup_dependent_environment(pkg.module, dep_spec, pkg.spec) + apply_environment_modifications(env) def fork(pkg, function): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 696adaf8965..224d7f8f4b9 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -63,6 +63,7 @@ import spack.url import spack.util.web import spack.fetch_strategy as fs +from spack.environment import EnvironmentModifications from spack.version import * from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive, extension @@ -983,6 +984,9 @@ def module(self): fromlist=[self.__class__.__name__]) + def environment_modifications(self, module, spec, dependent_spec): + return EnvironmentModifications() + def setup_dependent_environment(self, module, spec, dependent_spec): """Called before the install() method of dependents. diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index e2b3654c19a..c85e8febf39 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -46,14 +46,18 @@ class Mpich(Package): provides('mpi@:3.0', when='@3:') provides('mpi@:1.3', when='@1:') + def environment_modifications(self, module, spec, dependent_spec): + env = super(Mpich, self).environment_modifications(module, spec, dependent_spec) + env.set_env('MPICH_CC', os.environ['CC']) + env.set_env('MPICH_CXX', os.environ['CXX']) + env.set_env('MPICH_F77', os.environ['F77']) + env.set_env('MPICH_F90', os.environ['FC']) + env.set_env('MPICH_FC', os.environ['FC']) + return env + def setup_dependent_environment(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" - os.environ['MPICH_CC'] = os.environ['CC'] - os.environ['MPICH_CXX'] = os.environ['CXX'] - os.environ['MPICH_F77'] = os.environ['F77'] - os.environ['MPICH_F90'] = os.environ['FC'] - os.environ['MPICH_FC'] = os.environ['FC'] - + # FIXME : is this necessary ? Shouldn't this be part of a contract with MPI providers? module.mpicc = join_path(self.prefix.bin, 'mpicc') def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 22d538560e8..6dbf3674756 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -46,5 +46,4 @@ def setup_dependent_environment(self, module, spec, dependent_spec): spec['scalapack'].fc_link = '-L%s -lscalapack' % spec['scalapack'].prefix.lib spec['scalapack'].cc_link = spec['scalapack'].fc_link - spec['scalapack'].libraries = [join_path(spec['scalapack'].prefix.lib, - 'libscalapack%s' % lib_suffix)] + spec['scalapack'].libraries = [join_path(spec['scalapack'].prefix.lib, 'libscalapack%s' % lib_suffix)] diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index e4484af8c5e..83a3fe7a4f3 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -41,12 +41,14 @@ class Openmpi(Package): def url_for_version(self, version): return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) - def setup_dependent_environment(self, module, spec, dep_spec): - """For dependencies, make mpicc's use spack wrapper.""" - os.environ['OMPI_CC'] = 'cc' - os.environ['OMPI_CXX'] = 'c++' - os.environ['OMPI_FC'] = 'f90' - os.environ['OMPI_F77'] = 'f77' + def environment_modifications(self, module, spec, dependent_spec): + env = super(Openmpi, self).environment_modifications(module, spec, dependent_spec) + # FIXME : the compilers should point to the current wrappers, not to generic cc etc. + env.set_env('OMPI_CC', 'cc') + env.set_env('OMPI_CXX', 'c++') + env.set_env('OMPI_FC', 'f90') + env.set_env('OMPI_F77', 'f77') + return env def install(self, spec, prefix): config_args = ["--prefix=%s" % prefix, diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index dd240d1ea0a..39ced0a1200 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -90,6 +90,17 @@ def site_packages_dir(self): return os.path.join(self.python_lib_dir, 'site-packages') + def environment_modifications(self, module, spec, dependent_spec): + env = super(Python, self).environment_modifications(module, spec, dependent_spec) + # Set PYTHONPATH to include site-packages dir for the + # extension and any other python extensions it depends on. + python_paths = [] + for d in ext_spec.traverse(): + if d.package.extends(self.spec): + python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) + env.set_env['PYTHONPATH'] = ':'.join(python_paths) + + def setup_dependent_environment(self, module, spec, ext_spec): """Called before python modules' install() methods. @@ -111,15 +122,6 @@ def setup_dependent_environment(self, module, spec, ext_spec): # Make the site packages directory if it does not exist already. mkdirp(module.site_packages_dir) - # Set PYTHONPATH to include site-packages dir for the - # extension and any other python extensions it depends on. - python_paths = [] - for d in ext_spec.traverse(): - if d.package.extends(self.spec): - python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) - os.environ['PYTHONPATH'] = ':'.join(python_paths) - - # ======================================================================== # Handle specifics of activating and deactivating python modules. # ======================================================================== diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 91afa420c17..373623cd959 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -52,11 +52,13 @@ class Qt(Package): depends_on("mesa", when='@4:+mesa') depends_on("libxcb") - - def setup_dependent_environment(self, module, spec, dep_spec): - """Dependencies of Qt find it using the QTDIR environment variable.""" - os.environ['QTDIR'] = self.prefix - + def environment_modifications(self, module, spec, dep_spec): + """ + Dependencies of Qt find it using the QTDIR environment variable + """ + env = super(Qt, self).environment_modifications(module, spec, dep_spec) + env.set_env['QTDIR'] = self.prefix + return env def patch(self): if self.spec.satisfies('@4'): diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py index 6b6242362cd..9ec0afa2683 100644 --- a/var/spack/repos/builtin/packages/ruby/package.py +++ b/var/spack/repos/builtin/packages/ruby/package.py @@ -15,10 +15,21 @@ class Ruby(Package): def install(self, spec, prefix): configure("--prefix=%s" % prefix) - make() make("install") + def environment_modifications(self, module, spec, ext_spec): + env = super(Ruby, self).environment_modifications(module, spec, ext_spec) + # Set GEM_PATH to include dependent gem directories + ruby_paths = [] + for d in ext_spec.traverse(): + if d.package.extends(self.spec): + ruby_paths.append(d.prefix) + env.set_env('GEM_PATH', concatenate_paths(ruby_paths)) + # The actual installation path for this gem + env.set_env('GEM_HOME', ext_spec.prefix) + return env + def setup_dependent_environment(self, module, spec, ext_spec): """Called before ruby modules' install() methods. Sets GEM_HOME and GEM_PATH to values appropriate for the package being built. @@ -31,11 +42,4 @@ def setup_dependent_environment(self, module, spec, ext_spec): module.ruby = Executable(join_path(spec.prefix.bin, 'ruby')) module.gem = Executable(join_path(spec.prefix.bin, 'gem')) - # Set GEM_PATH to include dependent gem directories - ruby_paths = [] - for d in ext_spec.traverse(): - if d.package.extends(self.spec): - ruby_paths.append(d.prefix) - os.environ['GEM_PATH'] = ':'.join(ruby_paths) - # The actual installation path for this gem - os.environ['GEM_HOME'] = ext_spec.prefix + From 572cb93bf8131d222d2d08bca13fd9de6fded1f4 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Mar 2016 14:05:30 +0100 Subject: [PATCH 586/910] package : renamed `setup_dependent_environment` to `module_modifications` --- lib/spack/spack/build_environment.py | 4 ++-- lib/spack/spack/package.py | 4 ++-- var/spack/repos/builtin/packages/mpich/package.py | 6 +++--- .../builtin/packages/netlib-scalapack/package.py | 2 +- var/spack/repos/builtin/packages/openmpi/package.py | 4 ++-- var/spack/repos/builtin/packages/python/package.py | 10 ++++------ var/spack/repos/builtin/packages/qt/package.py | 7 ++----- var/spack/repos/builtin/packages/ruby/package.py | 13 ++++++------- 8 files changed, 22 insertions(+), 28 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index d321a0e495e..e86a7c413a9 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -284,8 +284,8 @@ def setup_package(pkg): # Allow dependencies to set up environment as well. for dep_spec in pkg.spec.traverse(root=False): - env.extend(dep_spec.package.environment_modifications(pkg.module, dep_spec, pkg.spec)) - dep_spec.package.setup_dependent_environment(pkg.module, dep_spec, pkg.spec) + dep_spec.package.module_modifications(pkg.module, dep_spec, pkg.spec) + env.extend(dep_spec.package.environment_modifications(pkg.spec)) apply_environment_modifications(env) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 224d7f8f4b9..c1a5c912be7 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -984,10 +984,10 @@ def module(self): fromlist=[self.__class__.__name__]) - def environment_modifications(self, module, spec, dependent_spec): + def environment_modifications(self, dependent_spec): return EnvironmentModifications() - def setup_dependent_environment(self, module, spec, dependent_spec): + def module_modifications(self, module, spec, dependent_spec): """Called before the install() method of dependents. Default implementation does nothing, but this can be diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index c85e8febf39..d298981c925 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -46,8 +46,8 @@ class Mpich(Package): provides('mpi@:3.0', when='@3:') provides('mpi@:1.3', when='@1:') - def environment_modifications(self, module, spec, dependent_spec): - env = super(Mpich, self).environment_modifications(module, spec, dependent_spec) + def environment_modifications(self, dependent_spec): + env = super(Mpich, self).environment_modifications(dependent_spec) env.set_env('MPICH_CC', os.environ['CC']) env.set_env('MPICH_CXX', os.environ['CXX']) env.set_env('MPICH_F77', os.environ['F77']) @@ -55,7 +55,7 @@ def environment_modifications(self, module, spec, dependent_spec): env.set_env('MPICH_FC', os.environ['FC']) return env - def setup_dependent_environment(self, module, spec, dep_spec): + def module_modifications(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" # FIXME : is this necessary ? Shouldn't this be part of a contract with MPI providers? module.mpicc = join_path(self.prefix.bin, 'mpicc') diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 6dbf3674756..ecdea464420 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -40,7 +40,7 @@ def install(self, spec, prefix): make() make("install") - def setup_dependent_environment(self, module, spec, dependent_spec): + def module_modifications(self, module, spec, dependent_spec): # TODO treat OS that are not Linux... lib_suffix = '.so' if '+shared' in spec['scalapack'] else '.a' diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 83a3fe7a4f3..3a141704576 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -41,8 +41,8 @@ class Openmpi(Package): def url_for_version(self, version): return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) - def environment_modifications(self, module, spec, dependent_spec): - env = super(Openmpi, self).environment_modifications(module, spec, dependent_spec) + def environment_modifications(self, dependent_spec): + env = super(Openmpi, self).environment_modifications(dependent_spec) # FIXME : the compilers should point to the current wrappers, not to generic cc etc. env.set_env('OMPI_CC', 'cc') env.set_env('OMPI_CXX', 'c++') diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 39ced0a1200..307cec726b6 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -89,19 +89,17 @@ def python_include_dir(self): def site_packages_dir(self): return os.path.join(self.python_lib_dir, 'site-packages') - - def environment_modifications(self, module, spec, dependent_spec): - env = super(Python, self).environment_modifications(module, spec, dependent_spec) + def environment_modifications(self, extension_spec): + env = super(Python, self).environment_modifications(extension_spec) # Set PYTHONPATH to include site-packages dir for the # extension and any other python extensions it depends on. python_paths = [] - for d in ext_spec.traverse(): + for d in extension_spec.traverse(): if d.package.extends(self.spec): python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) env.set_env['PYTHONPATH'] = ':'.join(python_paths) - - def setup_dependent_environment(self, module, spec, ext_spec): + def module_modifications(self, module, spec, ext_spec): """Called before python modules' install() methods. In most cases, extensions will only need to have one line:: diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 373623cd959..8391ded3e0b 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -52,11 +52,8 @@ class Qt(Package): depends_on("mesa", when='@4:+mesa') depends_on("libxcb") - def environment_modifications(self, module, spec, dep_spec): - """ - Dependencies of Qt find it using the QTDIR environment variable - """ - env = super(Qt, self).environment_modifications(module, spec, dep_spec) + def environment_modifications(self, dependent_spec): + env = super(Qt, self).environment_modifications(dependent_spec) env.set_env['QTDIR'] = self.prefix return env diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py index 9ec0afa2683..9caea30ef46 100644 --- a/var/spack/repos/builtin/packages/ruby/package.py +++ b/var/spack/repos/builtin/packages/ruby/package.py @@ -1,6 +1,5 @@ from spack import * -import spack -import os + class Ruby(Package): """A dynamic, open source programming language with a focus on @@ -18,19 +17,19 @@ def install(self, spec, prefix): make() make("install") - def environment_modifications(self, module, spec, ext_spec): - env = super(Ruby, self).environment_modifications(module, spec, ext_spec) + def environment_modifications(self, extension_spec): + env = super(Ruby, self).environment_modifications(extension_spec) # Set GEM_PATH to include dependent gem directories ruby_paths = [] - for d in ext_spec.traverse(): + for d in extension_spec.traverse(): if d.package.extends(self.spec): ruby_paths.append(d.prefix) env.set_env('GEM_PATH', concatenate_paths(ruby_paths)) # The actual installation path for this gem - env.set_env('GEM_HOME', ext_spec.prefix) + env.set_env('GEM_HOME', extension_spec.prefix) return env - def setup_dependent_environment(self, module, spec, ext_spec): + def module_modifications(self, module, spec, ext_spec): """Called before ruby modules' install() methods. Sets GEM_HOME and GEM_PATH to values appropriate for the package being built. From cc3d9f4eb751fe028db5a074fc7bc52fbe0a962a Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Mar 2016 15:09:35 +0100 Subject: [PATCH 587/910] environment : added test, modified docs --- lib/spack/spack/environment.py | 61 ++++++++++++++++++++--------- lib/spack/spack/package.py | 45 +++++++++++++-------- lib/spack/spack/test/environment.py | 10 ++++- lib/spack/spack/util/environment.py | 8 +--- 4 files changed, 80 insertions(+), 44 deletions(-) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 18fef1ef12c..5a68e10c997 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -3,33 +3,40 @@ import collections -class SetEnv(object): - def __init__(self, name, value, **kwargs): - self.name = name - self.value = value +class AttributeHolder(object): + """ + Policy that permits to store any kind of attribute on self. The attributes must be passed as key/value pairs + during the initialization of the instance. + """ + def __init__(self, **kwargs): for key, value in kwargs.items(): setattr(self, key, value) + +class SetEnv(AttributeHolder): + def __init__(self, name, value, **kwargs): + super(SetEnv, self).__init__(**kwargs) + self.name = name + self.value = value + def execute(self): os.environ[self.name] = str(self.value) -class UnsetEnv(object): +class UnsetEnv(AttributeHolder): def __init__(self, name, **kwargs): + super(UnsetEnv, self).__init__(**kwargs) self.name = name - for key, value in kwargs.items(): - setattr(self, key, value) def execute(self): os.environ.pop(self.name, None) # Avoid throwing if the variable was not set -class AppendPath(object): +class AppendPath(AttributeHolder): def __init__(self, name, path, **kwargs): + super(AppendPath, self).__init__(**kwargs) self.name = name self.path = path - for key, value in kwargs.items(): - setattr(self, key, value) def execute(self): environment_value = os.environ.get(self.name, '') @@ -39,12 +46,11 @@ def execute(self): os.environ[self.name] = ':'.join(directories) -class PrependPath(object): +class PrependPath(AttributeHolder): def __init__(self, name, path, **kwargs): + super(PrependPath, self).__init__(**kwargs) self.name = name self.path = path - for key, value in kwargs.items(): - setattr(self, key, value) def execute(self): environment_value = os.environ.get(self.name, '') @@ -54,12 +60,11 @@ def execute(self): os.environ[self.name] = ':'.join(directories) -class RemovePath(object): +class RemovePath(AttributeHolder): def __init__(self, name, path, **kwargs): + super(RemovePath, self).__init__(**kwargs) self.name = name self.path = path - for key, value in kwargs.items(): - setattr(self, key, value) def execute(self): environment_value = os.environ.get(self.name, '') @@ -70,18 +75,26 @@ def execute(self): class EnvironmentModifications(object): """ - Keeps track of requests to modify the current environment + Keeps track of requests to modify the current environment. """ def __init__(self, other=None): + """ + Initializes a new instance, copying commands from other if it is not None + + Args: + other: another instance of EnvironmentModifications from which (optional) + """ self.env_modifications = [] if other is not None: - self._check_other(other) - self.env_modifications.extend(other.env_modifications) + self.extend(other) def __iter__(self): return iter(self.env_modifications) + def __len__(self): + return len(self.env_modifications) + def extend(self, other): self._check_other(other) self.env_modifications.extend(other.env_modifications) @@ -147,8 +160,18 @@ def remove_path(self, name, path, **kwargs): def concatenate_paths(paths): + """ + Concatenates an iterable of paths into a column separated string + + Args: + paths: iterable of paths + + Returns: + column separated string + """ return ':'.join(str(item) for item in paths) + def validate_environment_modifications(env): modifications = collections.defaultdict(list) for item in env: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c1a5c912be7..5d8258d4cf0 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -985,30 +985,41 @@ def module(self): def environment_modifications(self, dependent_spec): + """ + Called before the install() method of dependents. + + Return the list of environment modifications needed by dependents (or extensions). Default implementation does + nothing, but this can be overridden by an extendable package to set up the install environment for its + extensions. This is useful if there are some common steps to installing all extensions for a certain package. + + Example : + + 1. Installing python modules generally requires `PYTHONPATH` to point to the lib/pythonX.Y/site-packages + directory in the module's install prefix. This could set that variable. + + 2. A lot of Qt extensions need `QTDIR` set. This can be used to do that. + + Args: + dependent_spec: dependent (or extension) of this spec + + Returns: + instance of environment modifications + """ return EnvironmentModifications() def module_modifications(self, module, spec, dependent_spec): - """Called before the install() method of dependents. + """ + Called before the install() method of dependents. - Default implementation does nothing, but this can be - overridden by an extendable package to set up the install - environment for its extensions. This is useful if there are - some common steps to installing all extensions for a + Default implementation does nothing, but this can be overridden by an extendable package to set up the module of + its extensions. This is useful if there are some common steps to installing all extensions for a certain package. - Some examples: - - 1. Installing python modules generally requires PYTHONPATH to - point to the lib/pythonX.Y/site-packages directory in the - module's install prefix. This could set that variable. - - 2. Extensions often need to invoke the 'python' interpreter - from the Python installation being extended. This routine can - put a 'python' Execuable object in the module scope for the - extension package to simplify extension installs. - - 3. A lot of Qt extensions need QTDIR set. This can be used to do that. + Example : + 1. Extensions often need to invoke the 'python' interpreter from the Python installation being extended. + This routine can put a 'python' Executable object in the module scope for the extension package to simplify + extension installs. """ pass diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index 17061c8fd0e..97581ecb76a 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -55,5 +55,11 @@ def test_extra_arguments(self): apply_environment_modifications(env) self.assertEqual('dummy value', os.environ['A']) - def test_copy(self): - pass + def test_extend(self): + env = EnvironmentModifications() + env.set_env('A', 'dummy value') + env.set_env('B', 3) + copy_construct = EnvironmentModifications(env) + self.assertEqual(len(copy_construct), 2) + for x, y in zip(env, copy_construct): + self.assertIs(x, y) diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index ae8e5708bef..00cda8d5084 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -40,11 +40,13 @@ def env_flag(name): return False +# FIXME : remove this function ? def path_set(var_name, directories): path_str = ":".join(str(dir) for dir in directories) os.environ[var_name] = path_str +# FIXME : remove this function ? def path_put_first(var_name, directories): """Puts the provided directories first in the path, adding them if they're not already there. @@ -59,12 +61,6 @@ def path_put_first(var_name, directories): path_set(var_name, new_path) -def pop_keys(dictionary, *keys): - for key in keys: - if key in dictionary: - dictionary.pop(key) - - def dump_environment(path): """Dump the current environment out to a file.""" with open(path, 'w') as env_file: From ccced9f290bc213561209db2479bccfb5605366d Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Mar 2016 15:12:39 +0100 Subject: [PATCH 588/910] package : optimized imports --- lib/spack/spack/package.py | 41 ++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 5d8258d4cf0..042833964ac 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -34,41 +34,34 @@ README. """ import os -import errno import re -import shutil -import time -import itertools -import subprocess -import platform as py_platform -import multiprocessing -from urlparse import urlparse, urljoin import textwrap -from StringIO import StringIO +import time import llnl.util.tty as tty -from llnl.util.tty.log import log_output -from llnl.util.link_tree import LinkTree -from llnl.util.filesystem import * -from llnl.util.lang import * - import spack -import spack.error -import spack.compilers -import spack.mirror -import spack.hooks -import spack.directives -import spack.repository import spack.build_environment +import spack.compilers +import spack.directives +import spack.error +import spack.fetch_strategy as fs +import spack.hooks +import spack.mirror +import spack.repository import spack.url import spack.util.web -import spack.fetch_strategy as fs +from StringIO import StringIO +from llnl.util.filesystem import * +from llnl.util.lang import * +from llnl.util.link_tree import LinkTree +from llnl.util.tty.log import log_output from spack.environment import EnvironmentModifications -from spack.version import * from spack.stage import Stage, ResourceStage, StageComposite -from spack.util.compression import allowed_archive, extension -from spack.util.executable import ProcessError +from spack.util.compression import allowed_archive from spack.util.environment import dump_environment +from spack.util.executable import ProcessError +from spack.version import * +from urlparse import urlparse """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] From c8cc6f4fc111d5dd2d55295e569a10cd5739ceee Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Mar 2016 16:59:29 +0100 Subject: [PATCH 589/910] test : fix for python 2.6 --- lib/spack/spack/test/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index 97581ecb76a..f44282d6f39 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -62,4 +62,4 @@ def test_extend(self): copy_construct = EnvironmentModifications(env) self.assertEqual(len(copy_construct), 2) for x, y in zip(env, copy_construct): - self.assertIs(x, y) + assert x is y From d8a402ae5bcfbea627967531f7ec35cf77946b4a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 11 Mar 2016 13:33:56 -0800 Subject: [PATCH 590/910] Add compiler info and a simple libdwarf build to the checks. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index ab379be4867..1bed6b08740 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,10 @@ before_install: script: - . share/spack/setup-env.sh + - spack compilers + - spack config get compilers - spack test + - spack install -v libdwarf notifications: email: From 15bbd088e6007a9a6df8c4427340a371e5871506 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 15 Mar 2016 14:38:06 -0700 Subject: [PATCH 591/910] Fix #551: version bug in `spack create` - `spack create` now sets a proper version in generated file, based on the filename, even if it can't find any tarballs for the package. --- lib/spack/spack/cmd/create.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 4564143f834..f0cd50b8df6 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -208,7 +208,7 @@ def find_repository(spec, args): return repo -def fetch_tarballs(url, name, args): +def fetch_tarballs(url, name, version): """Try to find versions of the supplied archive by scraping the web. Prompts the user to select how many to download if many are found. @@ -222,7 +222,7 @@ def fetch_tarballs(url, name, args): archives_to_fetch = 1 if not versions: # If the fetch failed for some reason, revert to what the user provided - versions = { "version" : url } + versions = { version : url } elif len(versions) > 1: tty.msg("Found %s versions of %s:" % (len(versions), name), *spack.cmd.elide_list( @@ -256,7 +256,7 @@ def create(parser, args): tty.msg("Creating template for package %s" % name) # Fetch tarballs (prompting user if necessary) - versions, urls = fetch_tarballs(url, name, args) + versions, urls = fetch_tarballs(url, name, version) # Try to guess what configure system is used. guesser = ConfigureGuesser() From 7eca383b10f017666c320ffd7deb6cae83b5c26b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 10 Mar 2016 01:12:25 -0800 Subject: [PATCH 592/910] Add sanity check paths to packages; fix #505 --- lib/spack/spack/package.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 696adaf8965..02fb3e58348 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -318,6 +318,17 @@ class SomePackage(Package): """Most packages are NOT extendable. Set to True if you want extensions.""" extendable = False + """List of prefix-relative file paths. If these do not exist after + install, or if they exist but are not files, sanity checks fail. + """ + sanity_check_files = [] + + """List of prefix-relative directory paths. If these do not exist + after install, or if they exist but are not directories, sanity + checks will fail. + """ + sanity_check_dirs = [] + def __init__(self, spec): # this determines how the package should be built. @@ -909,7 +920,7 @@ def build_process(): raise e # Ensure that something was actually installed. - self._sanity_check_install() + self.sanity_check_prefix() # Copy provenance into the install directory on success log_install_path = spack.install_layout.build_log_path(self.spec) @@ -952,7 +963,18 @@ def build_process(): spack.hooks.post_install(self) - def _sanity_check_install(self): + def sanity_check_prefix(self): + """This function checks whether install succeeded.""" + def check_paths(path_list, filetype, predicate): + for path in path_list: + abs_path = os.path.join(self.prefix, path) + if not predicate(abs_path): + raise InstallError("Install failed for %s. No such %s in prefix: %s" + % (self.name, filetype, path)) + + check_paths(self.sanity_check_files, 'file', os.path.isfile) + check_paths(self.sanity_check_dirs, 'directory', os.path.isdir) + installed = set(os.listdir(self.prefix)) installed.difference_update(spack.install_layout.hidden_file_paths) if not installed: From 68d22253eca40bc4223234d6c26204f292be6a0c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 15 Mar 2016 17:22:48 -0700 Subject: [PATCH 593/910] Add sanity checks to the libelf build. --- var/spack/repos/builtin/packages/libelf/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/libelf/package.py b/var/spack/repos/builtin/packages/libelf/package.py index 29bc21b65c4..0fcb56c164d 100644 --- a/var/spack/repos/builtin/packages/libelf/package.py +++ b/var/spack/repos/builtin/packages/libelf/package.py @@ -38,6 +38,9 @@ class Libelf(Package): provides('elf') + sanity_check_files = ['include/libelf.h'] + sanity_check_dirs = ['lib'] + def install(self, spec, prefix): configure("--prefix=" + prefix, "--enable-shared", From b45ec3f04e3627dbe3633239560873ae01bf3beb Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 16 Mar 2016 10:55:28 +0100 Subject: [PATCH 594/910] environment : simplified modification of the environment --- lib/spack/spack/build_environment.py | 12 ++-- lib/spack/spack/cmd/module.py | 2 +- lib/spack/spack/environment.py | 58 ++++++++++--------- lib/spack/spack/test/environment.py | 12 ++-- lib/spack/spack/util/environment.py | 2 +- .../repos/builtin/packages/python/package.py | 1 + 6 files changed, 47 insertions(+), 40 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index e86a7c413a9..e5d256a2e01 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -35,7 +35,7 @@ import spack from llnl.util.filesystem import * -from spack.environment import EnvironmentModifications, apply_environment_modifications, concatenate_paths +from spack.environment import EnvironmentModifications, concatenate_paths from spack.util.environment import * from spack.util.executable import Executable, which @@ -283,10 +283,12 @@ def setup_package(pkg): set_module_variables_for_package(pkg, mod) # Allow dependencies to set up environment as well. - for dep_spec in pkg.spec.traverse(root=False): - dep_spec.package.module_modifications(pkg.module, dep_spec, pkg.spec) - env.extend(dep_spec.package.environment_modifications(pkg.spec)) - apply_environment_modifications(env) + for dependency_spec in pkg.spec.traverse(root=False): + dependency_spec.package.module_modifications(pkg.module, dependency_spec, pkg.spec) + env.extend(dependency_spec.package.environment_modifications(pkg.spec)) + # TODO : implement validation + #validate(env) + env.apply_modifications() def fork(pkg, function): diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 1d6867c1d96..315d9fc9266 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -80,7 +80,7 @@ def module_find(mtype, spec_array): if not os.path.isfile(mod.file_name): tty.die("No %s module is installed for %s" % (mtype, spec)) - print mod.use_name + print(mod.use_name) def module_refresh(): diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 5a68e10c997..b557bf0bbcf 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -158,41 +158,43 @@ def remove_path(self, name, path, **kwargs): item = RemovePath(name, path, **kwargs) self.env_modifications.append(item) + def group_by_name(self): + """ + Returns a dict of the modifications grouped by variable name + + Returns: + dict mapping the environment variable name to the modifications to be done on it + """ + modifications = collections.defaultdict(list) + for item in self: + modifications[item.name].append(item) + return modifications + + def clear(self): + """ + Clears the current list of modifications + """ + self.env_modifications.clear() + + def apply_modifications(self): + """ + Applies the modifications and clears the list + """ + modifications = self.group_by_name() + # Apply the modifications to the environment variables one variable at a time + for name, actions in sorted(modifications.items()): + for x in actions: + x.execute() + def concatenate_paths(paths): """ - Concatenates an iterable of paths into a column separated string + Concatenates an iterable of paths into a string of column separated paths Args: paths: iterable of paths Returns: - column separated string + string """ return ':'.join(str(item) for item in paths) - - -def validate_environment_modifications(env): - modifications = collections.defaultdict(list) - for item in env: - modifications[item.name].append(item) - # TODO : once we organized the modifications into a dictionary that maps an environment variable - # TODO : to a list of action to be done on it, we may easily spot inconsistencies and warn the user if - # TODO : something suspicious is happening - return modifications - - -def apply_environment_modifications(env): - """ - Modifies the current environment according to the request in env - - Args: - env: object storing modifications to the environment - """ - modifications = validate_environment_modifications(env) - - # Cycle over the environment variables that will be modified - for variable, actions in modifications.items(): - # Execute all the actions in the order they were issued - for x in actions: - x.execute() diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index f44282d6f39..d0b093b0543 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -1,6 +1,6 @@ import unittest import os -from spack.environment import EnvironmentModifications, apply_environment_modifications +from spack.environment import EnvironmentModifications class EnvironmentTest(unittest.TestCase): @@ -15,7 +15,7 @@ def test_set_env(self): env = EnvironmentModifications() env.set_env('A', 'dummy value') env.set_env('B', 3) - apply_environment_modifications(env) + env.apply_modifications() self.assertEqual('dummy value', os.environ['A']) self.assertEqual(str(3), os.environ['B']) @@ -23,7 +23,7 @@ def test_unset_env(self): env = EnvironmentModifications() self.assertEqual('foo', os.environ['UNSET_ME']) env.unset_env('UNSET_ME') - apply_environment_modifications(env) + env.apply_modifications() self.assertRaises(KeyError, os.environ.__getitem__, 'UNSET_ME') def test_path_manipulation(self): @@ -43,7 +43,7 @@ def test_path_manipulation(self): env.remove_path('REMOVE_PATH_LIST', '/remove/this') env.remove_path('REMOVE_PATH_LIST', '/duplicate/') - apply_environment_modifications(env) + env.apply_modifications() self.assertEqual('/path/first:/path/second:/path/third:/path/last', os.environ['PATH_LIST']) self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST']) self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST']) @@ -52,7 +52,9 @@ def test_path_manipulation(self): def test_extra_arguments(self): env = EnvironmentModifications() env.set_env('A', 'dummy value', who='Pkg1') - apply_environment_modifications(env) + for x in env: + assert hasattr(x, 'who') + env.apply_modifications() self.assertEqual('dummy value', os.environ['A']) def test_extend(self): diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index 00cda8d5084..1485992b0f1 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -64,5 +64,5 @@ def path_put_first(var_name, directories): def dump_environment(path): """Dump the current environment out to a file.""" with open(path, 'w') as env_file: - for key,val in sorted(os.environ.items()): + for key, val in sorted(os.environ.items()): env_file.write("%s=%s\n" % (key, val)) diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 307cec726b6..2f9948d451c 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -98,6 +98,7 @@ def environment_modifications(self, extension_spec): if d.package.extends(self.spec): python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) env.set_env['PYTHONPATH'] = ':'.join(python_paths) + return env def module_modifications(self, module, spec, ext_spec): """Called before python modules' install() methods. From 9c2996667433e4f69eb25cce2d74a94caeb1646c Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 16 Mar 2016 12:57:17 +0100 Subject: [PATCH 595/910] Change urls in binutils to help 'spack checksum/versions' + adding latest version --- var/spack/repos/builtin/packages/binutils/package.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py index de04221e33a..897539a4391 100644 --- a/var/spack/repos/builtin/packages/binutils/package.py +++ b/var/spack/repos/builtin/packages/binutils/package.py @@ -4,10 +4,13 @@ class Binutils(Package): """GNU binutils, which contain the linker, assembler, objdump and others""" homepage = "http://www.gnu.org/software/binutils/" - version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2") - version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.24.tar.bz2") - version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.23.2.tar.bz2") - version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.20.1.tar.bz2") + url="https://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2" + + version('2.26', '64146a0faa3b411ba774f47d41de239f') + version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66') + version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b') + version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e') + version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764') # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell variant('krellpatch', default=False, description="build with openspeedshop based patch.") From 597727f8bedc894330dfd26eab1a82859980f2f1 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 16 Mar 2016 15:19:13 +0100 Subject: [PATCH 596/910] tclmodules : added hooks to process EnvironmentModifications objects --- lib/spack/spack/modules.py | 143 +++++++++++++----- .../repos/builtin/packages/mpich/package.py | 25 ++- .../repos/builtin/packages/python/package.py | 15 +- 3 files changed, 133 insertions(+), 50 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index c27043db8cd..1a0a0fd4d61 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -47,18 +47,18 @@ __all__ = ['EnvModule', 'Dotkit', 'TclModule'] import os +import os.path import re -import textwrap import shutil +import textwrap from glob import glob import llnl.util.tty as tty +import spack +from spack.environment import * from llnl.util.filesystem import join_path, mkdirp -import spack - -"""Registry of all types of modules. Entries created by EnvModule's - metaclass.""" +# Registry of all types of modules. Entries created by EnvModule's metaclass module_types = {} @@ -79,6 +79,32 @@ def print_help(): "") +class PathInspector(object): + dirname2varname = { + 'bin': ('PATH',), + 'man': ('MANPATH',), + 'lib': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), + 'lib64': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), + 'include': ('CPATH',), + 'pkgconfig': ('PKG_CONFIG_PATH',) + } + + def __call__(self, env, directory, names): + for name in names: + variables = PathInspector.dirname2varname.get(name, None) + if variables is None: + continue + absolute_path = join_path(os.path.abspath(directory), name) + for variable in variables: + env.prepend_path(variable, absolute_path) + + +def inspect_path(path): + env, inspector = EnvironmentModifications(), PathInspector() + os.path.walk(path, inspector, env) + return env + + class EnvModule(object): name = 'env_module' @@ -88,21 +114,27 @@ def __init__(cls, name, bases, dict): if cls.name != 'env_module': module_types[cls.name] = cls - def __init__(self, spec=None): # category in the modules system # TODO: come up with smarter category names. self.category = "spack" - # Descriptions for the module system's UI - self.short_description = "" - self.long_description = "" - # dict pathname -> list of directories to be prepended to in # the module file. self._paths = None self.spec = spec + self.pkg = spec.package # Just stored for convenience + # short description default is just the package + version + # packages can provide this optional attribute + self.short_description = spec.format("$_ $@") + if hasattr(self.pkg, 'short_description'): + self.short_description = self.pkg.short_description + + # long description is the docstring with reduced whitespace. + self.long_description = None + if self.spec.package.__doc__: + self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) @property def paths(self): @@ -130,26 +162,19 @@ def add_path(path_name, directory): add_path(var, directory) # Add python path unless it's an actual python installation - # TODO: is there a better way to do this? + # TODO : is there a better way to do this? + # FIXME : add PYTHONPATH to every python package if self.spec.name != 'python': site_packages = glob(join_path(self.spec.prefix.lib, "python*/site-packages")) if site_packages: add_path('PYTHONPATH', site_packages[0]) + # FIXME : Same for GEM_PATH if self.spec.package.extends(spack.spec.Spec('ruby')): - add_path('GEM_PATH', self.spec.prefix) - - # short description is just the package + version - # TODO: maybe packages can optionally provide it. - self.short_description = self.spec.format("$_ $@") - - # long description is the docstring with reduced whitespace. - if self.spec.package.__doc__: - self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) + add_path('GEM_PATH', self.spec.prefix) return self._paths - def write(self): """Write out a module file for this object.""" module_dir = os.path.dirname(self.file_name) @@ -160,9 +185,18 @@ def write(self): if not self.paths: return - with open(self.file_name, 'w') as f: - self._write(f) + # Construct the changes that needs to be done on the environment for + env = inspect_path(self.spec.prefix) + # FIXME : move the logic to inspection + env.prepend_path('CMAKE_PREFIX_PATH', self.spec.prefix) + # FIXME : decide how to distinguish between calls done in the installation and elsewhere + env.extend(self.spec.package.environment_modifications(None)) + # site_specific = ...` + if not env: + return + with open(self.file_name, 'w') as f: + self._write(f, env) def _write(self, stream): """To be implemented by subclasses.""" @@ -175,14 +209,12 @@ def file_name(self): where this module lives.""" raise NotImplementedError() - @property def use_name(self): """Subclasses should implement this to return the name the module command uses to refer to the package.""" raise NotImplementedError() - def remove(self): mod_file = self.file_name if os.path.exists(mod_file): @@ -205,7 +237,7 @@ def use_name(self): self.spec.compiler.version, self.spec.dag_hash()) - def _write(self, dk_file): + def _write(self, dk_file, env): # Category if self.category: dk_file.write('#c %s\n' % self.category) @@ -231,6 +263,10 @@ def _write(self, dk_file): class TclModule(EnvModule): name = 'tcl' path = join_path(spack.share_path, "modules") + formats = { + PrependPath: 'prepend-path {0.name} \"{0.path}\"\n', + SetEnv: 'setenv {0.name} \"{0.value}\"\n' + } @property def file_name(self): @@ -244,25 +280,56 @@ def use_name(self): self.spec.compiler.version, self.spec.dag_hash()) + def process_environment_command(self, env): + for command in env: + # FIXME : how should we handle errors here? + yield self.formats[type(command)].format(command) - def _write(self, m_file): - # TODO: cateogry? - m_file.write('#%Module1.0\n') + def _write(self, module_file, env): + """ + Writes a TCL module file for this package + Args: + module_file: module file stream + env: list of environment modifications to be written in the module file + """ + # TCL Modulefile header + module_file.write('#%Module1.0\n') + # TODO : category ? # Short description if self.short_description: - m_file.write('module-whatis \"%s\"\n\n' % self.short_description) + module_file.write('module-whatis \"%s\"\n\n' % self.short_description) # Long description if self.long_description: - m_file.write('proc ModulesHelp { } {\n') + module_file.write('proc ModulesHelp { } {\n') doc = re.sub(r'"', '\"', self.long_description) - m_file.write("puts stderr \"%s\"\n" % doc) - m_file.write('}\n\n') + module_file.write("puts stderr \"%s\"\n" % doc) + module_file.write('}\n\n') - # Path alterations - for var, dirs in self.paths.items(): - for directory in dirs: - m_file.write("prepend-path %s \"%s\"\n" % (var, directory)) + # Environment modifications + for line in self.process_environment_command(env): + module_file.write(line) - m_file.write("prepend-path CMAKE_PREFIX_PATH \"%s\"\n" % self.spec.prefix) + # FIXME : REMOVE + # def _write(self, m_file): + # # TODO: cateogry? + # m_file.write('#%Module1.0\n') + # + # # Short description + # if self.short_description: + # m_file.write('module-whatis \"%s\"\n\n' % self.short_description) + # + # # Long description + # if self.long_description: + # m_file.write('proc ModulesHelp { } {\n') + # doc = re.sub(r'"', '\"', self.long_description) + # m_file.write("puts stderr \"%s\"\n" % doc) + # m_file.write('}\n\n') + # + # # Path alterations + # for var, dirs in self.paths.items(): + # for directory in dirs: + # m_file.write("prepend-path %s \"%s\"\n" % (var, directory)) + # + # m_file.write("prepend-path CMAKE_PREFIX_PATH \"%s\"\n" % self.spec.prefix) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index d298981c925..4c34d0308f5 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -25,6 +25,7 @@ from spack import * import os + class Mpich(Package): """MPICH is a high performance and widely portable implementation of the Message Passing Interface (MPI) standard.""" @@ -48,11 +49,25 @@ class Mpich(Package): def environment_modifications(self, dependent_spec): env = super(Mpich, self).environment_modifications(dependent_spec) - env.set_env('MPICH_CC', os.environ['CC']) - env.set_env('MPICH_CXX', os.environ['CXX']) - env.set_env('MPICH_F77', os.environ['F77']) - env.set_env('MPICH_F90', os.environ['FC']) - env.set_env('MPICH_FC', os.environ['FC']) + + if dependent_spec is None: + # We are not using compiler wrappers + cc = self.compiler.cc + cxx = self.compiler.cxx + f77 = self.compiler.f77 + f90 = fc = self.compiler.fc + else: + # Spack compiler wrappers + cc = os.environ['CC'] + cxx = os.environ['CXX'] + f77 = os.environ['F77'] + f90 = fc = os.environ['FC'] + + env.set_env('MPICH_CC', cc) + env.set_env('MPICH_CXX', cxx) + env.set_env('MPICH_F77', f77) + env.set_env('MPICH_F90', f90) + env.set_env('MPICH_FC', fc) return env def module_modifications(self, module, spec, dep_spec): diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 2f9948d451c..acb36517268 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -91,13 +91,14 @@ def site_packages_dir(self): def environment_modifications(self, extension_spec): env = super(Python, self).environment_modifications(extension_spec) - # Set PYTHONPATH to include site-packages dir for the - # extension and any other python extensions it depends on. - python_paths = [] - for d in extension_spec.traverse(): - if d.package.extends(self.spec): - python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) - env.set_env['PYTHONPATH'] = ':'.join(python_paths) + if extension_spec is not None: + # Set PYTHONPATH to include site-packages dir for the + # extension and any other python extensions it depends on. + python_paths = [] + for d in extension_spec.traverse(): + if d.package.extends(self.spec): + python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) + env.set_env['PYTHONPATH'] = ':'.join(python_paths) return env def module_modifications(self, module, spec, ext_spec): From ac762e95a6213e1514f29d3d6501e4a95dd3e1d4 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 16 Mar 2016 16:23:02 +0100 Subject: [PATCH 597/910] modules : removed dead code --- lib/spack/spack/modules.py | 205 ++++++++++++------------------------- 1 file changed, 66 insertions(+), 139 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 1a0a0fd4d61..b64a8b32266 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -22,14 +22,12 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -"""This module contains code for creating environment modules, which -can include dotkits, tcl modules, lmod, and others. +""" +This module contains code for creating environment modules, which can include dotkits, tcl modules, lmod, and others. -The various types of modules are installed by post-install hooks and -removed after an uninstall by post-uninstall hooks. This class -consolidates the logic for creating an abstract description of the -information that module systems need. Currently that includes a -number of directories to be appended to paths in the user's environment: +The various types of modules are installed by post-install hooks and removed after an uninstall by post-uninstall hooks. +This class consolidates the logic for creating an abstract description of the information that module systems need. +Currently that includes a number of directories to be appended to paths in the user's environment: * /bin directories to be appended to PATH * /lib* directories for LD_LIBRARY_PATH @@ -37,26 +35,23 @@ * /man* and /share/man* directories for MANPATH * the package prefix for CMAKE_PREFIX_PATH -This module also includes logic for coming up with unique names for -the module files so that they can be found by the various -shell-support files in $SPACK/share/spack/setup-env.*. +This module also includes logic for coming up with unique names for the module files so that they can be found by the +various shell-support files in $SPACK/share/spack/setup-env.*. -Each hook in hooks/ implements the logic for writing its specific type -of module file. +Each hook in hooks/ implements the logic for writing its specific type of module file. """ -__all__ = ['EnvModule', 'Dotkit', 'TclModule'] - import os import os.path import re import shutil import textwrap -from glob import glob import llnl.util.tty as tty import spack -from spack.environment import * from llnl.util.filesystem import join_path, mkdirp +from spack.environment import * + +__all__ = ['EnvModule', 'Dotkit', 'TclModule'] # Registry of all types of modules. Entries created by EnvModule's metaclass module_types = {} @@ -79,34 +74,35 @@ def print_help(): "") -class PathInspector(object): - dirname2varname = { - 'bin': ('PATH',), - 'man': ('MANPATH',), - 'lib': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), - 'lib64': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), - 'include': ('CPATH',), - 'pkgconfig': ('PKG_CONFIG_PATH',) - } - - def __call__(self, env, directory, names): - for name in names: - variables = PathInspector.dirname2varname.get(name, None) - if variables is None: - continue - absolute_path = join_path(os.path.abspath(directory), name) - for variable in variables: - env.prepend_path(variable, absolute_path) - - def inspect_path(path): + class PathInspector(object): + dirname2varname = { + 'bin': ('PATH',), + 'man': ('MANPATH',), + 'lib': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), + 'lib64': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), + 'include': ('CPATH',), + 'pkgconfig': ('PKG_CONFIG_PATH',) + } + + def __call__(self, env, directory, names): + for name in names: + variables = PathInspector.dirname2varname.get(name, None) + if variables is None: + continue + absolute_path = join_path(os.path.abspath(directory), name) + for variable in variables: + env.prepend_path(variable, absolute_path) + env, inspector = EnvironmentModifications(), PathInspector() os.path.walk(path, inspector, env) + env.prepend_path('CMAKE_PREFIX_PATH', path) return env class EnvModule(object): name = 'env_module' + formats = {} class __metaclass__(type): def __init__(cls, name, bases, dict): @@ -119,9 +115,6 @@ def __init__(self, spec=None): # TODO: come up with smarter category names. self.category = "spack" - # dict pathname -> list of directories to be prepended to in - # the module file. - self._paths = None self.spec = spec self.pkg = spec.package # Just stored for convenience @@ -136,44 +129,21 @@ def __init__(self, spec=None): if self.spec.package.__doc__: self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) - @property - def paths(self): - if self._paths is None: - self._paths = {} - - def add_path(path_name, directory): - path = self._paths.setdefault(path_name, []) - path.append(directory) - - # Add paths if they exist. - for var, directory in [ - ('PATH', self.spec.prefix.bin), - ('MANPATH', self.spec.prefix.man), - ('MANPATH', self.spec.prefix.share_man), - ('LIBRARY_PATH', self.spec.prefix.lib), - ('LIBRARY_PATH', self.spec.prefix.lib64), - ('LD_LIBRARY_PATH', self.spec.prefix.lib), - ('LD_LIBRARY_PATH', self.spec.prefix.lib64), - ('CPATH', self.spec.prefix.include), - ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib, 'pkgconfig')), - ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib64, 'pkgconfig'))]: - - if os.path.isdir(directory): - add_path(var, directory) - - # Add python path unless it's an actual python installation - # TODO : is there a better way to do this? - # FIXME : add PYTHONPATH to every python package - if self.spec.name != 'python': - site_packages = glob(join_path(self.spec.prefix.lib, "python*/site-packages")) - if site_packages: - add_path('PYTHONPATH', site_packages[0]) - - # FIXME : Same for GEM_PATH - if self.spec.package.extends(spack.spec.Spec('ruby')): - add_path('GEM_PATH', self.spec.prefix) - - return self._paths + # @property + # def paths(self): + # # Add python path unless it's an actual python installation + # # TODO : is there a better way to do this? + # # FIXME : add PYTHONPATH to every python package + # if self.spec.name != 'python': + # site_packages = glob(join_path(self.spec.prefix.lib, "python*/site-packages")) + # if site_packages: + # add_path('PYTHONPATH', site_packages[0]) + # + # # FIXME : Same for GEM_PATH + # if self.spec.package.extends(spack.spec.Spec('ruby')): + # add_path('GEM_PATH', self.spec.prefix) + # + # return self._paths def write(self): """Write out a module file for this object.""" @@ -181,14 +151,9 @@ def write(self): if not os.path.exists(module_dir): mkdirp(module_dir) - # If there are no paths, no need for a dotkit. - if not self.paths: - return - - # Construct the changes that needs to be done on the environment for + # Environment modifications guessed by inspecting the installation prefix env = inspect_path(self.spec.prefix) - # FIXME : move the logic to inspection - env.prepend_path('CMAKE_PREFIX_PATH', self.spec.prefix) + # Package-specific environment modifications # FIXME : decide how to distinguish between calls done in the installation and elsewhere env.extend(self.spec.package.environment_modifications(None)) # site_specific = ...` @@ -196,12 +161,17 @@ def write(self): return with open(self.file_name, 'w') as f: - self._write(f, env) + self.write_header(f) + for line in self.process_environment_command(env): + f.write(line) - def _write(self, stream): - """To be implemented by subclasses.""" + def write_header(self, stream): raise NotImplementedError() + def process_environment_command(self, env): + for command in env: + # FIXME : how should we handle errors here? + yield self.formats[type(command)].format(command) @property def file_name(self): @@ -225,10 +195,14 @@ class Dotkit(EnvModule): name = 'dotkit' path = join_path(spack.share_path, "dotkit") + formats = { + PrependPath: 'dk_alter {0.name} {0.path}\n', + SetEnv: 'dk_setenv {0.name} {0.value}\n' + } + @property def file_name(self): - return join_path(Dotkit.path, self.spec.architecture, - '%s.dk' % self.use_name) + return join_path(Dotkit.path, self.spec.architecture, '%s.dk' % self.use_name) @property def use_name(self): @@ -237,7 +211,7 @@ def use_name(self): self.spec.compiler.version, self.spec.dag_hash()) - def _write(self, dk_file, env): + def write_header(self, dk_file): # Category if self.category: dk_file.write('#c %s\n' % self.category) @@ -251,18 +225,11 @@ def _write(self, dk_file, env): for line in textwrap.wrap(self.long_description, 72): dk_file.write("#h %s\n" % line) - # Path alterations - for var, dirs in self.paths.items(): - for directory in dirs: - dk_file.write("dk_alter %s %s\n" % (var, directory)) - - # Let CMake find this package. - dk_file.write("dk_alter CMAKE_PREFIX_PATH %s\n" % self.spec.prefix) - class TclModule(EnvModule): name = 'tcl' path = join_path(spack.share_path, "modules") + formats = { PrependPath: 'prepend-path {0.name} \"{0.path}\"\n', SetEnv: 'setenv {0.name} \"{0.value}\"\n' @@ -272,7 +239,6 @@ class TclModule(EnvModule): def file_name(self): return join_path(TclModule.path, self.spec.architecture, self.use_name) - @property def use_name(self): return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version, @@ -280,19 +246,7 @@ def use_name(self): self.spec.compiler.version, self.spec.dag_hash()) - def process_environment_command(self, env): - for command in env: - # FIXME : how should we handle errors here? - yield self.formats[type(command)].format(command) - - def _write(self, module_file, env): - """ - Writes a TCL module file for this package - - Args: - module_file: module file stream - env: list of environment modifications to be written in the module file - """ + def write_header(self, module_file): # TCL Modulefile header module_file.write('#%Module1.0\n') # TODO : category ? @@ -306,30 +260,3 @@ def _write(self, module_file, env): doc = re.sub(r'"', '\"', self.long_description) module_file.write("puts stderr \"%s\"\n" % doc) module_file.write('}\n\n') - - # Environment modifications - for line in self.process_environment_command(env): - module_file.write(line) - - # FIXME : REMOVE - # def _write(self, m_file): - # # TODO: cateogry? - # m_file.write('#%Module1.0\n') - # - # # Short description - # if self.short_description: - # m_file.write('module-whatis \"%s\"\n\n' % self.short_description) - # - # # Long description - # if self.long_description: - # m_file.write('proc ModulesHelp { } {\n') - # doc = re.sub(r'"', '\"', self.long_description) - # m_file.write("puts stderr \"%s\"\n" % doc) - # m_file.write('}\n\n') - # - # # Path alterations - # for var, dirs in self.paths.items(): - # for directory in dirs: - # m_file.write("prepend-path %s \"%s\"\n" % (var, directory)) - # - # m_file.write("prepend-path CMAKE_PREFIX_PATH \"%s\"\n" % self.spec.prefix) From bb04d5cc63182b6129f9abf78db260a2dab9f506 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 16 Mar 2016 14:27:22 -0500 Subject: [PATCH 598/910] Fix grammar in mirrors documentation --- lib/spack/docs/mirrors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index b20fedb55f1..dad04d053b0 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -38,7 +38,7 @@ contains tarballs for each package, named after each package. .. note:: - Archives are **not** named exactly they were in the package's fetch + Archives are **not** named exactly the way they were in the package's fetch URL. They have the form ``-.``, where ```` is Spack's name for the package, ```` is the version of the tarball, and ```` is whatever format the From fa2a66db6721ef5d7b54b9cede69aa2f1b5b7531 Mon Sep 17 00:00:00 2001 From: citibeth Date: Wed, 16 Mar 2016 15:46:59 -0400 Subject: [PATCH 599/910] 1. Disabled git versions known to have vulnerabilities. 2. Added autoconf command to allow building directly from GitHub source. --- .../repos/builtin/packages/git/package.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/git/package.py b/var/spack/repos/builtin/packages/git/package.py index ddc5078c4d3..586b6ce3c39 100644 --- a/var/spack/repos/builtin/packages/git/package.py +++ b/var/spack/repos/builtin/packages/git/package.py @@ -5,14 +5,22 @@ class Git(Package): system designed to handle everything from small to very large projects with speed and efficiency.""" homepage = "http://git-scm.com" - url = "https://www.kernel.org/pub/software/scm/git/git-2.2.1.tar.gz" + url = "https://github.com/git/git/tarball/v2.7.1" - version('2.6.3', 'b711be7628a4a2c25f38d859ee81b423') - version('2.6.2', 'da293290da69f45a86a311ad3cd43dc8') - version('2.6.1', '4c62ee9c5991fe93d99cf2a6b68397fd') - version('2.6.0', 'eb76a07148d94802a1745d759716a57e') - version('2.5.4', '3eca2390cf1fa698b48e2a233563a76b') - version('2.2.1', 'ff41fdb094eed1ec430aed8ee9b9849c') + version('2.8.0-rc2', 'c2cf9f2cc70e35f2fafbaf9258f82e4c') + version('2.7.3', 'fa1c008b56618c355a32ba4a678305f6') + version('2.7.1', 'bf0706b433a8dedd27a63a72f9a66060') + + + # See here for info on vulnerable Git versions: + # http://www.theregister.co.uk/2016/03/16/git_server_client_patch_now/ + # All the following are vulnerable + #version('2.6.3', 'b711be7628a4a2c25f38d859ee81b423') + #version('2.6.2', 'da293290da69f45a86a311ad3cd43dc8') + #version('2.6.1', '4c62ee9c5991fe93d99cf2a6b68397fd') + #version('2.6.0', 'eb76a07148d94802a1745d759716a57e') + #version('2.5.4', '3eca2390cf1fa698b48e2a233563a76b') + #version('2.2.1', 'ff41fdb094eed1ec430aed8ee9b9849c') # Git compiles with curl support by default on but if your system @@ -24,6 +32,7 @@ class Git(Package): variant("expat", default=False, description="Add the internal support of expat for https push") depends_on("openssl") + depends_on("autoconf") depends_on("curl", when="+curl") depends_on("expat", when="+expat") @@ -47,6 +56,7 @@ def install(self, spec, prefix): if '+expat' in spec: configure_args.append("--with-expat=%s" % spec['expat'].prefix) + which('autoreconf')('-i') configure(*configure_args) make() make("install") From f07d4c94393f6541d1a6afd9118d3f9c84bd08fc Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Wed, 16 Mar 2016 18:55:14 -0600 Subject: [PATCH 600/910] + Provide ~perl and ~shared variants. --- var/spack/repos/builtin/packages/graphviz/package.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/graphviz/package.py b/var/spack/repos/builtin/packages/graphviz/package.py index 7af7da1881a..2a2102bfcb6 100644 --- a/var/spack/repos/builtin/packages/graphviz/package.py +++ b/var/spack/repos/builtin/packages/graphviz/package.py @@ -7,6 +7,9 @@ class Graphviz(Package): version('2.38.0', '5b6a829b2ac94efcd5fa3c223ed6d3ae') + variant('perl', default=True, description='Disable if you have problems with the optional script language bindings') + variant('shared', default=True, description='Building static is required on AIX') + parallel = False depends_on("swig") @@ -14,8 +17,12 @@ class Graphviz(Package): depends_on("ghostscript") def install(self, spec, prefix): - configure("--prefix=%s" %prefix) + options = ['--prefix=%s' % prefix] + if '~perl' in spec: + options.append('--disable-perl') + if '~shared' in spec: + options.append('--enable-shared=no') + configure(*options) make() make("install") - From 9cdd79e33f8463699fcea0b668eafac1c9fae1d4 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 17 Mar 2016 14:14:18 +0100 Subject: [PATCH 601/910] modules : restored previous logic for path inspection --- lib/spack/spack/modules.py | 50 ++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index b64a8b32266..7e395736e4f 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -74,29 +74,37 @@ def print_help(): "") -def inspect_path(path): - class PathInspector(object): - dirname2varname = { - 'bin': ('PATH',), - 'man': ('MANPATH',), - 'lib': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), - 'lib64': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), - 'include': ('CPATH',), - 'pkgconfig': ('PKG_CONFIG_PATH',) - } +def inspect_path(prefix): + """ + Inspects the prefix of an installation to search for common layouts. Issues a request to modify the environment + accordingly when an item is found. - def __call__(self, env, directory, names): - for name in names: - variables = PathInspector.dirname2varname.get(name, None) - if variables is None: - continue - absolute_path = join_path(os.path.abspath(directory), name) - for variable in variables: - env.prepend_path(variable, absolute_path) + Args: + prefix: prefix of the installation - env, inspector = EnvironmentModifications(), PathInspector() - os.path.walk(path, inspector, env) - env.prepend_path('CMAKE_PREFIX_PATH', path) + Returns: + instance of EnvironmentModifications containing the requested modifications + """ + env = EnvironmentModifications() + # Inspect the prefix to check for the existence of common directories + prefix_inspections = { + 'bin': ('PATH',), + 'man': ('MANPATH',), + 'lib': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), + 'lib64': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), + 'include': ('CPATH',) + } + for attribute, variables in prefix_inspections.items(): + expected = getattr(prefix, attribute) + if os.path.isdir(expected): + for variable in variables: + env.prepend_path(variable, expected) + # PKGCONFIG + for expected in (join_path(prefix.lib, 'pkgconfig'), join_path(prefix.lib64, 'pkgconfig')): + if os.path.isdir(expected): + env.prepend_path('PKG_CONFIG_PATH', expected) + # CMake related variables + env.prepend_path('CMAKE_PREFIX_PATH', prefix) return env From f0f0663d1b9ece1e2b0b0a8f720b1325eec443bb Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 17 Mar 2016 15:11:39 +0100 Subject: [PATCH 602/910] package : split `environment_modifications` into `setup_environment` and `setup_dependent_environment`. package : renamed `module_modifications` to `modify_module` for consistency --- lib/spack/spack/build_environment.py | 4 +-- lib/spack/spack/modules.py | 5 ++- lib/spack/spack/package.py | 9 +++-- .../repos/builtin/packages/mpich/package.py | 35 +++++++------------ .../packages/netlib-scalapack/package.py | 2 +- .../repos/builtin/packages/openmpi/package.py | 19 +++++----- .../repos/builtin/packages/python/package.py | 25 ++++++------- .../repos/builtin/packages/qt/package.py | 4 +-- .../repos/builtin/packages/ruby/package.py | 6 ++-- 9 files changed, 49 insertions(+), 60 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index e5d256a2e01..68477145fef 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -284,8 +284,8 @@ def setup_package(pkg): # Allow dependencies to set up environment as well. for dependency_spec in pkg.spec.traverse(root=False): - dependency_spec.package.module_modifications(pkg.module, dependency_spec, pkg.spec) - env.extend(dependency_spec.package.environment_modifications(pkg.spec)) + dependency_spec.package.modify_module(pkg.module, dependency_spec, pkg.spec) + dependency_spec.package.setup_dependent_environment(env, pkg.spec) # TODO : implement validation #validate(env) env.apply_modifications() diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 7e395736e4f..09895d8c441 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -162,9 +162,8 @@ def write(self): # Environment modifications guessed by inspecting the installation prefix env = inspect_path(self.spec.prefix) # Package-specific environment modifications - # FIXME : decide how to distinguish between calls done in the installation and elsewhere - env.extend(self.spec.package.environment_modifications(None)) - # site_specific = ...` + self.spec.package.setup_environment(env) + # TODO : implement site-specific modifications and filters if not env: return diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 042833964ac..8e56dd1f976 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -977,7 +977,7 @@ def module(self): fromlist=[self.__class__.__name__]) - def environment_modifications(self, dependent_spec): + def setup_environment(self, env): """ Called before the install() method of dependents. @@ -998,9 +998,12 @@ def environment_modifications(self, dependent_spec): Returns: instance of environment modifications """ - return EnvironmentModifications() + pass - def module_modifications(self, module, spec, dependent_spec): + def setup_dependent_environment(self, env, dependent_spec): + self.setup_environment(env) + + def modify_module(self, module, spec, dependent_spec): """ Called before the install() method of dependents. diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 4c34d0308f5..5af9b585ea8 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -47,30 +47,21 @@ class Mpich(Package): provides('mpi@:3.0', when='@3:') provides('mpi@:1.3', when='@1:') - def environment_modifications(self, dependent_spec): - env = super(Mpich, self).environment_modifications(dependent_spec) + def setup_environment(self, env): + env.set_env('MPICH_CC', self.compiler.cc) + env.set_env('MPICH_CXX', self.compiler.cxx) + env.set_env('MPICH_F77', self.compiler.f77) + env.set_env('MPICH_F90', self.compiler.fc) + env.set_env('MPICH_FC', self.compiler.fc) - if dependent_spec is None: - # We are not using compiler wrappers - cc = self.compiler.cc - cxx = self.compiler.cxx - f77 = self.compiler.f77 - f90 = fc = self.compiler.fc - else: - # Spack compiler wrappers - cc = os.environ['CC'] - cxx = os.environ['CXX'] - f77 = os.environ['F77'] - f90 = fc = os.environ['FC'] + def setup_dependent_environment(self, env, dependent_spec): + env.set_env('MPICH_CC', spack_cc) + env.set_env('MPICH_CXX', spack_cxx) + env.set_env('MPICH_F77', spack_f77) + env.set_env('MPICH_F90', spack_f90) + env.set_env('MPICH_FC', spack_fc) - env.set_env('MPICH_CC', cc) - env.set_env('MPICH_CXX', cxx) - env.set_env('MPICH_F77', f77) - env.set_env('MPICH_F90', f90) - env.set_env('MPICH_FC', fc) - return env - - def module_modifications(self, module, spec, dep_spec): + def modify_module(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" # FIXME : is this necessary ? Shouldn't this be part of a contract with MPI providers? module.mpicc = join_path(self.prefix.bin, 'mpicc') diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index ecdea464420..7e7e5b2e2ee 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -40,7 +40,7 @@ def install(self, spec, prefix): make() make("install") - def module_modifications(self, module, spec, dependent_spec): + def modify_module(self, module, spec, dependent_spec): # TODO treat OS that are not Linux... lib_suffix = '.so' if '+shared' in spec['scalapack'] else '.a' diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 3a141704576..7783ca87666 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -41,14 +41,17 @@ class Openmpi(Package): def url_for_version(self, version): return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) - def environment_modifications(self, dependent_spec): - env = super(Openmpi, self).environment_modifications(dependent_spec) - # FIXME : the compilers should point to the current wrappers, not to generic cc etc. - env.set_env('OMPI_CC', 'cc') - env.set_env('OMPI_CXX', 'c++') - env.set_env('OMPI_FC', 'f90') - env.set_env('OMPI_F77', 'f77') - return env + def setup_environment(self, env): + env.set_env('OMPI_CC', self.compiler.cc) + env.set_env('OMPI_CXX', self.compiler.cxx) + env.set_env('OMPI_FC', self.compiler.fc) + env.set_env('OMPI_F77', self.compiler.f77) + + def setup_dependent_environment(self, env, dependent_spec): + env.set_env('OMPI_CC', spack_cc) + env.set_env('OMPI_CXX', spack_cxx) + env.set_env('OMPI_FC', spack_fc) + env.set_env('OMPI_F77', spack_f77) def install(self, spec, prefix): config_args = ["--prefix=%s" % prefix, diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index acb36517268..d47c1d1b2f3 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -89,24 +89,21 @@ def python_include_dir(self): def site_packages_dir(self): return os.path.join(self.python_lib_dir, 'site-packages') - def environment_modifications(self, extension_spec): - env = super(Python, self).environment_modifications(extension_spec) - if extension_spec is not None: - # Set PYTHONPATH to include site-packages dir for the - # extension and any other python extensions it depends on. - python_paths = [] - for d in extension_spec.traverse(): - if d.package.extends(self.spec): - python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) - env.set_env['PYTHONPATH'] = ':'.join(python_paths) - return env + def setup_dependent_environment(self, env, extension_spec): + # Set PYTHONPATH to include site-packages dir for the extension and any other python extensions it depends on. + python_paths = [] + for d in extension_spec.traverse(): + if d.package.extends(self.spec): + python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) + env.set_env['PYTHONPATH'] = ':'.join(python_paths) - def module_modifications(self, module, spec, ext_spec): - """Called before python modules' install() methods. + def modify_module(self, module, spec, ext_spec): + """ + Called before python modules' install() methods. In most cases, extensions will only need to have one line:: - python('setup.py', 'install', '--prefix=%s' % prefix) + python('setup.py', 'install', '--prefix=%s' % prefix) """ # Python extension builds can have a global python executable function if self.version >= Version("3.0.0") and self.version < Version("4.0.0"): diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 0adf352be21..35b9d684624 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -55,10 +55,8 @@ class Qt(Package): depends_on("mesa", when='@4:+mesa') depends_on("libxcb") - def environment_modifications(self, dependent_spec): - env = super(Qt, self).environment_modifications(dependent_spec) + def setup_environment(self, env): env.set_env['QTDIR'] = self.prefix - return env def patch(self): if self.spec.satisfies('@4'): diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py index 9caea30ef46..2d1da8c9af1 100644 --- a/var/spack/repos/builtin/packages/ruby/package.py +++ b/var/spack/repos/builtin/packages/ruby/package.py @@ -17,8 +17,7 @@ def install(self, spec, prefix): make() make("install") - def environment_modifications(self, extension_spec): - env = super(Ruby, self).environment_modifications(extension_spec) + def setup_dependent_environment(self, env, extension_spec): # Set GEM_PATH to include dependent gem directories ruby_paths = [] for d in extension_spec.traverse(): @@ -27,9 +26,8 @@ def environment_modifications(self, extension_spec): env.set_env('GEM_PATH', concatenate_paths(ruby_paths)) # The actual installation path for this gem env.set_env('GEM_HOME', extension_spec.prefix) - return env - def module_modifications(self, module, spec, ext_spec): + def modify_module(self, module, spec, ext_spec): """Called before ruby modules' install() methods. Sets GEM_HOME and GEM_PATH to values appropriate for the package being built. From 3da4d6664bbee38fde6faeffb39f889698ea320c Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 17 Mar 2016 15:38:08 +0100 Subject: [PATCH 603/910] environment : simplified hierarchy according to comments in review --- lib/spack/spack/environment.py | 53 +++++++++-------------------- lib/spack/spack/modules.py | 10 +++--- lib/spack/spack/package.py | 1 + lib/spack/spack/test/environment.py | 2 +- lib/spack/spack/util/environment.py | 2 -- 5 files changed, 24 insertions(+), 44 deletions(-) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index b557bf0bbcf..3ee917fcec6 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -3,73 +3,54 @@ import collections -class AttributeHolder(object): - """ - Policy that permits to store any kind of attribute on self. The attributes must be passed as key/value pairs - during the initialization of the instance. - """ - def __init__(self, **kwargs): - for key, value in kwargs.items(): - setattr(self, key, value) +class NameModifier(object): + def __init__(self, name, **kwargs): + self.name = name + self.args = {'name': name} + self.args.update(kwargs) -class SetEnv(AttributeHolder): +class NameValueModifier(object): def __init__(self, name, value, **kwargs): - super(SetEnv, self).__init__(**kwargs) self.name = name self.value = value + self.args = {'name': name, 'value': value} + self.args.update(kwargs) + +class SetEnv(NameValueModifier): def execute(self): os.environ[self.name] = str(self.value) -class UnsetEnv(AttributeHolder): - def __init__(self, name, **kwargs): - super(UnsetEnv, self).__init__(**kwargs) - self.name = name - +class UnsetEnv(NameModifier): def execute(self): os.environ.pop(self.name, None) # Avoid throwing if the variable was not set -class AppendPath(AttributeHolder): - def __init__(self, name, path, **kwargs): - super(AppendPath, self).__init__(**kwargs) - self.name = name - self.path = path - +class AppendPath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split(':') if environment_value else [] # TODO : Check if this is a valid directory name - directories.append(os.path.normpath(self.path)) + directories.append(os.path.normpath(self.value)) os.environ[self.name] = ':'.join(directories) -class PrependPath(AttributeHolder): - def __init__(self, name, path, **kwargs): - super(PrependPath, self).__init__(**kwargs) - self.name = name - self.path = path - +class PrependPath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split(':') if environment_value else [] # TODO : Check if this is a valid directory name - directories = [os.path.normpath(self.path)] + directories + directories = [os.path.normpath(self.value)] + directories os.environ[self.name] = ':'.join(directories) -class RemovePath(AttributeHolder): - def __init__(self, name, path, **kwargs): - super(RemovePath, self).__init__(**kwargs) - self.name = name - self.path = path - +class RemovePath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split(':') if environment_value else [] - directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.path)] + directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.value)] os.environ[self.name] = ':'.join(directories) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 09895d8c441..0f826c73634 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -178,7 +178,7 @@ def write_header(self, stream): def process_environment_command(self, env): for command in env: # FIXME : how should we handle errors here? - yield self.formats[type(command)].format(command) + yield self.formats[type(command)].format(**command.args) @property def file_name(self): @@ -203,8 +203,8 @@ class Dotkit(EnvModule): path = join_path(spack.share_path, "dotkit") formats = { - PrependPath: 'dk_alter {0.name} {0.path}\n', - SetEnv: 'dk_setenv {0.name} {0.value}\n' + PrependPath: 'dk_alter {name} {value}\n', + SetEnv: 'dk_setenv {name} {value}\n' } @property @@ -238,8 +238,8 @@ class TclModule(EnvModule): path = join_path(spack.share_path, "modules") formats = { - PrependPath: 'prepend-path {0.name} \"{0.path}\"\n', - SetEnv: 'setenv {0.name} \"{0.value}\"\n' + PrependPath: 'prepend-path {name} \"{value}\"\n', + SetEnv: 'setenv {name} \"{value}\"\n' } @property diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8e56dd1f976..680b26d69a7 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -979,6 +979,7 @@ def module(self): def setup_environment(self, env): """ + Called before the install() method of dependents. Return the list of environment modifications needed by dependents (or extensions). Default implementation does diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index d0b093b0543..3e03760c016 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -53,7 +53,7 @@ def test_extra_arguments(self): env = EnvironmentModifications() env.set_env('A', 'dummy value', who='Pkg1') for x in env: - assert hasattr(x, 'who') + assert 'who' in x.args env.apply_modifications() self.assertEqual('dummy value', os.environ['A']) diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index 1485992b0f1..55e653fd2fb 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -40,13 +40,11 @@ def env_flag(name): return False -# FIXME : remove this function ? def path_set(var_name, directories): path_str = ":".join(str(dir) for dir in directories) os.environ[var_name] = path_str -# FIXME : remove this function ? def path_put_first(var_name, directories): """Puts the provided directories first in the path, adding them if they're not already there. From 38c3c84969c25aaee8f643ff9770759c7e6d9c35 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 17 Mar 2016 17:37:33 +0100 Subject: [PATCH 604/910] environment : added caller information --- lib/spack/spack/environment.py | 27 +++++++++++++++++++++++++-- lib/spack/spack/modules.py | 8 ++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 3ee917fcec6..6d214595a3e 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -1,6 +1,7 @@ import os import os.path import collections +import inspect class NameModifier(object): @@ -32,7 +33,6 @@ class AppendPath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split(':') if environment_value else [] - # TODO : Check if this is a valid directory name directories.append(os.path.normpath(self.value)) os.environ[self.name] = ':'.join(directories) @@ -41,7 +41,6 @@ class PrependPath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split(':') if environment_value else [] - # TODO : Check if this is a valid directory name directories = [os.path.normpath(self.value)] + directories os.environ[self.name] = ':'.join(directories) @@ -57,6 +56,11 @@ def execute(self): class EnvironmentModifications(object): """ Keeps track of requests to modify the current environment. + + Each call to a method to modify the environment stores the extra information on the caller in the request: + - 'filename' : filename of the module where the caller is defined + - 'lineno': line number where the request occurred + - 'context' : line of code that issued the request that failed """ def __init__(self, other=None): @@ -85,6 +89,20 @@ def _check_other(other): if not isinstance(other, EnvironmentModifications): raise TypeError('other must be an instance of EnvironmentModifications') + def _get_outside_caller_attributes(self): + stack = inspect.stack() + try: + _, filename, lineno, _, context, index = stack[2] + context = context[index].strip() + except Exception: + filename, lineno, context = 'unknown file', 'unknown line', 'unknown context' + args = { + 'filename': filename, + 'lineno': lineno, + 'context': context + } + return args + def set_env(self, name, value, **kwargs): """ Stores in the current object a request to set an environment variable @@ -93,6 +111,7 @@ def set_env(self, name, value, **kwargs): name: name of the environment variable to be set value: value of the environment variable """ + kwargs.update(self._get_outside_caller_attributes()) item = SetEnv(name, value, **kwargs) self.env_modifications.append(item) @@ -103,6 +122,7 @@ def unset_env(self, name, **kwargs): Args: name: name of the environment variable to be set """ + kwargs.update(self._get_outside_caller_attributes()) item = UnsetEnv(name, **kwargs) self.env_modifications.append(item) @@ -114,6 +134,7 @@ def append_path(self, name, path, **kwargs): name: name of the path list in the environment path: path to be appended """ + kwargs.update(self._get_outside_caller_attributes()) item = AppendPath(name, path, **kwargs) self.env_modifications.append(item) @@ -125,6 +146,7 @@ def prepend_path(self, name, path, **kwargs): name: name of the path list in the environment path: path to be pre-pended """ + kwargs.update(self._get_outside_caller_attributes()) item = PrependPath(name, path, **kwargs) self.env_modifications.append(item) @@ -136,6 +158,7 @@ def remove_path(self, name, path, **kwargs): name: name of the path list in the environment path: path to be removed """ + kwargs.update(self._get_outside_caller_attributes()) item = RemovePath(name, path, **kwargs) self.env_modifications.append(item) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 0f826c73634..d192bbe004e 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -177,8 +177,12 @@ def write_header(self, stream): def process_environment_command(self, env): for command in env: - # FIXME : how should we handle errors here? - yield self.formats[type(command)].format(**command.args) + try: + yield self.formats[type(command)].format(**command.args) + except KeyError: + tty.warn('Cannot handle command of type {command} : skipping request'.format(command=type(command))) + tty.warn('{context} at {filename}:{lineno}'.format(**command.args)) + @property def file_name(self): From e673127263a03e63d6a64840d1071a788f0c099e Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 17 Mar 2016 18:11:23 +0100 Subject: [PATCH 605/910] package : added documentation --- lib/spack/spack/package.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 680b26d69a7..d0b94dbbeb9 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -976,32 +976,41 @@ def module(self): return __import__(self.__class__.__module__, fromlist=[self.__class__.__name__]) - def setup_environment(self, env): """ + Appends in `env` the list of environment modifications needed to use this package outside of spack. + Default implementation does nothing, but this can be overridden if the package needs a particular environment. + + Example : + + 1. A lot of Qt extensions need `QTDIR` set. This can be used to do that. + + Args: + env: list of environment modifications to be updated + """ + pass + + def setup_dependent_environment(self, env, dependent_spec): + """ Called before the install() method of dependents. - Return the list of environment modifications needed by dependents (or extensions). Default implementation does - nothing, but this can be overridden by an extendable package to set up the install environment for its - extensions. This is useful if there are some common steps to installing all extensions for a certain package. + Appends in `env` the list of environment modifications needed by dependents (or extensions) during the + installation of a package. The default implementation delegates to `setup_environment`, but can be overridden + if the modifications to the environment happen to be different from the one needed to use the package outside + of spack. + + This is useful if there are some common steps to installing all extensions for a certain package. Example : 1. Installing python modules generally requires `PYTHONPATH` to point to the lib/pythonX.Y/site-packages directory in the module's install prefix. This could set that variable. - 2. A lot of Qt extensions need `QTDIR` set. This can be used to do that. - Args: + env: list of environment modifications to be updated dependent_spec: dependent (or extension) of this spec - - Returns: - instance of environment modifications """ - pass - - def setup_dependent_environment(self, env, dependent_spec): self.setup_environment(env) def modify_module(self, module, spec, dependent_spec): @@ -1020,12 +1029,10 @@ def modify_module(self, module, spec, dependent_spec): """ pass - def install(self, spec, prefix): """Package implementations override this with their own build configuration.""" raise InstallError("Package %s provides no install method!" % self.name) - def do_uninstall(self, force=False): if not self.installed: raise InstallError(str(self.spec) + " is not installed.") From ac394718ec0aa67c468a8529b930eaade0bcbed1 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 17 Mar 2016 18:22:07 +0100 Subject: [PATCH 606/910] python : implemented possible solution --- lib/spack/spack/package.py | 11 ++++++++++- var/spack/repos/builtin/packages/py-nose/package.py | 3 ++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index d0b94dbbeb9..a7ab20137ea 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -37,6 +37,7 @@ import re import textwrap import time +import glob import llnl.util.tty as tty import spack @@ -55,7 +56,6 @@ from llnl.util.lang import * from llnl.util.link_tree import LinkTree from llnl.util.tty.log import log_output -from spack.environment import EnvironmentModifications from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive from spack.util.environment import dump_environment @@ -1236,6 +1236,15 @@ def rpath_args(self): return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) +class PythonExtension(Package): + def setup_dependent_environment(self, env, dependent_spec): + pass + + def setup_environment(self, env): + site_packages = glob.glob(join_path(self.spec.prefix.lib, "python*/site-packages")) + if site_packages: + env.prepend_path('PYTHONPATH', site_packages[0]) + def validate_package_url(url_string): """Determine whether spack can handle a particular URL or not.""" url = urlparse(url_string) diff --git a/var/spack/repos/builtin/packages/py-nose/package.py b/var/spack/repos/builtin/packages/py-nose/package.py index e7c6cf0264b..e817b8eb513 100644 --- a/var/spack/repos/builtin/packages/py-nose/package.py +++ b/var/spack/repos/builtin/packages/py-nose/package.py @@ -1,6 +1,7 @@ from spack import * +from spack.package import PythonExtension -class PyNose(Package): +class PyNose(PythonExtension): """nose extends the test loading and running features of unittest, making it easier to write, find and run tests.""" From e55880904376847ce767770344059eef1ba7707c Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 17 Mar 2016 19:32:31 +0100 Subject: [PATCH 607/910] python : fixed typo --- var/spack/repos/builtin/packages/python/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index d47c1d1b2f3..d46e1068b6c 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -95,7 +95,7 @@ def setup_dependent_environment(self, env, extension_spec): for d in extension_spec.traverse(): if d.package.extends(self.spec): python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) - env.set_env['PYTHONPATH'] = ':'.join(python_paths) + env.set_env('PYTHONPATH', ':'.join(python_paths)) def modify_module(self, module, spec, ext_spec): """ From 9e4c757f50ec98e55a74ad13fda70f12ce788891 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 17 Mar 2016 13:45:57 -0400 Subject: [PATCH 608/910] Update curl to 7.47.1 --- var/spack/repos/builtin/packages/curl/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/curl/package.py b/var/spack/repos/builtin/packages/curl/package.py index 9e684445c71..ab6305fc084 100644 --- a/var/spack/repos/builtin/packages/curl/package.py +++ b/var/spack/repos/builtin/packages/curl/package.py @@ -7,6 +7,7 @@ class Curl(Package): homepage = "http://curl.haxx.se" url = "http://curl.haxx.se/download/curl-7.46.0.tar.bz2" + version('7.47.1', '9ea3123449439bbd960cd25cf98796fb') version('7.46.0', '9979f989a2a9930d10f1b3deeabc2148') version('7.45.0', '62c1a352b28558f25ba6209214beadc8') version('7.44.0', '6b952ca00e5473b16a11f05f06aa8dae') From 90268876f7d740e5b7dcfd079168f248a337791a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 17 Mar 2016 18:49:58 -0700 Subject: [PATCH 609/910] Change sanity_check_[file|dir] to sanity_check_is_[file|dir], per #554 - Add documentation as well. --- lib/spack/docs/packaging_guide.rst | 84 +++++++++++++++---- lib/spack/spack/package.py | 22 +++-- .../repos/builtin/packages/libelf/package.py | 3 +- 3 files changed, 84 insertions(+), 25 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 169899212db..c1077e44977 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1559,11 +1559,11 @@ you ask for a particular spec. A user may have certain preferences for how packages should be concretized on their system. For example, one user may prefer packages built with OpenMPI and the Intel compiler. Another user may prefer -packages be built with MVAPICH and GCC. +packages be built with MVAPICH and GCC. Spack can be configured to prefer certain compilers, package versions, depends_on, and variants during concretization. -The preferred configuration can be controlled via the +The preferred configuration can be controlled via the ``~/.spack/packages.yaml`` file for user configuations, or the ``etc/spack/packages.yaml`` site configuration. @@ -1582,32 +1582,32 @@ Here's an example packages.yaml file that sets preferred packages: compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi] providers: mpi: [mvapich, mpich, openmpi] - + At a high level, this example is specifying how packages should be -concretized. The dyninst package should prefer using gcc 4.9 and +concretized. The dyninst package should prefer using gcc 4.9 and be built with debug options. The gperftools package should prefer version 2.2 over 2.4. Every package on the system should prefer mvapich for -its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). -These options are used to fill in implicit defaults. Any of them can be overwritten +its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). +These options are used to fill in implicit defaults. Any of them can be overwritten on the command line if explicitly requested. -Each packages.yaml file begins with the string ``packages:`` and +Each packages.yaml file begins with the string ``packages:`` and package names are specified on the next level. The special string ``all`` -applies settings to each package. Underneath each package name is -one or more components: ``compiler``, ``variants``, ``version``, -or ``providers``. Each component has an ordered list of spec +applies settings to each package. Underneath each package name is +one or more components: ``compiler``, ``variants``, ``version``, +or ``providers``. Each component has an ordered list of spec ``constraints``, with earlier entries in the list being preferred over later entries. -Sometimes a package installation may have constraints that forbid +Sometimes a package installation may have constraints that forbid the first concretization rule, in which case Spack will use the first legal concretization rule. Going back to the example, if a user -requests gperftools 2.3 or later, then Spack will install version 2.4 +requests gperftools 2.3 or later, then Spack will install version 2.4 as the 2.4 version of gperftools is preferred over 2.3. -An explicit concretization rule in the preferred section will always -take preference over unlisted concretizations. In the above example, +An explicit concretization rule in the preferred section will always +take preference over unlisted concretizations. In the above example, xlc isn't listed in the compiler list. Every listed compiler from gcc to pgi will thus be preferred over the xlc compiler. @@ -2160,6 +2160,62 @@ package, this allows us to avoid race conditions in the library's build system. +.. _sanity-checks: + +Sanity checking an intallation +-------------------------------- + +By default, Spack assumes that a build has failed if nothing is +written to the install prefix, and that it has succeeded if anything +(a file, a directory, etc.) is written to the install prefix after +``install()`` completes. + +Consider a simple autotools build like this: + +.. code-block:: python + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") + +If you are using using standard autotools or CMake, ``configure`` and +``make`` will not write anything to the install prefix. Only ``make +install`` writes the files, and only once the build is already +complete. Not all builds are like this. Many builds of scientific +software modify the install prefix *before* ``make install``. Builds +like this can falsely report that they were successfully installed if +an error occurs before the install is complete but after files have +been written to the ``prefix``. + + +``sanity_check_is_file`` and ``sanity_check_is_dir`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can optionally specify *sanity checks* to deal with this problem. +Add properties like this to your package: + +.. code-block:: python + + class MyPackage(Package): + ... + + sanity_check_is_file = ['include/libelf.h'] + sanity_check_is_dir = [lib] + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") + +Now, after ``install()`` runs, Spack will check whether +``$prefix/include/libelf.h`` exists and is a file, and whether +``$prefix/lib`` exists and is a directory. If the checks fail, then +the build will fail and the install prefix will be removed. If they +succeed, Spack considers the build succeeful and keeps the prefix in +place. + + .. _file-manipulation: File manipulation functions diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 02fb3e58348..291056a7b93 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -318,16 +318,17 @@ class SomePackage(Package): """Most packages are NOT extendable. Set to True if you want extensions.""" extendable = False - """List of prefix-relative file paths. If these do not exist after - install, or if they exist but are not files, sanity checks fail. + """List of prefix-relative file paths (or a single path). If these do + not exist after install, or if they exist but are not files, + sanity checks fail. """ - sanity_check_files = [] + sanity_check_is_file = [] - """List of prefix-relative directory paths. If these do not exist - after install, or if they exist but are not directories, sanity - checks will fail. + """List of prefix-relative directory paths (or a single path). If + these do not exist after install, or if they exist but are not + directories, sanity checks will fail. """ - sanity_check_dirs = [] + sanity_check_is_dir = [] def __init__(self, spec): @@ -966,14 +967,17 @@ def build_process(): def sanity_check_prefix(self): """This function checks whether install succeeded.""" def check_paths(path_list, filetype, predicate): + if isinstance(path_list, basestring): + path_list = [path_list] + for path in path_list: abs_path = os.path.join(self.prefix, path) if not predicate(abs_path): raise InstallError("Install failed for %s. No such %s in prefix: %s" % (self.name, filetype, path)) - check_paths(self.sanity_check_files, 'file', os.path.isfile) - check_paths(self.sanity_check_dirs, 'directory', os.path.isdir) + check_paths(self.sanity_check_is_file, 'file', os.path.isfile) + check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir) installed = set(os.listdir(self.prefix)) installed.difference_update(spack.install_layout.hidden_file_paths) diff --git a/var/spack/repos/builtin/packages/libelf/package.py b/var/spack/repos/builtin/packages/libelf/package.py index 0fcb56c164d..9f16708af5b 100644 --- a/var/spack/repos/builtin/packages/libelf/package.py +++ b/var/spack/repos/builtin/packages/libelf/package.py @@ -38,8 +38,7 @@ class Libelf(Package): provides('elf') - sanity_check_files = ['include/libelf.h'] - sanity_check_dirs = ['lib'] + sanity_check_is_file = 'include/libelf.h' def install(self, spec, prefix): configure("--prefix=" + prefix, From 802acb4d166a7ca3ff2db3c0a0c95a3ad8a6a81a Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 17 Mar 2016 13:43:40 -0400 Subject: [PATCH 610/910] Update PAPI to 5.4.3 --- var/spack/repos/builtin/packages/papi/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/papi/package.py b/var/spack/repos/builtin/packages/papi/package.py index 910e0aa9f9a..53d69e28d90 100644 --- a/var/spack/repos/builtin/packages/papi/package.py +++ b/var/spack/repos/builtin/packages/papi/package.py @@ -13,6 +13,7 @@ class Papi(Package): homepage = "http://icl.cs.utk.edu/papi/index.html" url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.4.1.tar.gz" + version('5.4.3', '3211b5a5bb389fe692370f5cf4cc2412') version('5.4.1', '9134a99219c79767a11463a76b0b01a2') version('5.3.0', '367961dd0ab426e5ae367c2713924ffb') From 1b279cd7ff29e09ef9afa1a328b964487b5b2030 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 17 Mar 2016 13:55:45 -0400 Subject: [PATCH 611/910] Update OpenBLAS to 0.2.16 --- var/spack/repos/builtin/packages/openblas/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 3c909360a41..e16d3fe89c1 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -5,6 +5,7 @@ class Openblas(Package): homepage = "http://www.openblas.net" url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + version('0.2.16', 'fef46ab92463bdbb1479dcec594ef6dc') version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') # virtual dependency From f3ea0420f80f418a0f3e628479c782f197e6a43e Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 17 Mar 2016 21:53:13 -0400 Subject: [PATCH 612/910] Import recent changes --- lib/spack/env/cc | 185 +++++++++++++++++------------------------------ 1 file changed, 65 insertions(+), 120 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index df6795c2640..c6a09724e9b 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -65,7 +65,7 @@ function die { } for param in $parameters; do - if [ -z "${!param}" ]; then + if [[ -z ${!param} ]]; then die "Spack compiler must be run from spack! Input $param was missing!" fi done @@ -78,10 +78,11 @@ done # 'command' is set based on the input command to $SPACK_[CC|CXX|F77|F90] # # 'mode' is set to one of: +# cpp preprocess # cc compile +# as assemble # ld link # ccld compile & link -# cpp preprocessor # vcheck version check # command=$(basename "$0") @@ -131,6 +132,9 @@ if [ -z "$mode" ]; then if [ "$arg" = -E ]; then mode=cpp break + elif [ "$arg" = -S ]; then + mode=as + break elif [ "$arg" = -c ]; then mode=cc break @@ -146,22 +150,25 @@ fi # Check that at least one of the real commands was actually selected, # otherwise we don't know what to execute. -if [ -z "$command" ]; then +if [[ -z $command ]]; then die "ERROR: Compiler '$SPACK_COMPILER_SPEC' does not support compiling $language programs." fi +if [ "$mode" == vcheck ] ; then + exec ${command} "$@" +fi + # Save original command for debug logging input_command="$@" args=("$@") -<<<<<<< HEAD # Dump parsed values for unit testing if asked for if [[ -n $SPACK_TEST_COMMAND ]]; then # # Now do real parsing of the command line args, trying hard to keep - # non-rpath linker arguments in the proper order w.r.t. other command - # line arguments. This is important for things like groups. + # non-rpath linker arguments in the proper order w.r.t. other command line + # arguments. This is important for things like groups. # includes=() libraries=() @@ -188,32 +195,44 @@ if [[ -n $SPACK_TEST_COMMAND ]]; then ;; -Wl,*) arg="${1#-Wl,}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ "$arg" = -rpath=* ]]; then - rpaths+=("${arg#-rpath=}") - elif [[ "$arg" = -rpath ]]; then + # TODO: Handle multiple -Wl, continuations of -Wl,-rpath + if [[ $arg == -rpath=* ]]; then + arg="${arg#-rpath=}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done + elif [[ $arg == -rpath,* ]]; then + arg="${arg#-rpath,}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done + elif [[ $arg == -rpath ]]; then shift; arg="$1" - if [[ "$arg" != -Wl,* ]]; then + if [[ $arg != '-Wl,'* ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi - rpaths+=("${arg#-Wl,}") + arg="${arg#-Wl,}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done else other_args+=("-Wl,$arg") fi ;; - -Xlinker,*) - arg="${1#-Xlinker,}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ "$arg" = -rpath=* ]]; then + -Xlinker) + shift; arg="$1"; + if [[ $arg = -rpath=* ]]; then rpaths+=("${arg#-rpath=}") - elif [[ "$arg" = -rpath ]]; then + elif [[ $arg = -rpath ]]; then shift; arg="$1" - if [[ "$arg" != -Xlinker,* ]]; then - die "-Xlinker,-rpath was not followed by -Xlinker,*" + if [[ $arg != -Xlinker ]]; then + die "-Xlinker -rpath was not followed by -Xlinker " fi - rpaths+=("${arg#-Xlinker,}") + shift; arg="$1" + rpaths+=("$arg") else - other_args+=("-Xlinker,$arg") + other_args+=("-Xlinker") + other_args+=("$arg") fi ;; *) @@ -222,88 +241,6 @@ if [[ -n $SPACK_TEST_COMMAND ]]; then esac shift done -======= -if [ "$mode" == vcheck ] ; then - exec ${command} "$@" -fi - -# -# Now do real parsing of the command line args, trying hard to keep -# non-rpath linker arguments in the proper order w.r.t. other command -# line arguments. This is important for things like groups. -# -includes=() -libraries=() -libs=() -rpaths=() -other_args=() - -while [ -n "$1" ]; do - case "$1" in - -I*) - arg="${1#-I}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - includes+=("$arg") - ;; - -L*) - arg="${1#-L}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - libraries+=("$arg") - ;; - -l*) - arg="${1#-l}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - libs+=("$arg") - ;; - -Wl,*) - arg="${1#-Wl,}" - # TODO: Handle multiple -Wl, continuations of -Wl,-rpath - if [[ $arg == -rpath=* ]]; then - arg="${arg#-rpath=}" - for rpath in ${arg//,/ }; do - rpaths+=("$rpath") - done - elif [[ $arg == -rpath,* ]]; then - arg="${arg#-rpath,}" - for rpath in ${arg//,/ }; do - rpaths+=("$rpath") - done - elif [[ $arg == -rpath ]]; then - shift; arg="$1" - if [[ $arg != '-Wl,'* ]]; then - die "-Wl,-rpath was not followed by -Wl,*" - fi - arg="${arg#-Wl,}" - for rpath in ${arg//,/ }; do - rpaths+=("$rpath") - done - else - other_args+=("-Wl,$arg") - fi - ;; - -Xlinker) - shift; arg="$1"; - if [[ $arg = -rpath=* ]]; then - rpaths+=("${arg#-rpath=}") - elif [[ $arg = -rpath ]]; then - shift; arg="$1" - if [[ $arg != -Xlinker ]]; then - die "-Xlinker -rpath was not followed by -Xlinker " - fi - shift; arg="$1" - rpaths+=("$arg") - else - other_args+=("-Xlinker") - other_args+=("$arg") - fi - ;; - *) - other_args+=("$1") - ;; - esac - shift -done ->>>>>>> develop IFS=$'\n' case "$SPACK_TEST_COMMAND" in @@ -329,8 +266,8 @@ done echo "${other_args[*]}" ;; *) - echo "ERROR: Unknown test command" - exit 1 ;; + die "ERROR: Unknown test command" + ;; esac exit fi @@ -338,11 +275,14 @@ fi # Read spack dependencies from the path environment variable IFS=':' read -ra deps <<< "$SPACK_DEPENDENCIES" for dep in "${deps[@]}"; do - if [ -d "$dep/include" ]; then - args=("-I$dep/include" "${args[@]}") + if [[ -d $dep/include ]]; then + if [[ $mode = cpp || $mode = cc || $mode = as || $mode = ccld ]]; then + args=("-I$dep/include" "${args[@]}") + fi fi - if [ -d "$dep/lib" ]; then + if [[ -d $dep/lib ]]; then + # libraries+=("$dep/lib") if [[ $mode = ccld ]]; then args=("-L$dep/lib" "-Wl,-rpath,$dep/lib" "${args[@]}") elif [[ $mode = ld ]]; then @@ -350,7 +290,7 @@ for dep in "${deps[@]}"; do fi fi - if [ -d "$dep/lib64" ]; then + if [[ -d $dep/lib64 ]]; then # libraries+=("$dep/lib64") if [[ $mode = ccld ]]; then args=("-L$dep/lib" "-Wl,-rpath,$dep/lib" "${args[@]}") @@ -362,8 +302,18 @@ done # Include all -L's and prefix/whatever dirs in rpath if [[ $mode = ccld ]]; then + # for dir in "${libraries[@]}"; do + # if [[ dir = $SPACK_INSTALL* ]]; then + # args=("-Wl,-rpath,$dir" "${args[@]}") + # fi + # done args=("-Wl,-rpath,$SPACK_PREFIX/lib" "-Wl,-rpath,$SPACK_PREFIX/lib64" "${args[@]}") elif [[ $mode = ld ]]; then + # for dir in "${libraries[@]}"; do + # if [[ dir = $SPACK_INSTALL* ]]; then + # args=("-rpath" "$dir" "${args[@]}") + # fi + # done args=("-rpath" "$SPACK_PREFIX/lib" "-rpath" "$SPACK_PREFIX/lib64" "${args[@]}") fi @@ -385,28 +335,23 @@ PATH="" for dir in "${env_path[@]}"; do remove="" for rm_dir in "${spack_env_dirs[@]}"; do - if [ "$dir" = "$rm_dir" ]; then remove=True; fi + if [[ $dir = $rm_dir ]]; then remove=True; fi done - if [ -z "$remove" ]; then - if [ -z "$PATH" ]; then - PATH="$dir" - else - PATH="$PATH:$dir" - fi + if [[ -z $remove ]]; then + PATH="${PATH:+$PATH:}$dir" fi done export PATH -full_command=("$command") -full_command+=("${args[@]}") +full_command=("$command" "${args[@]}") # # Write the input and output commands to debug logs if it's asked for. # -if [ "$SPACK_DEBUG" = "TRUE" ]; then +if [[ $SPACK_DEBUG = TRUE ]]; then input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.in.log" output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.out.log" - echo "$input_command" >> $input_log + echo "$input_command" >> $input_log echo "$mode ${full_command[@]}" >> $output_log fi From 4cf4bf3a0343fc93d6f0c14dd89956fa03b8f738 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Mar 2016 11:05:59 +0100 Subject: [PATCH 613/910] openssl : solved glitch to prevent spack to freeze when the network is unreachable --- .../repos/builtin/packages/openssl/package.py | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 70afaf4038a..22d538531a5 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -1,4 +1,4 @@ -import urllib +import urllib2 import llnl.util.tty as tty from spack import * @@ -37,17 +37,22 @@ def url_for_version(self, version): version_number = '.'.join([str(x) for x in version[:-1]]) older_url = older.format(version_number=version_number, version_full=version) latest_url = latest.format(version=version) - response = urllib.urlopen(latest.format(version=version)) - if response.getcode() == 404: - openssl_url = older_url - # Checks if we already warned the user for this particular version of OpenSSL. - # If not we display a warning message and mark this version - if not warnings_given_to_user.get(version, False): - tty.warn('This installation depends on an old version of OpenSSL, which may have known security issues. ') - tty.warn('Consider updating to the latest version of this package.') - tty.warn('More details at {homepage}'.format(homepage=Openssl.homepage)) - warnings_given_to_user[version] = True - else: + try: + response = urllib2.urlopen(latest.format(version=version), timeout=5) + if response.getcode() == 404: + openssl_url = older_url + # Checks if we already warned the user for this particular version of OpenSSL. + # If not we display a warning message and mark this version + if not warnings_given_to_user.get(version, False): + tty.warn('This installation depends on an old version of OpenSSL, which may have known security issues. ') + tty.warn('Consider updating to the latest version of this package.') + tty.warn('More details at {homepage}'.format(homepage=Openssl.homepage)) + warnings_given_to_user[version] = True + else: + openssl_url = latest_url + except urllib2.URLError: + tty.warn('Cannot connect to network to retrieve OpenSSL version. Using default url.') + warnings_given_to_user[version] = True openssl_url = latest_url # Store the computed URL openssl_urls[version] = openssl_url From 3b24db92b5ea64814b1294fc6c9c982bbaa9b3c6 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 17 Mar 2016 13:43:40 -0400 Subject: [PATCH 614/910] Update PAPI to 5.4.3 --- var/spack/repos/builtin/packages/papi/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/papi/package.py b/var/spack/repos/builtin/packages/papi/package.py index 910e0aa9f9a..53d69e28d90 100644 --- a/var/spack/repos/builtin/packages/papi/package.py +++ b/var/spack/repos/builtin/packages/papi/package.py @@ -13,6 +13,7 @@ class Papi(Package): homepage = "http://icl.cs.utk.edu/papi/index.html" url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.4.1.tar.gz" + version('5.4.3', '3211b5a5bb389fe692370f5cf4cc2412') version('5.4.1', '9134a99219c79767a11463a76b0b01a2') version('5.3.0', '367961dd0ab426e5ae367c2713924ffb') From 076cc764a78c387d0e0cc38047a9df8497416f90 Mon Sep 17 00:00:00 2001 From: Ben Morgan Date: Fri, 18 Mar 2016 11:26:22 +0000 Subject: [PATCH 615/910] Add additional suffixes for GCC compiler --- lib/spack/spack/compilers/gcc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 495b638a3aa..64214db32d3 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -40,7 +40,8 @@ class Gcc(Compiler): fc_names = ['gfortran'] # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. - suffixes = [r'-mp-\d\.\d'] + # Homebrew and Linuxes may build gcc with -X, -X.Y suffixes + suffixes = [r'-mp-\d\.\d', r'-\d\.\d', r'-\d'] # Named wrapper links within spack.build_env_path link_paths = {'cc' : 'gcc/gcc', From ec8cc2b52839007f0825815c8cfdee8f9a8629a6 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Mar 2016 14:40:53 +0100 Subject: [PATCH 616/910] PYTHONPATH : python patches methods for its extensions --- lib/spack/spack/build_environment.py | 4 ++- lib/spack/spack/cmd/uninstall.py | 12 +++---- lib/spack/spack/directory_layout.py | 2 +- lib/spack/spack/modules.py | 12 +++++-- lib/spack/spack/package.py | 10 ------ .../repos/builtin/packages/py-nose/package.py | 6 ++-- .../repos/builtin/packages/python/package.py | 34 +++++++++++++++++-- 7 files changed, 53 insertions(+), 27 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 68477145fef..c51fa584778 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -282,9 +282,11 @@ def setup_package(pkg): for mod in modules: set_module_variables_for_package(pkg, mod) - # Allow dependencies to set up environment as well. + # Allow dependencies to modify the module for dependency_spec in pkg.spec.traverse(root=False): dependency_spec.package.modify_module(pkg.module, dependency_spec, pkg.spec) + # Allow dependencies to set up environment as well + for dependency_spec in pkg.spec.traverse(root=False): dependency_spec.package.setup_dependent_environment(env, pkg.spec) # TODO : implement validation #validate(env) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index d01aa2136b0..2fab8f6f2a1 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -79,7 +79,7 @@ def uninstall(parser, args): try: # should work if package is known to spack pkgs.append(s.package) - except spack.repository.UnknownPackageError, e: + except spack.repository.UnknownPackageError as e: # The package.py file has gone away -- but still # want to uninstall. spack.Package(s).do_uninstall(force=True) @@ -94,11 +94,11 @@ def num_installed_deps(pkg): for pkg in pkgs: try: pkg.do_uninstall(force=args.force) - except PackageStillNeededError, e: + except PackageStillNeededError as e: tty.error("Will not uninstall %s" % e.spec.format("$_$@$%@$#", color=True)) - print - print "The following packages depend on it:" + print() + print("The following packages depend on it:") display_specs(e.dependents, long=True) - print - print "You can use spack uninstall -f to force this action." + print() + print("You can use spack uninstall -f to force this action.") sys.exit(1) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 39ee4e203d6..da8f1aa1bc0 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -150,7 +150,7 @@ def remove_install_directory(self, spec): if os.path.exists(path): try: shutil.rmtree(path) - except exceptions.OSError, e: + except exceptions.OSError as e: raise RemoveFailedError(spec, path, e) path = os.path.dirname(path) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index d192bbe004e..7303844229c 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -119,13 +119,13 @@ def __init__(cls, name, bases, dict): module_types[cls.name] = cls def __init__(self, spec=None): + self.spec = spec + self.pkg = spec.package # Just stored for convenience + # category in the modules system # TODO: come up with smarter category names. self.category = "spack" - self.spec = spec - self.pkg = spec.package # Just stored for convenience - # short description default is just the package + version # packages can provide this optional attribute self.short_description = spec.format("$_ $@") @@ -161,6 +161,12 @@ def write(self): # Environment modifications guessed by inspecting the installation prefix env = inspect_path(self.spec.prefix) + + # Let the extendee modify their extensions before asking for package-specific modifications + for extendee in self.pkg.extendees: + extendee_spec = self.spec[extendee] + extendee_spec.package.modify_module(self.pkg, extendee_spec, self.spec) + # Package-specific environment modifications self.spec.package.setup_environment(env) # TODO : implement site-specific modifications and filters diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 185e3ad2eea..72c84ec624f 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1261,16 +1261,6 @@ def rpath_args(self): """Get the rpath args as a string, with -Wl,-rpath, for each element.""" return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) - -class PythonExtension(Package): - def setup_dependent_environment(self, env, dependent_spec): - pass - - def setup_environment(self, env): - site_packages = glob.glob(join_path(self.spec.prefix.lib, "python*/site-packages")) - if site_packages: - env.prepend_path('PYTHONPATH', site_packages[0]) - def validate_package_url(url_string): """Determine whether spack can handle a particular URL or not.""" url = urlparse(url_string) diff --git a/var/spack/repos/builtin/packages/py-nose/package.py b/var/spack/repos/builtin/packages/py-nose/package.py index e817b8eb513..4fee99098ed 100644 --- a/var/spack/repos/builtin/packages/py-nose/package.py +++ b/var/spack/repos/builtin/packages/py-nose/package.py @@ -1,12 +1,12 @@ from spack import * -from spack.package import PythonExtension -class PyNose(PythonExtension): + +class PyNose(Package): """nose extends the test loading and running features of unittest, making it easier to write, find and run tests.""" homepage = "https://pypi.python.org/pypi/nose" - url = "https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz" + url = "https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz" version('1.3.4', '6ed7169887580ddc9a8e16048d38274d') version('1.3.6', '0ca546d81ca8309080fc80cb389e7a16') diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index d46e1068b6c..c445d26369e 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -1,11 +1,14 @@ +import functools +import glob +import inspect import os import re from contextlib import closing -from llnl.util.lang import match_predicate -from spack.util.environment import * -from spack import * import spack +from llnl.util.lang import match_predicate +from spack import * +from spack.util.environment import * class Python(Package): @@ -111,6 +114,31 @@ def modify_module(self, module, spec, ext_spec): else: module.python = Executable(join_path(spec.prefix.bin, 'python')) + # The code below patches the any python extension to have good defaults for `setup_dependent_environment` and + # `setup_environment` only if the extension didn't override any of these functions explicitly. + def _setup_env(self, env): + site_packages = glob.glob(join_path(self.spec.prefix.lib, "python*/site-packages")) + if site_packages: + env.prepend_path('PYTHONPATH', site_packages[0]) + + def _setup_denv(self, env, extension_spec): + pass + + pkg_cls = type(ext_spec.package) # Retrieve the type we may want to patch + if 'python' in pkg_cls.extendees: + # List of overrides we are interested in + interesting_overrides = ['setup_environment', 'setup_dependent_environment'] + overrides_found = [ + (name, defining_cls) for name, _, defining_cls, _, in inspect.classify_class_attrs(pkg_cls) + if + name in interesting_overrides and # The attribute has the right name + issubclass(defining_cls, Package) and defining_cls is not Package # and is an actual override + ] + if not overrides_found: + # If no override were found go on patching + pkg_cls.setup_environment = functools.wraps(Package.setup_environment)(_setup_env) + pkg_cls.setup_dependent_environment = functools.wraps(Package.setup_dependent_environment)(_setup_denv) + # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) module.python_include_dir = os.path.join(ext_spec.prefix, self.python_include_dir) From 5f6f2d5f51fc3700dce13c6aa035943bbc42f6f7 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Mar 2016 14:42:24 +0100 Subject: [PATCH 617/910] Revert "openssl : solved glitch to prevent spack to freeze when the network is unreachable" This reverts commit 4cf4bf3a0343fc93d6f0c14dd89956fa03b8f738. --- .../repos/builtin/packages/openssl/package.py | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 22d538531a5..70afaf4038a 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -1,4 +1,4 @@ -import urllib2 +import urllib import llnl.util.tty as tty from spack import * @@ -37,22 +37,17 @@ def url_for_version(self, version): version_number = '.'.join([str(x) for x in version[:-1]]) older_url = older.format(version_number=version_number, version_full=version) latest_url = latest.format(version=version) - try: - response = urllib2.urlopen(latest.format(version=version), timeout=5) - if response.getcode() == 404: - openssl_url = older_url - # Checks if we already warned the user for this particular version of OpenSSL. - # If not we display a warning message and mark this version - if not warnings_given_to_user.get(version, False): - tty.warn('This installation depends on an old version of OpenSSL, which may have known security issues. ') - tty.warn('Consider updating to the latest version of this package.') - tty.warn('More details at {homepage}'.format(homepage=Openssl.homepage)) - warnings_given_to_user[version] = True - else: - openssl_url = latest_url - except urllib2.URLError: - tty.warn('Cannot connect to network to retrieve OpenSSL version. Using default url.') - warnings_given_to_user[version] = True + response = urllib.urlopen(latest.format(version=version)) + if response.getcode() == 404: + openssl_url = older_url + # Checks if we already warned the user for this particular version of OpenSSL. + # If not we display a warning message and mark this version + if not warnings_given_to_user.get(version, False): + tty.warn('This installation depends on an old version of OpenSSL, which may have known security issues. ') + tty.warn('Consider updating to the latest version of this package.') + tty.warn('More details at {homepage}'.format(homepage=Openssl.homepage)) + warnings_given_to_user[version] = True + else: openssl_url = latest_url # Store the computed URL openssl_urls[version] = openssl_url From 67ca2c704b9c367b41ce29342a06427e156a30e9 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Mar 2016 15:18:26 +0100 Subject: [PATCH 618/910] modules : fixed bug in `modify_module` arguments --- lib/spack/spack/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 7303844229c..f5bf2138456 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -165,7 +165,7 @@ def write(self): # Let the extendee modify their extensions before asking for package-specific modifications for extendee in self.pkg.extendees: extendee_spec = self.spec[extendee] - extendee_spec.package.modify_module(self.pkg, extendee_spec, self.spec) + extendee_spec.package.modify_module(self.pkg.module, extendee_spec, self.spec) # Package-specific environment modifications self.spec.package.setup_environment(env) From ccd90df62ffd69b5c3b4e1f8dbaf82146721a52a Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Mar 2016 15:41:14 +0100 Subject: [PATCH 619/910] modules : turned category into a property (logic can be extended later) --- lib/spack/spack/modules.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index f5bf2138456..bb1a444e5a6 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -122,10 +122,6 @@ def __init__(self, spec=None): self.spec = spec self.pkg = spec.package # Just stored for convenience - # category in the modules system - # TODO: come up with smarter category names. - self.category = "spack" - # short description default is just the package + version # packages can provide this optional attribute self.short_description = spec.format("$_ $@") @@ -137,6 +133,17 @@ def __init__(self, spec=None): if self.spec.package.__doc__: self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) + @property + def category(self): + # Anything defined at the package level takes precedence + if hasattr(self.pkg, 'category'): + return self.pkg.category + # Extensions + for extendee in self.pkg.extendees: + return '{extendee} extension'.format(extendee=extendee) + # Not very descriptive fallback + return 'spack installed package' + # @property # def paths(self): # # Add python path unless it's an actual python installation From 1e468c55414822365b4ba7f7c52fbfdb5f30d3b2 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Mar 2016 16:02:44 +0100 Subject: [PATCH 620/910] modules : added formats mapping --- lib/spack/spack/modules.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index bb1a444e5a6..0378110c8cb 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -256,7 +256,10 @@ class TclModule(EnvModule): formats = { PrependPath: 'prepend-path {name} \"{value}\"\n', - SetEnv: 'setenv {name} \"{value}\"\n' + AppendPath: 'append-path {name} \"{value}\"\n', + RemovePath: 'remove-path {name} \"{value}\"\n', + SetEnv: 'setenv {name} \"{value}\"\n', + UnsetEnv: 'unsetenv {name}\n' } @property From 491babd5cd435fc307b6e977b850e4e64d2b0ccf Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Mar 2016 17:09:20 +0100 Subject: [PATCH 621/910] env modifications : added a validation rule --- lib/spack/spack/build_environment.py | 6 +++--- lib/spack/spack/environment.py | 30 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index c51fa584778..59b234624cf 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -34,8 +34,9 @@ import sys import spack +import llnl.util.tty as tty from llnl.util.filesystem import * -from spack.environment import EnvironmentModifications, concatenate_paths +from spack.environment import EnvironmentModifications, concatenate_paths, validate from spack.util.environment import * from spack.util.executable import Executable, which @@ -288,8 +289,7 @@ def setup_package(pkg): # Allow dependencies to set up environment as well for dependency_spec in pkg.spec.traverse(root=False): dependency_spec.package.setup_dependent_environment(env, pkg.spec) - # TODO : implement validation - #validate(env) + validate(env, tty.warn) env.apply_modifications() diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 6d214595a3e..74aef57fe88 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -202,3 +202,33 @@ def concatenate_paths(paths): string """ return ':'.join(str(item) for item in paths) + + +def set_or_unset_not_first(variable, changes, errstream): + """ + Check if we are going to set or unset something after other modifications have already been requested + """ + indexes = [ii for ii, item in enumerate(changes) if ii != 0 and type(item) in [SetEnv, UnsetEnv]] + if indexes: + good = '\t \t{context} at {filename}:{lineno}' + nogood = '\t--->\t{context} at {filename}:{lineno}' + errstream('Suspicious requests to set or unset the variable \'{var}\' found'.format(var=variable)) + for ii, item in enumerate(changes): + print_format = nogood if ii in indexes else good + errstream(print_format.format(**item.args)) + + +def validate(env, errstream): + """ + Validates the environment modifications to check for the presence of suspicious patterns. Prompts a warning for + everything that was found + + Current checks: + - set or unset variables after other changes on the same variable + + Args: + env: list of environment modifications + """ + modifications = env.group_by_name() + for variable, list_of_changes in sorted(modifications.items()): + set_or_unset_not_first(variable, list_of_changes, errstream) From 1fa38689d87480bb0d291af9b9cf0dbcf7557eb5 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 18 Mar 2016 09:35:56 -0700 Subject: [PATCH 622/910] Created flatten_dependencies function that dummy packages can use to create sane install environments. --- lib/spack/spack/directory_layout.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 242eb1afa04..9f174040627 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -33,6 +33,7 @@ import llnl.util.tty as tty from llnl.util.filesystem import join_path, mkdirp +from llnl.util.link_tree import * from spack.spec import Spec from spack.error import SpackError @@ -131,6 +132,23 @@ def remove_extension(self, spec, ext_spec): raise NotImplementedError() + def flatten_dependencies(self, spec, flat_dir): + """Make each dependency of spec present in dir via symlink.""" + for dep in spec.traverse(root=False): + name = dep.name + + dep_path = self.path_for_spec(dep) + dep_files = LinkTree(dep_path) + + os.mkdir(flat_dir+'/'+name) + + conflict = dep_files.find_conflict(flat_dir+'/'+name) + if conflict: + raise DependencyConflictError(conflict) + + dep_files.merge(flat_dir+'/'+name) + + def path_for_spec(self, spec): """Return an absolute path from the root to a directory for the spec.""" _check_concrete(spec) From 80495e50f9b3a5de6d0aadb054a770a685939e79 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 18 Mar 2016 09:46:18 -0700 Subject: [PATCH 623/910] added error class for error that should never come up --- lib/spack/spack/directory_layout.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 9f174040627..dfd0dc42e4f 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -498,3 +498,10 @@ def __init__(self, spec, ext_spec): super(NoSuchExtensionError, self).__init__( "%s cannot be removed from %s because it's not activated."% ( ext_spec.short_spec, spec.short_spec)) + +class DependencyConflictError(SpackError): + """Raised when the dependencies cannot be flattened as asked for.""" + def __init__(self, conflict): + super(DependencyConflictError, self).__init__( + "%s conflicts with another file in the flattened directory." %( + conflict)) From 6acb830263e663b57bc8c077b348c897b0d1f612 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 18 Mar 2016 13:03:55 -0500 Subject: [PATCH 624/910] Add support for .tar files --- lib/spack/spack/url.py | 2 +- lib/spack/spack/util/compression.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index ad551a6dedb..f51f05cad7d 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -142,7 +142,7 @@ def split_url_extension(path): def downloaded_file_extension(path): """This returns the type of archive a URL refers to. This is - sometimes confusing becasue of URLs like: + sometimes confusing because of URLs like: (1) https://github.com/petdance/ack/tarball/1.93_02 diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py index ea1f233bce8..5ae58674280 100644 --- a/lib/spack/spack/util/compression.py +++ b/lib/spack/spack/util/compression.py @@ -27,13 +27,12 @@ from itertools import product from spack.util.executable import which -# Supported archvie extensions. +# Supported archive extensions. PRE_EXTS = ["tar"] EXTS = ["gz", "bz2", "xz", "Z", "zip", "tgz"] -# Add EXTS last so that .tar.gz is matched *before* tar.gz -ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + EXTS - +# Add PRE_EXTS and EXTS last so that .tar.gz is matched *before* .tar or .gz +ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + PRE_EXTS + EXTS def allowed_archive(path): return any(path.endswith(t) for t in ALLOWED_ARCHIVE_TYPES) From 76672a4e34e37aec0223e67d140fd19e9115b5a6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 18 Mar 2016 11:28:44 -0700 Subject: [PATCH 625/910] Refactoring flat_install --- lib/spack/spack/__init__.py | 3 +++ lib/spack/spack/directory_layout.py | 24 ----------------------- lib/spack/spack/package.py | 30 +++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index ab78ecef308..155e190597b 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -174,3 +174,6 @@ import spack.util.executable from spack.util.executable import * __all__ += spack.util.executable.__all__ + +from spack.package import flat_install, flatten_dependencies, DependencyConflictError +__all__ += ['flat_install', 'flatten_dependencies', 'DependencyConflictError'] diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index dfd0dc42e4f..e275031387c 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -132,23 +132,6 @@ def remove_extension(self, spec, ext_spec): raise NotImplementedError() - def flatten_dependencies(self, spec, flat_dir): - """Make each dependency of spec present in dir via symlink.""" - for dep in spec.traverse(root=False): - name = dep.name - - dep_path = self.path_for_spec(dep) - dep_files = LinkTree(dep_path) - - os.mkdir(flat_dir+'/'+name) - - conflict = dep_files.find_conflict(flat_dir+'/'+name) - if conflict: - raise DependencyConflictError(conflict) - - dep_files.merge(flat_dir+'/'+name) - - def path_for_spec(self, spec): """Return an absolute path from the root to a directory for the spec.""" _check_concrete(spec) @@ -498,10 +481,3 @@ def __init__(self, spec, ext_spec): super(NoSuchExtensionError, self).__init__( "%s cannot be removed from %s because it's not activated."% ( ext_spec.short_spec, spec.short_spec)) - -class DependencyConflictError(SpackError): - """Raised when the dependencies cannot be flattened as asked for.""" - def __init__(self, conflict): - super(DependencyConflictError, self).__init__( - "%s conflicts with another file in the flattened directory." %( - conflict)) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 972a0410b9a..1a962268cf4 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1211,6 +1211,28 @@ def rpath_args(self): return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) +def flat_install(pkg, spec, prefix): + """Execute a dummy install and flatten dependencies""" + os.mkdir(prefix+'/libs') + flatten_dependencies(spec, prefix+'/libs') + +def flatten_dependencies(spec, flat_dir): + """Make each dependency of spec present in dir via symlink.""" + for dep in spec.traverse(root=False): + name = dep.name + + dep_path = spack.install_layout.path_for_spec(dep) + dep_files = LinkTree(dep_path) + + os.mkdir(flat_dir+'/'+name) + + conflict = dep_files.find_conflict(flat_dir+'/'+name) + if conflict: + raise DependencyConflictError(conflict) + + dep_files.merge(flat_dir+'/'+name) + + def validate_package_url(url_string): """Determine whether spack can handle a particular URL or not.""" url = urlparse(url_string) @@ -1348,3 +1370,11 @@ def __init__(self, path): class ActivationError(ExtensionError): def __init__(self, msg, long_msg=None): super(ActivationError, self).__init__(msg, long_msg) + + +class DependencyConflictError(spack.error.SpackError): + """Raised when the dependencies cannot be flattened as asked for.""" + def __init__(self, conflict): + super(DependencyConflictError, self).__init__( + "%s conflicts with another file in the flattened directory." %( + conflict)) From af7e3cadde02b2ab9b6de2c588a41663141928dd Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 18 Mar 2016 11:34:07 -0700 Subject: [PATCH 626/910] cleanup --- lib/spack/spack/directory_layout.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index e275031387c..242eb1afa04 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -33,7 +33,6 @@ import llnl.util.tty as tty from llnl.util.filesystem import join_path, mkdirp -from llnl.util.link_tree import * from spack.spec import Spec from spack.error import SpackError From 151b04637ef01b8cde5967df6165bd245311b90a Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 18 Mar 2016 11:55:31 -0700 Subject: [PATCH 627/910] changed function name and removed hardcoded libs dir --- lib/spack/spack/__init__.py | 4 ++-- lib/spack/spack/package.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 155e190597b..5d62d597cb7 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -175,5 +175,5 @@ from spack.util.executable import * __all__ += spack.util.executable.__all__ -from spack.package import flat_install, flatten_dependencies, DependencyConflictError -__all__ += ['flat_install', 'flatten_dependencies', 'DependencyConflictError'] +from spack.package import install_dependency_symlinks, flatten_dependencies, DependencyConflictError +__all__ += ['install_dependency_symlinks', 'flatten_dependencies', 'DependencyConflictError'] diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 1a962268cf4..4c458522e05 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1211,10 +1211,9 @@ def rpath_args(self): return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) -def flat_install(pkg, spec, prefix): +def install_dependency_symlinks(pkg, spec, prefix): """Execute a dummy install and flatten dependencies""" - os.mkdir(prefix+'/libs') - flatten_dependencies(spec, prefix+'/libs') + flatten_dependencies(spec, prefix) def flatten_dependencies(spec, flat_dir): """Make each dependency of spec present in dir via symlink.""" From 23f3f1adfe01f7b1301959e7e54584724679c7b3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 18 Mar 2016 13:32:37 -0700 Subject: [PATCH 628/910] Make InstallError and ExternalPackageError visiible in packages. --- lib/spack/spack/__init__.py | 8 ++++++-- lib/spack/spack/package.py | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 0ba42bbbfcd..aee11f061f3 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -189,5 +189,9 @@ from spack.util.executable import * __all__ += spack.util.executable.__all__ -from spack.package import install_dependency_symlinks, flatten_dependencies, DependencyConflictError -__all__ += ['install_dependency_symlinks', 'flatten_dependencies', 'DependencyConflictError'] +from spack.package import \ + install_dependency_symlinks, flatten_dependencies, DependencyConflictError, \ + InstallError, ExternalPackageError +__all__ += [ + 'install_dependency_symlinks', 'flatten_dependencies', 'DependencyConflictError', + 'InstallError', 'ExternalPackageError'] diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b488e4c49db..dafad0b184a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1351,6 +1351,10 @@ def __init__(self, message, long_msg=None): super(InstallError, self).__init__(message, long_msg) +class ExternalPackageError(InstallError): + """Raised by install() when a package is only for external use.""" + + class PackageStillNeededError(InstallError): """Raised when package is still needed by another on uninstall.""" def __init__(self, spec, dependents): From e9126baaabbfaa5e12641b3d44a5c5fd49e43ca3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 18 Mar 2016 13:42:45 -0700 Subject: [PATCH 629/910] Add docs for InstallError. --- lib/spack/docs/packaging_guide.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index c1077e44977..519c0da2329 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1844,6 +1844,20 @@ dedicated process. .. _prefix-objects: + +Failing the build +---------------------- + +Sometimes you don't want a package to successfully install unless some +condition is true. You can explicitly cause the build to fail from +``install()`` by raising an ``InstallError``, for example: + +.. code-block:: python + + if spec.architecture.startswith('darwin'): + raise InstallError('This package does not build on Mac OS X!') + + Prefix objects ---------------------- From f5f7d47be1a5fd8488ed4cdc692173fe136d9791 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 19 Mar 2016 18:42:06 -0400 Subject: [PATCH 630/910] Support Bash "-u" mode --- share/spack/setup-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 586a5b836b1..764af684001 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -141,7 +141,7 @@ function _spack_pathadd { fi # Do the actual prepending here. - eval "_pa_oldvalue=\$${_pa_varname}" + eval "_pa_oldvalue=\${${_pa_varname}:-}" if [ -d "$_pa_new_path" ] && [[ ":$_pa_oldvalue:" != *":$_pa_new_path:"* ]]; then if [ -n "$_pa_oldvalue" ]; then From ed0b5e649dd6bc480e3faa6ffcca121eb9e264b8 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Fri, 18 Mar 2016 15:54:26 -0600 Subject: [PATCH 631/910] + Provide fpic variant for netlib-lapack. --- var/spack/repos/builtin/packages/netlib-lapack/package.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py index 741f4af4215..78c5a053fe5 100644 --- a/var/spack/repos/builtin/packages/netlib-lapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -20,13 +20,13 @@ class NetlibLapack(Package): version('3.3.1', 'd0d533ec9a5b74933c2a1e84eedc58b4') variant('shared', default=False, description="Build shared library version") + variant('fpic', default=False, description="Build with -fpic compiler option") # virtual dependency provides('lapack') # blas is a virtual dependency. depends_on('blas') - depends_on('cmake') # Doesn't always build correctly in parallel @@ -37,24 +37,23 @@ def get_blas_libs(self): blas = self.spec['netlib-blas'] return [join_path(blas.prefix.lib, 'blas.a')] - @when('^atlas') def get_blas_libs(self): blas = self.spec['atlas'] return [join_path(blas.prefix.lib, l) for l in ('libf77blas.a', 'libatlas.a')] - def install(self, spec, prefix): blas_libs = ";".join(self.get_blas_libs()) cmake_args = [".", '-DBLAS_LIBRARIES=' + blas_libs] if '+shared' in spec: cmake_args.append('-DBUILD_SHARED_LIBS=ON') + if '+fpic' in spec: + cmake_args.append('-DCMAKE_POSITION_INDEPENDENT_CODE=ON') cmake_args += std_cmake_args cmake(*cmake_args) make() make("install") - From 620c120503c8c23d10feb14e14d37dfc36f4befb Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sat, 19 Mar 2016 19:32:44 +0100 Subject: [PATCH 632/910] fix openblas suffix for OS-X --- var/spack/repos/builtin/packages/openblas/package.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index e16d3fe89c1..781a1e2ec8f 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -1,4 +1,5 @@ from spack import * +import sys class Openblas(Package): """OpenBLAS: An optimized BLAS library""" @@ -16,13 +17,14 @@ def install(self, spec, prefix): make('libs', 'netlib', 'shared', 'CC=cc', 'FC=f77') make('install', "PREFIX='%s'" % prefix) + lib_dsuffix = 'dylib' if sys.platform == 'darwin' else 'so' # Blas virtual package should provide blas.a and libblas.a with working_dir(prefix.lib): symlink('libopenblas.a', 'blas.a') symlink('libopenblas.a', 'libblas.a') - symlink('libopenblas.so', 'libblas.so') + symlink('libopenblas.%s' % lib_dsuffix, 'libblas.%s' % lib_dsuffix) # Lapack virtual package should provide liblapack.a with working_dir(prefix.lib): symlink('libopenblas.a', 'liblapack.a') - symlink('libopenblas.so', 'liblapack.so') + symlink('libopenblas.%s' % lib_dsuffix, 'liblapack.%s' % lib_dsuffix) From 95ad2875b6bb4dc15eb43b44c1dbdd521c843119 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sun, 20 Mar 2016 09:25:02 +0100 Subject: [PATCH 633/910] fix scalapack suffix for osx --- .../builtin/packages/netlib-scalapack/package.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 22d538560e8..a5b6eedafbe 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -1,8 +1,9 @@ from spack import * +import sys class NetlibScalapack(Package): """ScaLAPACK is a library of high-performance linear algebra routines for parallel distributed memory machines""" - + homepage = "http://www.netlib.org/scalapack/" url = "http://www.netlib.org/scalapack/scalapack-2.0.2.tgz" @@ -32,17 +33,17 @@ def install(self, spec, prefix): "-DCMAKE_C_FLAGS=-fPIC", "-DCMAKE_Fortran_FLAGS=-fPIC" ]) - + options.extend(std_cmake_args) - + with working_dir('spack-build', create=True): cmake('..', *options) make() make("install") def setup_dependent_environment(self, module, spec, dependent_spec): - # TODO treat OS that are not Linux... - lib_suffix = '.so' if '+shared' in spec['scalapack'] else '.a' + lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' + lib_suffix = lib_dsuffix if '+shared' in spec['scalapack'] else '.a' spec['scalapack'].fc_link = '-L%s -lscalapack' % spec['scalapack'].prefix.lib spec['scalapack'].cc_link = spec['scalapack'].fc_link From db61a09cf14cc791d7692aac0329e8958b228fb6 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 18 Mar 2016 16:28:36 +0100 Subject: [PATCH 634/910] fix mumps: when installed with parmetis has to be linked with metis --- .../repos/builtin/packages/mumps/package.py | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index 44a37903cc8..5c120c37df8 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -20,10 +20,10 @@ class Mumps(Package): variant('complex', default=True, description='Activate the compilation of cmumps and/or zmumps') variant('idx64', default=False, description='Use int64_t/integer*8 as default index type') - + depends_on('scotch + esmumps', when='~ptscotch+scotch') depends_on('scotch + esmumps + mpi', when='+ptscotch') - depends_on('metis', when='~parmetis+metis') + depends_on('metis', when='+metis') depends_on('parmetis', when="+parmetis") depends_on('blas') depends_on('lapack') @@ -38,11 +38,11 @@ class Mumps(Package): def write_makefile_inc(self): if ('+parmetis' in self.spec or '+ptscotch' in self.spec) and '+mpi' not in self.spec: raise RuntimeError('You cannot use the variants parmetis or ptscotch without mpi') - + makefile_conf = ["LIBBLAS = -L%s -lblas" % self.spec['blas'].prefix.lib] orderings = ['-Dpord'] - + if '+ptscotch' in self.spec or '+scotch' in self.spec: join_lib = ' -l%s' % ('pt' if '+ptscotch' in self.spec else '') makefile_conf.extend( @@ -54,15 +54,19 @@ def write_makefile_inc(self): if '+ptscotch' in self.spec: orderings.append('-Dptscotch') - if '+parmetis' in self.spec or '+metis' in self.spec: + if '+parmetis' in self.spec and '+metis' in self.spec: libname = 'parmetis' if '+parmetis' in self.spec else 'metis' makefile_conf.extend( - ["IMETIS = -I%s" % self.spec[libname].prefix.include, - "LMETIS = -L%s -l%s" % (self.spec[libname].prefix.lib, libname)]) + ["IMETIS = -I%s" % self.spec['parmetis'].prefix.include, + "LMETIS = -L%s -l%s -L%s -l%s" % (self.spec['parmetis'].prefix.lib, 'parmetis',self.spec['metis'].prefix.lib, 'metis')]) + + orderings.append('-Dparmetis') + elif '+metis' in self.spec: + makefile_conf.extend( + ["IMETIS = -I%s" % self.spec['metis'].prefix.include, + "LMETIS = -L%s -l%s" % (self.spec['metis'].prefix.lib, 'metis')]) orderings.append('-Dmetis') - if '+parmetis' in self.spec: - orderings.append('-Dparmetis') makefile_conf.append("ORDERINGSF = %s" % (' '.join(orderings))) @@ -101,12 +105,12 @@ def write_makefile_inc(self): # compiler possible values are -DAdd_, -DAdd__ and/or -DUPPER makefile_conf.append("CDEFS = -DAdd_") - + makefile_inc_template = join_path(os.path.dirname(self.module.__file__), 'Makefile.inc') with open(makefile_inc_template, "r") as fh: makefile_conf.extend(fh.read().split('\n')) - + with working_dir('.'): with open("Makefile.inc", "w") as fh: makefile_inc = '\n'.join(makefile_conf) @@ -130,7 +134,7 @@ def install(self, spec, prefix): make_libs.append('zexamples') self.write_makefile_inc() - + make(*make_libs) install_tree('lib', prefix.lib) From d493658a583231eeb778613cf7b6c8c8641c63a0 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Sun, 20 Mar 2016 15:18:36 -0700 Subject: [PATCH 635/910] cmake package: add variant for openssl support The openssl variant defaults to true to preserve spack's current CMake configuration, which is using OpenSSL. Signed-off-by: Cyrus Harrison --- var/spack/repos/builtin/packages/cmake/package.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index cc93c7067c3..be930a90abe 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -38,6 +38,7 @@ class Cmake(Package): version('2.8.10.2', '097278785da7182ec0aea8769d06860c') variant('ncurses', default=True, description='Enables the build of the ncurses gui') + variant('openssl', default=True, description="Enables CMake's OpenSSL features") variant('qt', default=False, description='Enables the build of cmake-gui') variant('doc', default=False, description='Enables the generation of html and man page documentation') @@ -77,8 +78,9 @@ def install(self, spec, prefix): options.append('--sphinx-html') options.append('--sphinx-man') - options.append('--') - options.append('-DCMAKE_USE_OPENSSL=ON') + if '+openssl' in spec: + options.append('--') + options.append('-DCMAKE_USE_OPENSSL=ON') configure(*options) make() From bba66cbe2836e364b58a611a5a5f5ae0d9692df4 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Sun, 20 Mar 2016 19:27:22 -0700 Subject: [PATCH 636/910] cmake: depend on openssl when openssl variant=true --- var/spack/repos/builtin/packages/cmake/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index be930a90abe..1f93d39769b 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -43,6 +43,7 @@ class Cmake(Package): variant('doc', default=False, description='Enables the generation of html and man page documentation') depends_on('ncurses', when='+ncurses') + depends_on('openssl', when='+openssl') depends_on('qt', when='+qt') depends_on('python@2.7.11:', when='+doc') depends_on('py-sphinx', when='+doc') From 7b283bfaffa5509915d91392878203bba8a8d2cb Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 21 Mar 2016 07:48:17 +0100 Subject: [PATCH 637/910] fix petsc on osx --- .../repos/builtin/packages/petsc/package.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index efe172fc083..7239baaf7f5 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -61,14 +61,24 @@ def mpi_dependent_options(self): errors = ['incompatible variants given'] + errors raise RuntimeError('\n'.join(errors)) else: - compiler_opts = [ - '--with-mpi=1', - '--with-mpi-dir=%s' % self.spec['mpi'].prefix, - ] + if self.compiler.name == "clang": + compiler_opts = [ + '--with-mpi=1', + '--with-cc=%s -Qunused-arguments' % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), # Avoid confusing PETSc config by clang: warning: argument unused during compilation + '--with-cxx=%s -Qunused-arguments' % join_path(self.spec['mpi'].prefix.bin, 'mpic++'), + '--with-fc=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + '--with-f77=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif77'), + ] + else: + compiler_opts = [ + '--with-mpi=1', + '--with-mpi-dir=%s' % self.spec['mpi'].prefix, + ] return compiler_opts def install(self, spec, prefix): - options = [] + options = ['--with-debugging=0', + '--with-ssl=0'] options.extend(self.mpi_dependent_options()) options.extend([ '--with-precision=%s' % ('double' if '+double' in spec else 'single'), From aef6d50b08daf893174046ba14c09a7019fe1212 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 21 Mar 2016 09:23:25 +0100 Subject: [PATCH 638/910] uninstall : fixed typo (print statement vs. print function) --- lib/spack/spack/cmd/uninstall.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 2fab8f6f2a1..1ece8386124 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -96,9 +96,9 @@ def num_installed_deps(pkg): pkg.do_uninstall(force=args.force) except PackageStillNeededError as e: tty.error("Will not uninstall %s" % e.spec.format("$_$@$%@$#", color=True)) - print() + print('') print("The following packages depend on it:") display_specs(e.dependents, long=True) - print() + print('') print("You can use spack uninstall -f to force this action.") sys.exit(1) From b926d8a0cdcf47de029e372cf15e281266930378 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 21 Mar 2016 14:33:00 +0100 Subject: [PATCH 639/910] build parmetis against external metis --- .../parmetis/enable_external_metis.patch | 13 ++++++++++ .../builtin/packages/parmetis/package.py | 24 ++++--------------- 2 files changed, 17 insertions(+), 20 deletions(-) create mode 100644 var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch diff --git a/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch b/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch new file mode 100644 index 00000000000..514781b8b88 --- /dev/null +++ b/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch @@ -0,0 +1,13 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index ca945dd..1bf94e9 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -33,7 +33,7 @@ include_directories(${GKLIB_PATH}) + include_directories(${METIS_PATH}/include) + + # List of directories that cmake will look for CMakeLists.txt +-add_subdirectory(${METIS_PATH}/libmetis ${CMAKE_BINARY_DIR}/libmetis) ++#add_subdirectory(${METIS_PATH}/libmetis ${CMAKE_BINARY_DIR}/libmetis) + add_subdirectory(include) + add_subdirectory(libparmetis) + add_subdirectory(programs) diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py index c897dec7e44..3f18dd76b87 100644 --- a/var/spack/repos/builtin/packages/parmetis/package.py +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -25,9 +25,6 @@ from spack import * -# FIXME : lot of code is duplicated from packages/metis/package.py . Inheriting from there may reduce -# FIXME : the installation rules to just a few lines - class Parmetis(Package): """ @@ -43,13 +40,11 @@ class Parmetis(Package): variant('debug', default=False, description='Builds the library in debug mode') variant('gdb', default=False, description='Enables gdb support') - variant('idx64', default=False, description='Use int64_t as default index type') - variant('double', default=False, description='Use double precision floating point types') - depends_on('cmake @2.8:') # build dependency depends_on('mpi') - # FIXME : this should conflict with metis as it builds its own version internally + patch('enable_external_metis.patch') + depends_on('metis') depends_on('gdb', when='+gdb') @@ -63,8 +58,8 @@ def install(self, spec, prefix): # FIXME : Once a contract is defined, MPI compilers should be retrieved indirectly via spec['mpi'] in case # FIXME : they use a non-standard name - options.extend(['-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=metis_source), - '-DMETIS_PATH:PATH={metis_source}'.format(metis_source=metis_source), + options.extend(['-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=metis_source), # still need headers from METIS source, and they are not installed with METIS. shame... + '-DMETIS_PATH:PATH={metis_source}'.format(metis_source=spec['metis'].prefix), '-DCMAKE_C_COMPILER:STRING=mpicc', '-DCMAKE_CXX_COMPILER:STRING=mpicxx']) @@ -78,18 +73,7 @@ def install(self, spec, prefix): if '+gdb' in spec: options.append('-DGDB:BOOL=ON') - metis_header = join_path(metis_source, 'include', 'metis.h') - - if '+idx64' in spec: - filter_file('IDXTYPEWIDTH 32', 'IDXTYPEWIDTH 64', metis_header) - - if '+double' in spec: - filter_file('REALTYPEWIDTH 32', 'REALTYPEWIDTH 64', metis_header) - with working_dir(build_directory, create=True): cmake(source_directory, *options) make() make("install") - # Parmetis build system doesn't allow for an external metis to be used, but doesn't copy the required - # metis header either - install(metis_header, self.prefix.include) From 3fcaf5b9031c3d55b152844b97cd432078340f4b Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 21 Mar 2016 14:47:05 +0100 Subject: [PATCH 640/910] llvm : update to 3.8.0 --- var/spack/repos/builtin/packages/llvm/package.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index 280e400f696..5c38b5b88c3 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -132,6 +132,21 @@ class Llvm(Package): 'llvm-libunwind' : 'http://llvm.org/svn/llvm-project/libunwind/trunk', } }, + { + 'version' : '3.8.0', + 'md5':'07a7a74f3c6bd65de4702bf941b511a0', + 'resources' : { + 'compiler-rt' : 'd6fcbe14352ffb708e4d1ac2e48bb025', + 'openmp' : '8fd7cc35d48051613cf1e750e9f22e40', + 'polly' : '1b3b20f52d34a4024e21a4ea7112caa7', + 'libcxx' : 'd6e0bdbbee39f7907ad74fd56d03b88a', + 'libcxxabi' : 'bbe6b4d72c7c5978550d370af529bcf7', + 'clang' : 'cc99e7019bb74e6459e80863606250c5', + 'clang-tools-extra' : 'c2344f50e0eea0b402f0092a80ddc036', + 'lldb' : 'a5da35ed9cc8c8817ee854e3dbfba00e', + 'llvm-libunwind' : '162ade468607f153cca12be90b5194fa', + } + }, { 'version' : '3.7.1', 'md5':'bf8b3a2c79e61212c5409041dfdbd319', From 3a3443dff63167fc0331672871af4c0cc1fe5903 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 21 Mar 2016 15:14:48 +0100 Subject: [PATCH 641/910] parmetis patches by PETSc developers --- .../builtin/packages/parmetis/package.py | 6 ++ ...9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch | 77 +++++++++++++++++++ ...9d68aa1d6cbc70740d0f35024aae17f7d5cb.patch | 35 +++++++++ 3 files changed, 118 insertions(+) create mode 100644 var/spack/repos/builtin/packages/parmetis/pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch create mode 100644 var/spack/repos/builtin/packages/parmetis/pkg-parmetis-82409d68aa1d6cbc70740d0f35024aae17f7d5cb.patch diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py index 3f18dd76b87..c691cf4191d 100644 --- a/var/spack/repos/builtin/packages/parmetis/package.py +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -46,6 +46,12 @@ class Parmetis(Package): patch('enable_external_metis.patch') depends_on('metis') + # bug fixes from PETSc developers + # https://bitbucket.org/petsc/pkg-parmetis/commits/1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b/raw/ + patch('pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch') + # https://bitbucket.org/petsc/pkg-parmetis/commits/82409d68aa1d6cbc70740d0f35024aae17f7d5cb/raw/ + patch('pkg-parmetis-82409d68aa1d6cbc70740d0f35024aae17f7d5cb.patch') + depends_on('gdb', when='+gdb') def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/parmetis/pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch b/var/spack/repos/builtin/packages/parmetis/pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch new file mode 100644 index 00000000000..e6b8056c210 --- /dev/null +++ b/var/spack/repos/builtin/packages/parmetis/pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch @@ -0,0 +1,77 @@ +From 1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b Mon Sep 17 00:00:00 2001 +From: Jed Brown +Date: Fri, 12 Oct 2012 15:45:10 -0500 +Subject: [PATCH] ParMetis bug fixes reported by John Fettig [petsc-maint + #133631] + +''' +I have also reported to to Karypis but have received zero +response and he hasn't released any updates to the original release +either. At least he approved my forum posting so that other people +can see the bug and the fix. +http://glaros.dtc.umn.edu/gkhome/node/837 +''' + +Hg-commit: 1c2b9fe39201d404b493885093b5992028b9b8d4 +--- + libparmetis/xyzpart.c | 12 ++++++------ + 1 file changed, 6 insertions(+), 6 deletions(-) + +diff --git a/libparmetis/xyzpart.c b/libparmetis/xyzpart.c +index 3a2c289..63abfcb 100644 +--- a/libparmetis/xyzpart.c ++++ b/libparmetis/xyzpart.c +@@ -104,7 +104,7 @@ void IRBinCoordinates(ctrl_t *ctrl, graph_t *graph, idx_t ndims, real_t *xyz, + + for (i=0; i +Date: Tue, 20 Mar 2012 11:59:44 -0500 +Subject: [PATCH] parmetis: fix bug reported by jfettig; '<' to '<=' in xyzpart + +Hg-commit: 2dd2eae596acaabbc80e0ef875182616f868dbc2 +--- + libparmetis/xyzpart.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/libparmetis/xyzpart.c b/libparmetis/xyzpart.c +index 307aed9..3a2c289 100644 +--- a/libparmetis/xyzpart.c ++++ b/libparmetis/xyzpart.c +@@ -111,7 +111,7 @@ void IRBinCoordinates(ctrl_t *ctrl, graph_t *graph, idx_t ndims, real_t *xyz, + /* determine bucket counts */ + iset(nbins, 0, lcounts); + for (j=0, i=0; i Date: Mon, 21 Mar 2016 16:18:02 +0100 Subject: [PATCH 642/910] llvm : does not support python 3 --- var/spack/repos/builtin/packages/llvm/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index 5c38b5b88c3..1d25d59e504 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -52,7 +52,7 @@ class Llvm(Package): depends_on('cmake @2.8.12.2:') # Universal dependency - depends_on('python@2.7:') + depends_on('python@2.7:2.8') # Seems not to support python 3.X.Y # lldb dependencies depends_on('ncurses', when='+lldb') From c5c92d50eb78673c635a7ed26145bf10a15c20cb Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 21 Mar 2016 11:24:36 -0400 Subject: [PATCH 643/910] paraview: remove trailing whitespace --- var/spack/repos/builtin/packages/paraview/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index ccf2d14c06e..b3312736f16 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -37,11 +37,11 @@ class Paraview(Package): #depends_on('protobuf') # version mismatches? #depends_on('sqlite') # external version not supported depends_on('zlib') - + def url_for_version(self, version): """Handle ParaView version-based custom URLs.""" return self._url_str % (version.up_to(2), version) - + def install(self, spec, prefix): with working_dir('spack-build', create=True): From 8e77f1776020c2a6edf9eb284615d99c905821e9 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 21 Mar 2016 11:54:58 -0400 Subject: [PATCH 644/910] paraview: fix variant description typo --- var/spack/repos/builtin/packages/paraview/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index b3312736f16..0c3c7c16214 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -16,7 +16,7 @@ class Paraview(Package): variant('osmesa', default=False, description='Enable OSMesa support') variant('qt', default=False, description='Enable Qt support') - variant('opengl2', default=False, description='Enable OPengl2 backend') + variant('opengl2', default=False, description='Enable OpenGL2 backend') depends_on('python', when='+python') depends_on('py-numpy', when='+python') From 553fff270a13d40fb9a41569bebf5075a740d8ac Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 21 Mar 2016 11:24:49 -0400 Subject: [PATCH 645/910] paraview: disallow python3 ParaView is not Python3-ready. --- var/spack/repos/builtin/packages/paraview/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index 0c3c7c16214..c16054816c1 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -18,7 +18,7 @@ class Paraview(Package): variant('qt', default=False, description='Enable Qt support') variant('opengl2', default=False, description='Enable OpenGL2 backend') - depends_on('python', when='+python') + depends_on('python@2:2.7', when='+python') depends_on('py-numpy', when='+python') depends_on('py-matplotlib', when='+python') depends_on('tcl', when='+tcl') From 953fb35f82c099936f833f31dcd292ffa1a3ec06 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 21 Mar 2016 18:00:32 +0100 Subject: [PATCH 646/910] add superlu_dist package --- .../builtin/packages/superlu-dist/package.py | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 var/spack/repos/builtin/packages/superlu-dist/package.py diff --git a/var/spack/repos/builtin/packages/superlu-dist/package.py b/var/spack/repos/builtin/packages/superlu-dist/package.py new file mode 100644 index 00000000000..c4c76909b35 --- /dev/null +++ b/var/spack/repos/builtin/packages/superlu-dist/package.py @@ -0,0 +1,63 @@ +from spack import * + +class SuperluDist(Package): + """A general purpose library for the direct solution of large, sparse, nonsymmetric systems of linear equations on high performance machines.""" + homepage = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/" + url = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_dist_4.1.tar.gz" + + version('4.3', 'ee66c84e37b4f7cc557771ccc3dc43ae') + version('4.2', 'ae9fafae161f775fbac6eba11e530a65') + version('4.1', '4edee38cc29f687bd0c8eb361096a455') + version('4.0', 'c0b98b611df227ae050bc1635c6940e0') + + depends_on ('mpi') + depends_on ('blas') + depends_on ('lapack') + depends_on ('parmetis') + depends_on ('metis') + + def install(self, spec, prefix): + makefile_inc = [] + makefile_inc.extend([ + 'PLAT = _mac_x', + 'DSuperLUroot = %s' % self.stage.source_path, #self.stage.path, prefix + 'DSUPERLULIB = $(DSuperLUroot)/lib/libsuperlu_dist.a', + 'BLASDEF = -DUSE_VENDOR_BLAS', + 'BLASLIB = -L%s -llapack %s -lblas' % (spec['lapack'].prefix.lib, spec['blas'].prefix.lib), # FIXME: avoid hardcoding blas/lapack lib names + 'METISLIB = -L%s -lmetis' % spec['metis'].prefix.lib, + 'PARMETISLIB = -L%s -lparmetis' % spec['parmetis'].prefix.lib, + 'FLIBS =', + 'LIBS = $(DSUPERLULIB) $(BLASLIB) $(PARMETISLIB) $(METISLIB)', + 'ARCH = ar', + 'ARCHFLAGS = cr', + 'RANLIB = true', + 'CC = mpicc', # FIXME avoid hardcoding MPI compiler names + 'CFLAGS = -fPIC -std=c99 -O2 -I%s -I%s' %(spec['parmetis'].prefix.include, spec['metis'].prefix.include), + 'NOOPTS = -fPIC -std=c99', + 'FORTRAN = mpif77', + 'F90FLAGS = -O2', + 'LOADER = mpif77', + 'LOADOPTS =', + 'CDEFS = -DAdd_' + ]) + + #with working_dir('src'): + with open('make.inc', 'w') as fh: + fh.write('\n'.join(makefile_inc)) + + make("lib", parallel=False) + + # FIXME: + # cd "EXAMPLE" do + # system "make" + + # need to install by hand + headers_location = join_path(self.prefix.include,'superlu_dist') + mkdirp(headers_location) + # FIXME: fetch all headers in the folder automatically + for header in ['Cnames.h','cublas_utils.h','dcomplex.h','html_mainpage.h','machines.h','old_colamd.h','psymbfact.h','superlu_ddefs.h','superlu_defs.h','superlu_enum_consts.h','superlu_zdefs.h','supermatrix.h','util_dist.h']: + superludist_header = join_path(self.stage.source_path, 'SRC/',header) + install(superludist_header, headers_location) + + superludist_lib = join_path(self.stage.source_path, 'lib/libsuperlu_dist.a') + install(superludist_lib,self.prefix.lib) From b79fce76cb4d412cf718b48b05d549470e4c55ab Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 21 Mar 2016 14:28:07 -0700 Subject: [PATCH 647/910] Fix issue 573 where Spack was ignoring user's compiler preference in concretization --- lib/spack/spack/concretize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 8083f919828..2e576743ec6 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -241,7 +241,7 @@ def concretize_compiler(self, spec): return False #Find the another spec that has a compiler, or the root if none do - other_spec = find_spec(spec, lambda(x) : x.compiler) + other_spec = spec if spec.compiler else find_spec(spec, lambda(x) : x.compiler) if not other_spec: other_spec = spec.root other_compiler = other_spec.compiler @@ -288,7 +288,7 @@ def find_spec(spec, condition): if condition(spec): return spec - return None # Nohting matched the condition. + return None # Nothing matched the condition. def cmp_specs(lhs, rhs): From 5d06daeb5eed28b4b91ba62a8f99165d87b5ef86 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 21 Mar 2016 14:28:34 -0700 Subject: [PATCH 648/910] Add test for issue 573, child with compiler not respected in concretization --- lib/spack/spack/test/concretize.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index f264faf17a2..08cce096747 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -309,3 +309,10 @@ def test_find_spec_none(self): Spec('d')), Spec('e')) self.assertEqual(None, find_spec(s['b'], lambda s: '+foo' in s)) + + + def test_compiler_child(self): + s = Spec('mpileaks%clang ^dyninst%gcc') + s.concretize() + self.assertTrue(s['mpileaks'].satisfies('%clang')) + self.assertTrue(s['dyninst'].satisfies('%gcc')) From 48b35bb495bc154c64d23d1ad364506c4572b913 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 21 Mar 2016 00:11:18 -0700 Subject: [PATCH 649/910] Fix print function in uninstall.py --- lib/spack/spack/cmd/uninstall.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 1ece8386124..350ef372cb6 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +from __future__ import print_function import sys import argparse @@ -63,12 +64,12 @@ def uninstall(parser, args): matching_specs = spack.installed_db.query(spec) if not args.all and len(matching_specs) > 1: tty.error("%s matches multiple packages:" % spec) - print + print() display_specs(matching_specs, long=True) - print - print "You can either:" - print " a) Use a more specific spec, or" - print " b) use spack uninstall -a to uninstall ALL matching specs." + print() + print("You can either:") + print(" a) Use a more specific spec, or") + print(" b) use spack uninstall -a to uninstall ALL matching specs.") sys.exit(1) if len(matching_specs) == 0: From e88df95b42fdcaa49552811853f8ca4ecc52cf9f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 21 Mar 2016 00:35:13 -0700 Subject: [PATCH 650/910] Remove unused code from modules.py --- lib/spack/spack/modules.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 0378110c8cb..4e98d500013 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -133,6 +133,7 @@ def __init__(self, spec=None): if self.spec.package.__doc__: self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) + @property def category(self): # Anything defined at the package level takes precedence @@ -144,21 +145,6 @@ def category(self): # Not very descriptive fallback return 'spack installed package' - # @property - # def paths(self): - # # Add python path unless it's an actual python installation - # # TODO : is there a better way to do this? - # # FIXME : add PYTHONPATH to every python package - # if self.spec.name != 'python': - # site_packages = glob(join_path(self.spec.prefix.lib, "python*/site-packages")) - # if site_packages: - # add_path('PYTHONPATH', site_packages[0]) - # - # # FIXME : Same for GEM_PATH - # if self.spec.package.extends(spack.spec.Spec('ruby')): - # add_path('GEM_PATH', self.spec.prefix) - # - # return self._paths def write(self): """Write out a module file for this object.""" @@ -166,16 +152,20 @@ def write(self): if not os.path.exists(module_dir): mkdirp(module_dir) - # Environment modifications guessed by inspecting the installation prefix + # Environment modifications guessed by inspecting the + # installation prefix env = inspect_path(self.spec.prefix) - # Let the extendee modify their extensions before asking for package-specific modifications + # Let the extendee modify their extensions before asking for + # package-specific modifications for extendee in self.pkg.extendees: extendee_spec = self.spec[extendee] - extendee_spec.package.modify_module(self.pkg.module, extendee_spec, self.spec) + extendee_spec.package.modify_module( + self.pkg.module, extendee_spec, self.spec) # Package-specific environment modifications self.spec.package.setup_environment(env) + # TODO : implement site-specific modifications and filters if not env: return @@ -232,7 +222,7 @@ def file_name(self): def use_name(self): return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version, self.spec.compiler.name, - self.spec.compiler.version, + self.spec.compiler.version, self.spec.dag_hash()) def write_header(self, dk_file): @@ -270,7 +260,7 @@ def file_name(self): def use_name(self): return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version, self.spec.compiler.name, - self.spec.compiler.version, + self.spec.compiler.version, self.spec.dag_hash()) def write_header(self, module_file): From 439d47b4e45c674ab9aa4ebd0c2bfaf6911ade60 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 21 Mar 2016 01:48:18 -0700 Subject: [PATCH 651/910] Refactor environment setup. - Gave setup_environment and setup_dependent_environment more similar signatures. They now allows editing the Spack env and the runtime env for *this* package and dependents, respectively. - modify_module renamed to setup_dependent_python_module for symmetry with setup_dependent_environment and to avoid confusion with environment modules. - removed need for patching Package objects at runtime. - adjust packages to reflect these changes. --- lib/spack/spack/build_environment.py | 35 +++-- lib/spack/spack/modules.py | 3 +- lib/spack/spack/package.py | 120 ++++++++++++++---- .../repos/builtin/packages/mpich/package.py | 9 +- .../packages/netlib-scalapack/package.py | 2 +- .../repos/builtin/packages/openmpi/package.py | 16 +-- .../repos/builtin/packages/python/package.py | 39 ++---- .../repos/builtin/packages/qt/package.py | 10 +- .../repos/builtin/packages/ruby/package.py | 12 +- 9 files changed, 150 insertions(+), 96 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 59b234624cf..5688d47e2d3 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -84,7 +84,7 @@ def __call__(self, *args, **kwargs): return super(MakeExecutable, self).__call__(*args, **kwargs) -def set_compiler_environment_variables(pkg): +def set_compiler_environment_variables(pkg, env): assert pkg.spec.concrete # Set compiler variables used by CMake and autotools assert all(key in pkg.compiler.link_paths for key in ('cc', 'cxx', 'f77', 'fc')) @@ -92,7 +92,6 @@ def set_compiler_environment_variables(pkg): # Populate an object with the list of environment modifications # and return it # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc. - env = EnvironmentModifications() link_dir = spack.build_env_path env.set_env('CC', join_path(link_dir, pkg.compiler.link_paths['cc'])) env.set_env('CXX', join_path(link_dir, pkg.compiler.link_paths['cxx'])) @@ -113,7 +112,7 @@ def set_compiler_environment_variables(pkg): return env -def set_build_environment_variables(pkg): +def set_build_environment_variables(pkg, env): """ This ensures a clean install environment when we build packages """ @@ -134,7 +133,6 @@ def set_build_environment_variables(pkg): if os.path.isdir(ci): env_paths.append(ci) - env = EnvironmentModifications() for item in reversed(env_paths): env.prepend_path('PATH', item) env.set_env(SPACK_ENV_PATH, concatenate_paths(env_paths)) @@ -180,7 +178,7 @@ def set_build_environment_variables(pkg): return env -def set_module_variables_for_package(pkg, m): +def set_module_variables_for_package(pkg, module): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. """ @@ -190,6 +188,8 @@ def set_module_variables_for_package(pkg, m): jobs = 1 elif pkg.make_jobs: jobs = pkg.make_jobs + + m = module m.make_jobs = jobs # TODO: make these build deps that can be installed if not found. @@ -271,9 +271,12 @@ def parent_class_modules(cls): def setup_package(pkg): """Execute all environment setup routines.""" - env = EnvironmentModifications() - env.extend(set_compiler_environment_variables(pkg)) - env.extend(set_build_environment_variables(pkg)) + spack_env = EnvironmentModifications() + run_env = EnvironmentModifications() + + set_compiler_environment_variables(pkg, spack_env) + set_build_environment_variables(pkg, spack_env) + # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from # an existing class like spack.repos.original.libelf.Libelf, @@ -285,12 +288,20 @@ def setup_package(pkg): # Allow dependencies to modify the module for dependency_spec in pkg.spec.traverse(root=False): - dependency_spec.package.modify_module(pkg.module, dependency_spec, pkg.spec) + dpkg = dependency_spec.package + dpkg.setup_dependent_python_module(pkg.module, pkg.spec) + # Allow dependencies to set up environment as well for dependency_spec in pkg.spec.traverse(root=False): - dependency_spec.package.setup_dependent_environment(env, pkg.spec) - validate(env, tty.warn) - env.apply_modifications() + dpkg = dependency_spec.package + dpkg.setup_dependent_environment(spack_env, run_env, pkg.spec) + + # Allow the package to apply some settings. + pkg.setup_environment(spack_env, run_env) + + # Make sure nothing's strange about the Spack environment. + validate(spack_env, tty.warn) + spack_env.apply_modifications() def fork(pkg, function): diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 4e98d500013..05c93cd3e65 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -164,7 +164,8 @@ def write(self): self.pkg.module, extendee_spec, self.spec) # Package-specific environment modifications - self.spec.package.setup_environment(env) + spack_env = EnvironmentModifications() + self.spec.package.setup_environment(spack_env, env) # TODO : implement site-specific modifications and filters if not env: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index acad5a28f6c..9d8ac87bd70 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1002,56 +1002,120 @@ def module(self): return __import__(self.__class__.__module__, fromlist=[self.__class__.__name__]) - def setup_environment(self, env): - """ - Appends in `env` the list of environment modifications needed to use this package outside of spack. + def setup_environment(self, spack_env, run_env): + """Set up the compile and runtime environemnts for a package. - Default implementation does nothing, but this can be overridden if the package needs a particular environment. + `spack_env` and `run_env` are `EnvironmentModifications` + objects. Package authors can call methods on them to alter + the environment within Spack and at runtime. - Example : + Both `spack_env` and `run_env` are applied within the build + process, before this package's `install()` method is called. - 1. A lot of Qt extensions need `QTDIR` set. This can be used to do that. + Modifications in `run_env` will *also* be added to the + generated environment modules for this package. + + Default implementation does nothing, but this can be + overridden if the package needs a particular environment. + + Examples: + + 1. Qt extensions need `QTDIR` set. Args: - env: list of environment modifications to be updated + spack_env (EnvironmentModifications): list of + modifications to be applied when this package is built + within Spack. + + run_env (EnvironmentModifications): list of environment + changes to be applied when this package is run outside + of Spack. + """ pass - def setup_dependent_environment(self, env, dependent_spec): - """ - Called before the install() method of dependents. - Appends in `env` the list of environment modifications needed by dependents (or extensions) during the - installation of a package. The default implementation delegates to `setup_environment`, but can be overridden - if the modifications to the environment happen to be different from the one needed to use the package outside - of spack. + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + """Set up the environment of packages that depend on this one. - This is useful if there are some common steps to installing all extensions for a certain package. + This is similar to `setup_environment`, but it is used to + modify the compile and runtime environments of packages that + *depend* on this one. This gives packages like Python and + others that follow the extension model a way to implement + common environment or compile-time settings for dependencies. + + By default, this delegates to self.setup_environment() Example : - 1. Installing python modules generally requires `PYTHONPATH` to point to the lib/pythonX.Y/site-packages - directory in the module's install prefix. This could set that variable. + 1. Installing python modules generally requires + `PYTHONPATH` to point to the lib/pythonX.Y/site-packages + directory in the module's install prefix. This could + set that variable. Args: - env: list of environment modifications to be updated - dependent_spec: dependent (or extension) of this spec - """ - self.setup_environment(env) - def modify_module(self, module, spec, dependent_spec): + spack_env (EnvironmentModifications): list of + modifications to be applied when the dependent package + is bulit within Spack. + + run_env (EnvironmentModifications): list of environment + changes to be applied when the dependent package is + run outside of Spack. + + dependent_spec (Spec): The spec of the dependent package + about to be built. This allows the extendee (self) to + query the dependent's state. Note that *this* + package's spec is available as `self.spec`. + + This is useful if there are some common steps to installing + all extensions for a certain package. + """ + self.setup_environment(spack_env, run_env) + + + def setup_dependent_python_module(self, module, dependent_spec): + """Set up Python module-scope variables for dependent packages. + Called before the install() method of dependents. - Default implementation does nothing, but this can be overridden by an extendable package to set up the module of - its extensions. This is useful if there are some common steps to installing all extensions for a - certain package. + Default implementation does nothing, but this can be + overridden by an extendable package to set up the module of + its extensions. This is useful if there are some common steps + to installing all extensions for a certain package. Example : - 1. Extensions often need to invoke the 'python' interpreter from the Python installation being extended. - This routine can put a 'python' Executable object in the module scope for the extension package to simplify - extension installs. + 1. Extensions often need to invoke the `python` + interpreter from the Python installation being + extended. This routine can put a 'python' Executable + object in the module scope for the extension package to + simplify extension installs. + + 2. MPI compilers could set some variables in the + dependent's scope that point to `mpicc`, `mpicxx`, + etc., allowing them to be called by common names + regardless of which MPI is used. + + 3. BLAS/LAPACK implementations can set some variables + indicating the path to their libraries, since these + paths differ by BLAS/LAPACK implementation. + + Args: + + module (module): The Python `module` object of the + dependent package. Packages can use this to set + module-scope variables for the dependent to use. + + dependent_spec (Spec): The spec of the dependent package + about to be built. This allows the extendee (self) to + query the dependent's state. Note that *this* + package's spec is available as `self.spec`. + + This is useful if there are some common steps to installing + all extensions for a certain package. + """ pass diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 5af9b585ea8..90b5d42eab4 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -47,13 +47,6 @@ class Mpich(Package): provides('mpi@:3.0', when='@3:') provides('mpi@:1.3', when='@1:') - def setup_environment(self, env): - env.set_env('MPICH_CC', self.compiler.cc) - env.set_env('MPICH_CXX', self.compiler.cxx) - env.set_env('MPICH_F77', self.compiler.f77) - env.set_env('MPICH_F90', self.compiler.fc) - env.set_env('MPICH_FC', self.compiler.fc) - def setup_dependent_environment(self, env, dependent_spec): env.set_env('MPICH_CC', spack_cc) env.set_env('MPICH_CXX', spack_cxx) @@ -61,7 +54,7 @@ def setup_dependent_environment(self, env, dependent_spec): env.set_env('MPICH_F90', spack_f90) env.set_env('MPICH_FC', spack_fc) - def modify_module(self, module, spec, dep_spec): + def setup_dependent_python_module(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" # FIXME : is this necessary ? Shouldn't this be part of a contract with MPI providers? module.mpicc = join_path(self.prefix.bin, 'mpicc') diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 36f506f7cdf..62abfcc48e3 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -40,7 +40,7 @@ def install(self, spec, prefix): make() make("install") - def modify_module(self, module, spec, dependent_spec): + def setup_dependent_python_module(self, module, spec, dependent_spec): lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' lib_suffix = lib_dsuffix if '+shared' in spec['scalapack'] else '.a' diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 7783ca87666..c91a13e3760 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -41,17 +41,13 @@ class Openmpi(Package): def url_for_version(self, version): return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) - def setup_environment(self, env): - env.set_env('OMPI_CC', self.compiler.cc) - env.set_env('OMPI_CXX', self.compiler.cxx) - env.set_env('OMPI_FC', self.compiler.fc) - env.set_env('OMPI_F77', self.compiler.f77) - def setup_dependent_environment(self, env, dependent_spec): - env.set_env('OMPI_CC', spack_cc) - env.set_env('OMPI_CXX', spack_cxx) - env.set_env('OMPI_FC', spack_fc) - env.set_env('OMPI_F77', spack_f77) + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + spack_env.set_env('OMPI_CC', spack_cc) + spack_env.set_env('OMPI_CXX', spack_cxx) + spack_env.set_env('OMPI_FC', spack_fc) + spack_env.set_env('OMPI_F77', spack_f77) + def install(self, spec, prefix): config_args = ["--prefix=%s" % prefix, diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index c445d26369e..593a27708c8 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -92,13 +92,21 @@ def python_include_dir(self): def site_packages_dir(self): return os.path.join(self.python_lib_dir, 'site-packages') - def setup_dependent_environment(self, env, extension_spec): - # Set PYTHONPATH to include site-packages dir for the extension and any other python extensions it depends on. + + def setup_dependent_environment(self, spack_env, run_env, extension_spec): + # TODO: do this only for actual extensions. + + # Set PYTHONPATH to include site-packages dir for the + # extension and any other python extensions it depends on. python_paths = [] for d in extension_spec.traverse(): if d.package.extends(self.spec): python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) - env.set_env('PYTHONPATH', ':'.join(python_paths)) + + pythonpath = ':'.join(python_paths) + spack_env.set_env('PYTHONPATH', pythonpath) + run_env.set_env('PYTHONPATH', pythonpath) + def modify_module(self, module, spec, ext_spec): """ @@ -114,31 +122,6 @@ def modify_module(self, module, spec, ext_spec): else: module.python = Executable(join_path(spec.prefix.bin, 'python')) - # The code below patches the any python extension to have good defaults for `setup_dependent_environment` and - # `setup_environment` only if the extension didn't override any of these functions explicitly. - def _setup_env(self, env): - site_packages = glob.glob(join_path(self.spec.prefix.lib, "python*/site-packages")) - if site_packages: - env.prepend_path('PYTHONPATH', site_packages[0]) - - def _setup_denv(self, env, extension_spec): - pass - - pkg_cls = type(ext_spec.package) # Retrieve the type we may want to patch - if 'python' in pkg_cls.extendees: - # List of overrides we are interested in - interesting_overrides = ['setup_environment', 'setup_dependent_environment'] - overrides_found = [ - (name, defining_cls) for name, _, defining_cls, _, in inspect.classify_class_attrs(pkg_cls) - if - name in interesting_overrides and # The attribute has the right name - issubclass(defining_cls, Package) and defining_cls is not Package # and is an actual override - ] - if not overrides_found: - # If no override were found go on patching - pkg_cls.setup_environment = functools.wraps(Package.setup_environment)(_setup_env) - pkg_cls.setup_dependent_environment = functools.wraps(Package.setup_dependent_environment)(_setup_denv) - # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) module.python_include_dir = os.path.join(ext_spec.prefix, self.python_include_dir) diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 35b9d684624..039aeb3c314 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -55,8 +55,14 @@ class Qt(Package): depends_on("mesa", when='@4:+mesa') depends_on("libxcb") - def setup_environment(self, env): - env.set_env['QTDIR'] = self.prefix + + def setup_environment(self, spack_env, env): + env.set_env('QTDIR', self.prefix) + + + def setup_dependent_environment(self, spack_env, run_env, dspec): + spack_env.set_env('QTDIR', self.prefix) + def patch(self): if self.spec.satisfies('@4'): diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py index 2d1da8c9af1..39f65f51d2c 100644 --- a/var/spack/repos/builtin/packages/ruby/package.py +++ b/var/spack/repos/builtin/packages/ruby/package.py @@ -2,7 +2,7 @@ class Ruby(Package): - """A dynamic, open source programming language with a focus on + """A dynamic, open source programming language with a focus on simplicity and productivity.""" homepage = "https://www.ruby-lang.org/" @@ -17,15 +17,17 @@ def install(self, spec, prefix): make() make("install") - def setup_dependent_environment(self, env, extension_spec): + def setup_dependent_environment(self, spack_env, run_env, extension_spec): + # TODO: do this only for actual extensions. # Set GEM_PATH to include dependent gem directories ruby_paths = [] for d in extension_spec.traverse(): if d.package.extends(self.spec): ruby_paths.append(d.prefix) - env.set_env('GEM_PATH', concatenate_paths(ruby_paths)) + + spack_env.set_env('GEM_PATH', concatenate_paths(ruby_paths)) # The actual installation path for this gem - env.set_env('GEM_HOME', extension_spec.prefix) + spack_env.set_env('GEM_HOME', extension_spec.prefix) def modify_module(self, module, spec, ext_spec): """Called before ruby modules' install() methods. Sets GEM_HOME @@ -38,5 +40,3 @@ def modify_module(self, module, spec, ext_spec): # Ruby extension builds have global ruby and gem functions module.ruby = Executable(join_path(spec.prefix.bin, 'ruby')) module.gem = Executable(join_path(spec.prefix.bin, 'gem')) - - From b1516f64eb75c108eded1e9ee7e0480a4552236a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 21 Mar 2016 02:21:31 -0700 Subject: [PATCH 652/910] Rename some environment methods to be less repetitive, add set_path. --- lib/spack/spack/build_environment.py | 45 ++++++++++--------- lib/spack/spack/environment.py | 22 ++++++++- lib/spack/spack/test/environment.py | 22 +++++---- .../repos/builtin/packages/mpich/package.py | 10 ++--- .../repos/builtin/packages/openmpi/package.py | 8 ++-- .../repos/builtin/packages/python/package.py | 4 +- .../repos/builtin/packages/qt/package.py | 4 +- .../repos/builtin/packages/ruby/package.py | 5 ++- 8 files changed, 73 insertions(+), 47 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 5688d47e2d3..fc5b7d6207f 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -36,7 +36,7 @@ import spack import llnl.util.tty as tty from llnl.util.filesystem import * -from spack.environment import EnvironmentModifications, concatenate_paths, validate +from spack.environment import EnvironmentModifications, validate from spack.util.environment import * from spack.util.executable import Executable, which @@ -93,22 +93,23 @@ def set_compiler_environment_variables(pkg, env): # and return it # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc. link_dir = spack.build_env_path - env.set_env('CC', join_path(link_dir, pkg.compiler.link_paths['cc'])) - env.set_env('CXX', join_path(link_dir, pkg.compiler.link_paths['cxx'])) - env.set_env('F77', join_path(link_dir, pkg.compiler.link_paths['f77'])) - env.set_env('FC', join_path(link_dir, pkg.compiler.link_paths['fc'])) + env.set('CC', join_path(link_dir, pkg.compiler.link_paths['cc'])) + env.set('CXX', join_path(link_dir, pkg.compiler.link_paths['cxx'])) + env.set('F77', join_path(link_dir, pkg.compiler.link_paths['f77'])) + env.set('FC', join_path(link_dir, pkg.compiler.link_paths['fc'])) + # Set SPACK compiler variables so that our wrapper knows what to call compiler = pkg.compiler if compiler.cc: - env.set_env('SPACK_CC', compiler.cc) + env.set('SPACK_CC', compiler.cc) if compiler.cxx: - env.set_env('SPACK_CXX', compiler.cxx) + env.set('SPACK_CXX', compiler.cxx) if compiler.f77: - env.set_env('SPACK_F77', compiler.f77) + env.set('SPACK_F77', compiler.f77) if compiler.fc: - env.set_env('SPACK_FC', compiler.fc) + env.set('SPACK_FC', compiler.fc) - env.set_env('SPACK_COMPILER_SPEC', str(pkg.spec.compiler)) + env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler)) return env @@ -135,25 +136,25 @@ def set_build_environment_variables(pkg, env): for item in reversed(env_paths): env.prepend_path('PATH', item) - env.set_env(SPACK_ENV_PATH, concatenate_paths(env_paths)) + env.set_path(SPACK_ENV_PATH, env_paths) # Prefixes of all of the package's dependencies go in SPACK_DEPENDENCIES dep_prefixes = [d.prefix for d in pkg.spec.traverse(root=False)] - env.set_env(SPACK_DEPENDENCIES, concatenate_paths(dep_prefixes)) - env.set_env('CMAKE_PREFIX_PATH', concatenate_paths(dep_prefixes)) # Add dependencies to CMAKE_PREFIX_PATH + env.set_path(SPACK_DEPENDENCIES, dep_prefixes) + env.set_path('CMAKE_PREFIX_PATH', dep_prefixes) # Add dependencies to CMAKE_PREFIX_PATH # Install prefix - env.set_env(SPACK_PREFIX, pkg.prefix) + env.set(SPACK_PREFIX, pkg.prefix) # Install root prefix - env.set_env(SPACK_INSTALL, spack.install_path) + env.set(SPACK_INSTALL, spack.install_path) # Remove these vars from the environment during build because they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. - env.unset_env('LD_LIBRARY_PATH') - env.unset_env('LD_RUN_PATH') - env.unset_env('DYLD_LIBRARY_PATH') + env.unset('LD_LIBRARY_PATH') + env.unset('LD_RUN_PATH') + env.unset('DYLD_LIBRARY_PATH') # Add bin directories from dependencies to the PATH for the build. bin_dirs = reversed(filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes])) @@ -162,9 +163,9 @@ def set_build_environment_variables(pkg, env): # Working directory for the spack command itself, for debug logs. if spack.debug: - env.set_env(SPACK_DEBUG, 'TRUE') - env.set_env(SPACK_SHORT_SPEC, pkg.spec.short_spec) - env.set_env(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir) + env.set(SPACK_DEBUG, 'TRUE') + env.set(SPACK_SHORT_SPEC, pkg.spec.short_spec) + env.set(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir) # Add any pkgconfig directories to PKG_CONFIG_PATH pkg_config_dirs = [] @@ -173,7 +174,7 @@ def set_build_environment_variables(pkg, env): pcdir = join_path(p, libdir, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) - env.set_env('PKG_CONFIG_PATH', concatenate_paths(pkg_config_dirs)) + env.set_path('PKG_CONFIG_PATH', pkg_config_dirs) return env diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 74aef57fe88..72aafa4e2d6 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -29,6 +29,12 @@ def execute(self): os.environ.pop(self.name, None) # Avoid throwing if the variable was not set +class SetPath(NameValueModifier): + def execute(self): + string_path = concatenate_paths(self.value) + os.environ[self.name] = string_path + + class AppendPath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') @@ -103,7 +109,7 @@ def _get_outside_caller_attributes(self): } return args - def set_env(self, name, value, **kwargs): + def set(self, name, value, **kwargs): """ Stores in the current object a request to set an environment variable @@ -115,7 +121,7 @@ def set_env(self, name, value, **kwargs): item = SetEnv(name, value, **kwargs) self.env_modifications.append(item) - def unset_env(self, name, **kwargs): + def unset(self, name, **kwargs): """ Stores in the current object a request to unset an environment variable @@ -126,6 +132,18 @@ def unset_env(self, name, **kwargs): item = UnsetEnv(name, **kwargs) self.env_modifications.append(item) + def set_path(self, name, elts, **kwargs): + """ + Stores a request to set a path generated from a list. + + Args: + name: name o the environment variable to be set. + elts: elements of the path to set. + """ + kwargs.update(self._get_outside_caller_attributes()) + item = SetPath(name, elts, **kwargs) + self.env_modifications.append(item) + def append_path(self, name, path, **kwargs): """ Stores in the current object a request to append a path to a path list diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index 3e03760c016..6c8f5ea43c3 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -11,21 +11,27 @@ def setUp(self): os.environ['PATH_LIST'] = '/path/second:/path/third' os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' - def test_set_env(self): + def test_set(self): env = EnvironmentModifications() - env.set_env('A', 'dummy value') - env.set_env('B', 3) + env.set('A', 'dummy value') + env.set('B', 3) env.apply_modifications() self.assertEqual('dummy value', os.environ['A']) self.assertEqual(str(3), os.environ['B']) - def test_unset_env(self): + def test_unset(self): env = EnvironmentModifications() self.assertEqual('foo', os.environ['UNSET_ME']) - env.unset_env('UNSET_ME') + env.unset('UNSET_ME') env.apply_modifications() self.assertRaises(KeyError, os.environ.__getitem__, 'UNSET_ME') + def test_set_path(self): + env = EnvironmentModifications() + env.set_path('A', ['foo', 'bar', 'baz']) + env.apply_modifications() + self.assertEqual('foo:bar:baz', os.environ['A']) + def test_path_manipulation(self): env = EnvironmentModifications() @@ -51,7 +57,7 @@ def test_path_manipulation(self): def test_extra_arguments(self): env = EnvironmentModifications() - env.set_env('A', 'dummy value', who='Pkg1') + env.set('A', 'dummy value', who='Pkg1') for x in env: assert 'who' in x.args env.apply_modifications() @@ -59,8 +65,8 @@ def test_extra_arguments(self): def test_extend(self): env = EnvironmentModifications() - env.set_env('A', 'dummy value') - env.set_env('B', 3) + env.set('A', 'dummy value') + env.set('B', 3) copy_construct = EnvironmentModifications(env) self.assertEqual(len(copy_construct), 2) for x, y in zip(env, copy_construct): diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 90b5d42eab4..c4d9940bb72 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -48,11 +48,11 @@ class Mpich(Package): provides('mpi@:1.3', when='@1:') def setup_dependent_environment(self, env, dependent_spec): - env.set_env('MPICH_CC', spack_cc) - env.set_env('MPICH_CXX', spack_cxx) - env.set_env('MPICH_F77', spack_f77) - env.set_env('MPICH_F90', spack_f90) - env.set_env('MPICH_FC', spack_fc) + env.set('MPICH_CC', spack_cc) + env.set('MPICH_CXX', spack_cxx) + env.set('MPICH_F77', spack_f77) + env.set('MPICH_F90', spack_f90) + env.set('MPICH_FC', spack_fc) def setup_dependent_python_module(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index c91a13e3760..9a127f18123 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -43,10 +43,10 @@ def url_for_version(self, version): def setup_dependent_environment(self, spack_env, run_env, dependent_spec): - spack_env.set_env('OMPI_CC', spack_cc) - spack_env.set_env('OMPI_CXX', spack_cxx) - spack_env.set_env('OMPI_FC', spack_fc) - spack_env.set_env('OMPI_F77', spack_f77) + spack_env.set('OMPI_CC', spack_cc) + spack_env.set('OMPI_CXX', spack_cxx) + spack_env.set('OMPI_FC', spack_fc) + spack_env.set('OMPI_F77', spack_f77) def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 593a27708c8..4f55bc803ea 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -104,8 +104,8 @@ def setup_dependent_environment(self, spack_env, run_env, extension_spec): python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) pythonpath = ':'.join(python_paths) - spack_env.set_env('PYTHONPATH', pythonpath) - run_env.set_env('PYTHONPATH', pythonpath) + spack_env.set('PYTHONPATH', pythonpath) + run_env.set('PYTHONPATH', pythonpath) def modify_module(self, module, spec, ext_spec): diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 039aeb3c314..d08e8e81e1a 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -57,11 +57,11 @@ class Qt(Package): def setup_environment(self, spack_env, env): - env.set_env('QTDIR', self.prefix) + env.set('QTDIR', self.prefix) def setup_dependent_environment(self, spack_env, run_env, dspec): - spack_env.set_env('QTDIR', self.prefix) + spack_env.set('QTDIR', self.prefix) def patch(self): diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py index 39f65f51d2c..7ff1898ce91 100644 --- a/var/spack/repos/builtin/packages/ruby/package.py +++ b/var/spack/repos/builtin/packages/ruby/package.py @@ -25,9 +25,10 @@ def setup_dependent_environment(self, spack_env, run_env, extension_spec): if d.package.extends(self.spec): ruby_paths.append(d.prefix) - spack_env.set_env('GEM_PATH', concatenate_paths(ruby_paths)) + spack_env.set_path('GEM_PATH', ruby_paths) + # The actual installation path for this gem - spack_env.set_env('GEM_HOME', extension_spec.prefix) + spack_env.set('GEM_HOME', extension_spec.prefix) def modify_module(self, module, spec, ext_spec): """Called before ruby modules' install() methods. Sets GEM_HOME From 4b46e21685b557188d8738bf11af2e0195c14046 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Mon, 21 Mar 2016 19:55:54 -0600 Subject: [PATCH 653/910] + Provide package numdiff. --- .../repos/builtin/packages/numdiff/package.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 var/spack/repos/builtin/packages/numdiff/package.py diff --git a/var/spack/repos/builtin/packages/numdiff/package.py b/var/spack/repos/builtin/packages/numdiff/package.py new file mode 100644 index 00000000000..a51f6206cf1 --- /dev/null +++ b/var/spack/repos/builtin/packages/numdiff/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Numdiff(Package): + """Numdiff is a little program that can be used to compare putatively + similar files line by line and field by field, ignoring small numeric + differences or/and different numeric formats.""" + + homepage = 'https://www.nongnu.org/numdiff' + url = 'http://nongnu.askapache.com/numdiff/numdiff-5.8.1.tar.gz' + + version('5.8.1', 'a295eb391f6cb1578209fc6b4f9d994e') + + def install(self, spec, prefix): + options = ['--prefix=%s' % prefix] + configure(*options) + make() + make('install') From 7ec450912dc71915205857da75bb418ffc3d0904 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Mon, 21 Mar 2016 21:14:03 -0600 Subject: [PATCH 654/910] + Provide subversion and required packages apr and apr-util. + Installation of subversion may need to be modified to include python and perl bindings. I have provided comments to this end, but the code is not active. --- .../packages/apr-util/apr-util.new/package.py | 44 ++++++++++++++ .../builtin/packages/apr/apr.new/package.py | 51 ++++++++++++++++ .../subversion/subversion.new/package.py | 59 +++++++++++++++++++ .../repos/builtin/packages/swig/package.py | 5 +- 4 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 var/spack/repos/builtin/packages/apr-util/apr-util.new/package.py create mode 100644 var/spack/repos/builtin/packages/apr/apr.new/package.py create mode 100644 var/spack/repos/builtin/packages/subversion/subversion.new/package.py diff --git a/var/spack/repos/builtin/packages/apr-util/apr-util.new/package.py b/var/spack/repos/builtin/packages/apr-util/apr-util.new/package.py new file mode 100644 index 00000000000..8f19c84d225 --- /dev/null +++ b/var/spack/repos/builtin/packages/apr-util/apr-util.new/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class AprUtil(Package): + """Apache Portable Runtime Utility""" + homepage = 'https://apr.apache.org/' + url = 'http://archive.apache.org/dist/apr/apr-util-1.5.4.tar.gz' + + version('1.5.4', '866825c04da827c6e5f53daff5569f42') + + depends_on('apr') + + def install(self, spec, prefix): + + # configure, build, install: + options = ['--prefix=%s' % prefix] + options.append('--with-apr=%s' % spec['apr'].prefix) + + configure(*options) + make() + make('install') diff --git a/var/spack/repos/builtin/packages/apr/apr.new/package.py b/var/spack/repos/builtin/packages/apr/apr.new/package.py new file mode 100644 index 00000000000..5cbbad350bd --- /dev/null +++ b/var/spack/repos/builtin/packages/apr/apr.new/package.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Apr(Package): + """Apache portable runtime.""" + homepage = 'https://apr.apache.org/' + url = 'http://archive.apache.org/dist/apr/apr-1.5.2.tar.gz' + + version('1.5.2', '98492e965963f852ab29f9e61b2ad700') + + #variant('ncurses', default=True, description='Enables the build of the ncurses gui') + #variant('qt', default=False, description='Enables the build of cmake-gui') + #variant('doc', default=False, description='Enables the generation of html and man page documentation') + + #depends_on('ncurses', when='+ncurses') + #depends_on('qt', when='+qt') + #depends_on('python@2.7.11:', when='+doc') + #depends_on('py-sphinx', when='+doc') + + #def url_for_version(self, version): + # """Handle CMake's version-based custom URLs.""" + # return 'https://cmake.org/files/v%s/cmake-%s.tar.gz' % (version.up_to(2), version) + + def install(self, spec, prefix): + options = ['--prefix=%s' % prefix] + configure(*options) + make() + make('install') diff --git a/var/spack/repos/builtin/packages/subversion/subversion.new/package.py b/var/spack/repos/builtin/packages/subversion/subversion.new/package.py new file mode 100644 index 00000000000..b1c3380238a --- /dev/null +++ b/var/spack/repos/builtin/packages/subversion/subversion.new/package.py @@ -0,0 +1,59 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +#import os + +class Subversion(Package): + """Apache Subversion - an open source version control system.""" + homepage = 'https://subversion.apache.org/' + url = 'http://archive.apache.org/dist/subversion/subversion-1.8.13.tar.gz' + + version('1.8.13', '8065b3698d799507fb72dd7926ed32b6') + version('1.9.3', 'a92bcfaec4e5038f82c74a7b5bbd2f46') + + depends_on('apr') + depends_on('apr-util') + depends_on('zlib') + depends_on('sqlite') + #depends_on('swig') + + def install(self, spec, prefix): + + # configure, build, install: + # Ref: http://www.linuxfromscratch.org/blfs/view/svn/general/subversion.html + options = ['--prefix=%s' % prefix] + options.append('--with-apr=%s' % spec['apr'].prefix) + options.append('--with-apr-util=%s' % spec['apr-util'].prefix) + options.append('--with-zlib=%s' % spec['zlib'].prefix) + options.append('--with-sqlite=%s' % spec['sqlite'].prefix) + #options.append('--with-swig=%s' % spec['swig'].prefix) + + configure(*options) + make() + make('install') + #make('swig-py') # python bindings + #make('install-swig-py') + #make('swig-pl') # perl bindings + #make('install-swig-pl') diff --git a/var/spack/repos/builtin/packages/swig/package.py b/var/spack/repos/builtin/packages/swig/package.py index 8d46c4fe465..de6055e9656 100644 --- a/var/spack/repos/builtin/packages/swig/package.py +++ b/var/spack/repos/builtin/packages/swig/package.py @@ -36,11 +36,14 @@ class Swig(Package): homepage = "http://www.swig.org" url = "http://prdownloads.sourceforge.net/swig/swig-3.0.2.tar.gz" - version('3.0.2', '62f9b0d010cef36a13a010dc530d0d41') + version('3.0.8', 'c96a1d5ecb13d38604d7e92148c73c97') + version('3.0.2', '62f9b0d010cef36a13a010dc530d0d41') + version('2.0.12', 'c3fb0b2d710cc82ed0154b91e43085a4') depends_on('pcre') def install(self, spec, prefix): configure("--prefix=%s" % prefix) + make() make("install") From 23bf70296e4a02f49c301b1b744055802b1e8dc1 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 09:22:42 +0100 Subject: [PATCH 655/910] arpack: add a patch for 3.3 --- .../repos/builtin/packages/arpack-ng/package.py | 4 ++++ .../builtin/packages/arpack-ng/pdlamch10.patch | 15 +++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 var/spack/repos/builtin/packages/arpack-ng/pdlamch10.patch diff --git a/var/spack/repos/builtin/packages/arpack-ng/package.py b/var/spack/repos/builtin/packages/arpack-ng/package.py index 0b49d142024..614071cf534 100644 --- a/var/spack/repos/builtin/packages/arpack-ng/package.py +++ b/var/spack/repos/builtin/packages/arpack-ng/package.py @@ -35,6 +35,10 @@ class ArpackNg(Package): variant('shared', default=True, description='Enables the build of shared libraries') variant('mpi', default=False, description='Activates MPI support') + # The function pdlamch10 does not set the return variable. This is fixed upstream + # see https://github.com/opencollab/arpack-ng/issues/34 + patch('pdlamch10.patch', when='@3.3:') + depends_on('blas') depends_on('lapack') depends_on('mpi', when='+mpi') diff --git a/var/spack/repos/builtin/packages/arpack-ng/pdlamch10.patch b/var/spack/repos/builtin/packages/arpack-ng/pdlamch10.patch new file mode 100644 index 00000000000..922828909f8 --- /dev/null +++ b/var/spack/repos/builtin/packages/arpack-ng/pdlamch10.patch @@ -0,0 +1,15 @@ +diff --git a/PARPACK/SRC/MPI/pdlamch10.f b/PARPACK/SRC/MPI/pdlamch10.f +index 6571da9..2882c2e 100644 +--- a/PARPACK/SRC/MPI/pdlamch10.f ++++ b/PARPACK/SRC/MPI/pdlamch10.f +@@ -86,8 +86,8 @@ + TEMP = TEMP1 + END IF + * +- PDLAMCH = TEMP ++ PDLAMCH10 = TEMP + * +-* End of PDLAMCH ++* End of PDLAMCH10 + * + END From aca8b5c89de7058ab58f2eb3392e8b023b16fe8a Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 09:27:00 +0100 Subject: [PATCH 656/910] move suite sparse to a proper folder --- .../builtin/packages/{SuiteSparse => suite-sparse}/package.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename var/spack/repos/builtin/packages/{SuiteSparse => suite-sparse}/package.py (100%) diff --git a/var/spack/repos/builtin/packages/SuiteSparse/package.py b/var/spack/repos/builtin/packages/suite-sparse/package.py similarity index 100% rename from var/spack/repos/builtin/packages/SuiteSparse/package.py rename to var/spack/repos/builtin/packages/suite-sparse/package.py From 3c5a1605d5e3cba881b743bf561f0966b1a3ad8b Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 09:27:39 +0100 Subject: [PATCH 657/910] adjust the class name for suite sparse --- var/spack/repos/builtin/packages/suite-sparse/package.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/suite-sparse/package.py b/var/spack/repos/builtin/packages/suite-sparse/package.py index 6e130d118fe..b57f9967c30 100644 --- a/var/spack/repos/builtin/packages/suite-sparse/package.py +++ b/var/spack/repos/builtin/packages/suite-sparse/package.py @@ -1,7 +1,7 @@ from spack import * -class Suitesparse(Package): +class SuiteSparse(Package): """ SuiteSparse is a suite of sparse matrix algorithms """ @@ -24,4 +24,3 @@ def install(self, spec, prefix): # FIXME : this actually uses the current workaround # FIXME : (blas / lapack always provide libblas and liblapack as aliases) make('install', 'INSTALL=%s' % prefix, 'BLAS=-lblas', 'LAPACK=-llapack') - From cb97e8dd9301e5bbc6752caed2f4861a2a06f494 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 09:43:10 +0100 Subject: [PATCH 658/910] adjust eigen and octave to use new names of suite sparse --- var/spack/repos/builtin/packages/eigen/package.py | 2 +- var/spack/repos/builtin/packages/octave/package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/eigen/package.py b/var/spack/repos/builtin/packages/eigen/package.py index e40046b4529..8d6e672f86c 100644 --- a/var/spack/repos/builtin/packages/eigen/package.py +++ b/var/spack/repos/builtin/packages/eigen/package.py @@ -48,7 +48,7 @@ class Eigen(Package): depends_on('metis', when='+metis') depends_on('scotch', when='+scotch') depends_on('fftw', when='+fftw') - depends_on('SuiteSparse', when='+suitesparse') + depends_on('suite-sparse', when='+suitesparse') depends_on('mpfr@2.3.0:') # Eigen 3.2.7 requires at least 2.3.0 depends_on('gmp') diff --git a/var/spack/repos/builtin/packages/octave/package.py b/var/spack/repos/builtin/packages/octave/package.py index 38b355159dd..6e99c236520 100644 --- a/var/spack/repos/builtin/packages/octave/package.py +++ b/var/spack/repos/builtin/packages/octave/package.py @@ -62,7 +62,7 @@ class Octave(Package): depends_on('qrupdate', when='+qrupdate') #depends_on('qscintilla', when='+qscintilla) # TODO: add package depends_on('qt', when='+qt') - depends_on('SuiteSparse', when='+suitesparse') + depends_on('suite-sparse',when='+suitesparse') depends_on('zlib', when='+zlib') From a26992ef55fed958c15b45b989fc0a4d57f02251 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 22 Mar 2016 01:56:16 -0700 Subject: [PATCH 659/910] Change from PR #552: rename setup_dependent_python_module -> setup_dependent_package - Fixed in package.py - Fixed wrong prototypes in packages that use it. - Fixed build_environment to set module variables properly - added hacky fix to ensure spec/package consistency in build processes. - Need to think about defensive spec copy done by `Repo.get`. May be time to think about an immutable spec implementation. --- lib/spack/spack/build_environment.py | 52 ++++++++++++++----- lib/spack/spack/package.py | 2 +- .../repos/builtin/packages/mpich/package.py | 2 +- .../packages/netlib-scalapack/package.py | 22 ++++---- 4 files changed, 52 insertions(+), 26 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index fc5b7d6207f..119a255a349 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -225,7 +225,7 @@ def set_module_variables_for_package(pkg, module): m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc']) m.spack_cxx = join_path(link_dir, pkg.compiler.link_paths['cxx']) m.spack_f77 = join_path(link_dir, pkg.compiler.link_paths['f77']) - m.spack_f90 = join_path(link_dir, pkg.compiler.link_paths['fc']) + m.spack_fc = join_path(link_dir, pkg.compiler.link_paths['fc']) # Emulate some shell commands for convenience m.pwd = os.getcwd @@ -270,32 +270,56 @@ def parent_class_modules(cls): return result +def setup_module_variables_for_dag(pkg): + """Set module-scope variables for all packages in the DAG.""" + for spec in pkg.spec.traverse(order='post'): + # If a user makes their own package repo, e.g. + # spack.repos.mystuff.libelf.Libelf, and they inherit from + # an existing class like spack.repos.original.libelf.Libelf, + # then set the module variables for both classes so the + # parent class can still use them if it gets called. + spkg = spec.package + modules = parent_class_modules(spkg.__class__) + for mod in modules: + set_module_variables_for_package(spkg, mod) + set_module_variables_for_package(spkg, spkg.module) + + def setup_package(pkg): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() run_env = EnvironmentModifications() + # Before proceeding, ensure that specs and packages are consistent + # + # This is a confusing behavior due to how packages are + # constructed. `setup_dependent_package` may set attributes on + # specs in the DAG for use by other packages' install + # method. However, spec.package will look up a package via + # spack.repo, which defensively copies specs into packages. This + # code ensures that all packages in the DAG have pieces of the + # same spec object at build time. + # + # This is safe for the build process, b/c the build process is a + # throwaway environment, but it is kind of dirty. + # + # TODO: Think about how to avoid this fix and do something cleaner. + for s in pkg.spec.traverse(): s.package.spec = s + set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env) - - # If a user makes their own package repo, e.g. - # spack.repos.mystuff.libelf.Libelf, and they inherit from - # an existing class like spack.repos.original.libelf.Libelf, - # then set the module variables for both classes so the - # parent class can still use them if it gets called. - modules = parent_class_modules(pkg.__class__) - for mod in modules: - set_module_variables_for_package(pkg, mod) + setup_module_variables_for_dag(pkg) # Allow dependencies to modify the module - for dependency_spec in pkg.spec.traverse(root=False): + spec = pkg.spec + for dependency_spec in spec.traverse(root=False): dpkg = dependency_spec.package - dpkg.setup_dependent_python_module(pkg.module, pkg.spec) + dpkg.setup_dependent_package(pkg.module, spec) # Allow dependencies to set up environment as well - for dependency_spec in pkg.spec.traverse(root=False): + for dependency_spec in spec.traverse(root=False): dpkg = dependency_spec.package - dpkg.setup_dependent_environment(spack_env, run_env, pkg.spec) + dpkg.setup_dependent_environment(spack_env, run_env, spec) # Allow the package to apply some settings. pkg.setup_environment(spack_env, run_env) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 9d8ac87bd70..9af32218374 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1075,7 +1075,7 @@ def setup_dependent_environment(self, spack_env, run_env, dependent_spec): self.setup_environment(spack_env, run_env) - def setup_dependent_python_module(self, module, dependent_spec): + def setup_dependent_package(self, module, dependent_spec): """Set up Python module-scope variables for dependent packages. Called before the install() method of dependents. diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index c4d9940bb72..b20dc8dd60c 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -54,7 +54,7 @@ def setup_dependent_environment(self, env, dependent_spec): env.set('MPICH_F90', spack_f90) env.set('MPICH_FC', spack_fc) - def setup_dependent_python_module(self, module, spec, dep_spec): + def setup_dependent_package(self, module, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" # FIXME : is this necessary ? Shouldn't this be part of a contract with MPI providers? module.mpicc = join_path(self.prefix.bin, 'mpicc') diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 62abfcc48e3..c3e6822cdfe 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -1,4 +1,5 @@ from spack import * +import sys class NetlibScalapack(Package): """ScaLAPACK is a library of high-performance linear algebra routines for parallel distributed memory machines""" @@ -11,16 +12,16 @@ class NetlibScalapack(Package): version('2.0.0', '9e76ae7b291be27faaad47cfc256cbfe') # versions before 2.0.0 are not using cmake and requires blacs as # a separated package - + variant('shared', default=True, description='Build the shared library version') variant('fpic', default=False, description="Build with -fpic compiler option") - + provides('scalapack') - + depends_on('mpi') depends_on('lapack') - - def install(self, spec, prefix): + + def install(self, spec, prefix): options = [ "-DBUILD_SHARED_LIBS:BOOL=%s" % ('ON' if '+shared' in spec else 'OFF'), "-DBUILD_STATIC_LIBS:BOOL=%s" % ('OFF' if '+shared' in spec else 'ON'), @@ -40,10 +41,11 @@ def install(self, spec, prefix): make() make("install") - def setup_dependent_python_module(self, module, spec, dependent_spec): + def setup_dependent_package(self, module, dependent_spec): + spec = self.spec lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' - lib_suffix = lib_dsuffix if '+shared' in spec['scalapack'] else '.a' + lib_suffix = lib_dsuffix if '+shared' in spec else '.a' - spec['scalapack'].fc_link = '-L%s -lscalapack' % spec['scalapack'].prefix.lib - spec['scalapack'].cc_link = spec['scalapack'].fc_link - spec['scalapack'].libraries = [join_path(spec['scalapack'].prefix.lib, 'libscalapack%s' % lib_suffix)] + spec.fc_link = '-L%s -lscalapack' % spec.prefix.lib + spec.cc_link = spec.fc_link + spec.libraries = [join_path(spec.prefix.lib, 'libscalapack%s' % lib_suffix)] From 6eb97a30a36f0a0f072826d473a641dd143345ed Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 14:24:27 +0100 Subject: [PATCH 660/910] arpack: fix for +mpi variant --- var/spack/repos/builtin/packages/arpack-ng/package.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/arpack-ng/package.py b/var/spack/repos/builtin/packages/arpack-ng/package.py index 614071cf534..dd86b17a53a 100644 --- a/var/spack/repos/builtin/packages/arpack-ng/package.py +++ b/var/spack/repos/builtin/packages/arpack-ng/package.py @@ -50,7 +50,10 @@ def install(self, spec, prefix): options = ['--prefix=%s' % prefix] if '+mpi' in spec: - options.append('--enable-mpi') + options.extend([ + '--enable-mpi', + 'F77=mpif77' #FIXME: avoid hardcoding MPI wrapper names + ]) if '~shared' in spec: options.append('--enable-shared=no') From d25026b74936f9988cdaf701258c937b3965c106 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 16:26:16 +0100 Subject: [PATCH 661/910] add OCE package --- .../repos/builtin/packages/oce/package.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 var/spack/repos/builtin/packages/oce/package.py diff --git a/var/spack/repos/builtin/packages/oce/package.py b/var/spack/repos/builtin/packages/oce/package.py new file mode 100644 index 00000000000..06acb96736e --- /dev/null +++ b/var/spack/repos/builtin/packages/oce/package.py @@ -0,0 +1,47 @@ +from spack import * +import platform + +class Oce(Package): + """ + Open CASCADE Community Edition: + patches/improvements/experiments contributed by users over the official Open CASCADE library. + """ + homepage = "https://github.com/tpaviot/oce" + url = "https://github.com/tpaviot/oce/archive/OCE-0.17.tar.gz" + + version('0.17.1', '36c67b87093c675698b483454258af91') + version('0.17' , 'f1a89395c4b0d199bea3db62b85f818d') + version('0.16.1', '4d591b240c9293e879f50d86a0cb2bb3') + version('0.16' , '7a4b4df5a104d75a537e25e7dd387eca') + version('0.15' , '7ec541a1c350ca8a684f74980e48801c') + + depends_on('cmake@2.8:') + + def install(self, spec, prefix): + options = [] + options.extend(std_cmake_args) + options.extend([ + '-DOCE_INSTALL_PREFIX=%s' % prefix, + '-DOCE_BUILD_SHARED_LIB:BOOL=ON', + '-DOCE_BUILD_TYPE:STRING=Release', + '-DOCE_DATAEXCHANGE:BOOL=ON', + '-DOCE_DISABLE_X11:BOOL=ON', + '-DOCE_DRAW:BOOL=OFF', + '-DOCE_MODEL:BOOL=ON', + '-DOCE_MULTITHREAD_LIBRARY:STRING=NONE', # FIXME: add tbb + '-DOCE_OCAF:BOOL=ON', + '-DOCE_USE_TCL_TEST_FRAMEWORK:BOOL=OFF', + '-DOCE_VISUALISATION:BOOL=OFF', + '-DOCE_WITH_FREEIMAGE:BOOL=OFF', + '-DOCE_WITH_GL2PS:BOOL=OFF', + '-DOCE_WITH_OPENCL:BOOL=OFF' + ]) + + if platform.system() == 'Darwin': + options.extend([ + '-DOCE_OSX_USE_COCOA:BOOL=ON', + ]) + + cmake('.', *options) + + make("install/strip") From 1f8d79b69ab005a6a372ff34806525148c0ed854 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Tue, 22 Mar 2016 09:33:49 -0600 Subject: [PATCH 662/910] + Provide a ~perl variant to support building on systems w/o the perl-devel package installed. --- var/spack/repos/builtin/packages/graphviz/package.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/graphviz/package.py b/var/spack/repos/builtin/packages/graphviz/package.py index 2a2102bfcb6..84bd696af2b 100644 --- a/var/spack/repos/builtin/packages/graphviz/package.py +++ b/var/spack/repos/builtin/packages/graphviz/package.py @@ -8,7 +8,6 @@ class Graphviz(Package): version('2.38.0', '5b6a829b2ac94efcd5fa3c223ed6d3ae') variant('perl', default=True, description='Disable if you have problems with the optional script language bindings') - variant('shared', default=True, description='Building static is required on AIX') parallel = False @@ -20,8 +19,6 @@ def install(self, spec, prefix): options = ['--prefix=%s' % prefix] if '~perl' in spec: options.append('--disable-perl') - if '~shared' in spec: - options.append('--enable-shared=no') configure(*options) make() From 7618dc504cb58e115a0a95b130bcfc485cd8341d Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 22 Mar 2016 19:17:39 +0100 Subject: [PATCH 663/910] temporary fix Hypre for OSX by building static lib by default --- var/spack/repos/builtin/packages/hypre/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py index 242ee100d7a..8d93d48d1f6 100644 --- a/var/spack/repos/builtin/packages/hypre/package.py +++ b/var/spack/repos/builtin/packages/hypre/package.py @@ -1,5 +1,5 @@ from spack import * -import os +import os, sys class Hypre(Package): """Hypre is a library of high performance preconditioners that @@ -12,7 +12,8 @@ class Hypre(Package): version('2.10.1', 'dc048c4cabb3cd549af72591474ad674') version('2.10.0b', '768be38793a35bb5d055905b271f5b8e') - variant('shared', default=True, description="Build shared library version (disables static library)") + # hypre does not know how to build shared libraries on Darwin + variant('shared', default=sys.platform!='darwin', description="Build shared library version (disables static library)") depends_on("mpi") depends_on("blas") From e56f5c4d6b4626a428b3c209f00eff59cd37b22f Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Tue, 22 Mar 2016 16:59:40 +0100 Subject: [PATCH 664/910] add Intel TBB package --- .../repos/builtin/packages/tbb/package.py | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 var/spack/repos/builtin/packages/tbb/package.py diff --git a/var/spack/repos/builtin/packages/tbb/package.py b/var/spack/repos/builtin/packages/tbb/package.py new file mode 100644 index 00000000000..56ffe4c27c6 --- /dev/null +++ b/var/spack/repos/builtin/packages/tbb/package.py @@ -0,0 +1,79 @@ +from spack import * +import os +import glob + +class Tbb(Package): + """Widely used C++ template library for task parallelism. + Intel Threading Building Blocks (Intel TBB) lets you easily write parallel + C++ programs that take full advantage of multicore performance, that are + portable and composable, and that have future-proof scalability. + """ + homepage = "http://www.threadingbuildingblocks.org/" + + # Only version-specific URL's work for TBB + version('4.4.3', '80707e277f69d9b20eeebdd7a5f5331137868ce1', url='https://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb44_20160128oss_src_0.tgz') + + def coerce_to_spack(self,tbb_build_subdir): + for compiler in ["icc","gcc","clang"]: + fs = glob.glob(join_path(tbb_build_subdir,"*.%s.inc" % compiler )) + for f in fs: + lines = open(f).readlines() + of = open(f,"w") + for l in lines: + if l.strip().startswith("CPLUS ="): + of.write("# coerced to spack\n") + of.write("CPLUS = $(CXX)\n") + elif l.strip().startswith("CPLUS ="): + of.write("# coerced to spack\n") + of.write("CONLY = $(CC)\n") + else: + of.write(l); + + def install(self, spec, prefix): + # + # we need to follow TBB's compiler selection logic to get the proper build + link flags + # but we still need to use spack's compiler wrappers + # to accomplish this, we do two things: + # + # * Look at the spack spec to determine which compiler we should pass to tbb's Makefile + # + # * patch tbb's build system to use the compiler wrappers (CC, CXX) for + # icc, gcc, clang + # (see coerce_to_spack()) + # + self.coerce_to_spack("build") + + if spec.satisfies('%clang'): + tbb_compiler = "clang" + elif spec.satisfies('%intel'): + tbb_compiler = "icc" + else: + tbb_compiler = "gcc" + + + mkdirp(prefix) + mkdirp(prefix.lib) + + # + # tbb does not have a configure script or make install target + # we simply call make, and try to put the pieces together + # + make("compiler=%s" %(tbb_compiler)) + + # install headers to {prefix}/include + install_tree('include',prefix.include) + + # install libs to {prefix}/lib + tbb_lib_names = ["libtbb", + "libtbbmalloc", + "libtbbmalloc_proxy"] + + for lib_name in tbb_lib_names: + # install release libs + fs = glob.glob(join_path("build","*release",lib_name + ".*")) + for f in fs: + install(f, prefix.lib) + # install debug libs if they exist + fs = glob.glob(join_path("build","*debug",lib_name + "_debug.*")) + for f in fs: + install(f, prefix.lib) From 1eb7b8cf4624910121ac66996f17a212ed2cfc91 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Tue, 22 Mar 2016 12:31:27 -0600 Subject: [PATCH 665/910] + Add a validate function that refuses to build numdiff if arch is darwin-x86_64. --- .../repos/builtin/packages/numdiff/package.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/var/spack/repos/builtin/packages/numdiff/package.py b/var/spack/repos/builtin/packages/numdiff/package.py index a51f6206cf1..91286127fbb 100644 --- a/var/spack/repos/builtin/packages/numdiff/package.py +++ b/var/spack/repos/builtin/packages/numdiff/package.py @@ -34,7 +34,22 @@ class Numdiff(Package): version('5.8.1', 'a295eb391f6cb1578209fc6b4f9d994e') + def validate(self, spec): + """ + Checks if we are attempting to build on an incompatible + architecture. + + Ref. https://github.com/davydden/homebrew-dealiisuite/blob/master/numdiff.rb#L13-L15. + + :param spec: spec of the package + :raises RuntimeError: in case of inconsistencies. + """ + + if spec.satisfies("=darwin-x86_64"): + raise RuntimeError(msg) + def install(self, spec, prefix): + self.validate(spec) options = ['--prefix=%s' % prefix] configure(*options) make() From 5f68e14d3bd90e4a71f6366a328dcc97fc6e8a92 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 19:41:31 +0100 Subject: [PATCH 666/910] add gettext package --- .../repos/builtin/packages/gettext/package.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 var/spack/repos/builtin/packages/gettext/package.py diff --git a/var/spack/repos/builtin/packages/gettext/package.py b/var/spack/repos/builtin/packages/gettext/package.py new file mode 100644 index 00000000000..05712d7392c --- /dev/null +++ b/var/spack/repos/builtin/packages/gettext/package.py @@ -0,0 +1,30 @@ +from spack import * + +class Gettext(Package): + """GNU internationalization (i18n) and localization (l10n) library.""" + homepage = "https://www.gnu.org/software/gettext/" + url = "http://ftpmirror.gnu.org/gettext/gettext-0.19.7.tar.xz" + + version('0.19.7', 'f81e50556da41b44c1d59ac93474dca5') + + def install(self, spec, prefix): + options = ['--disable-dependency-tracking', + '--disable-silent-rules', + '--disable-debug', + '--prefix=%s' % prefix, + '--with-included-gettext', + '--with-included-glib', + '--with-included-libcroco', + '--with-included-libunistring', + '--with-emacs', + '--with-lispdir=%s/emacs/site-lisp/gettext' % prefix.share, + '--disable-java', + '--disable-csharp', + '--without-git', # Don't use VCS systems to create these archives + '--without-cvs', + '--without-xz'] + + configure(*options) + + make() + make("install") From fd1784369a20650c3fc6f84d57ed4e6be59afc44 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 19:50:15 +0100 Subject: [PATCH 667/910] add muparser package --- .../repos/builtin/packages/muparser/package.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 var/spack/repos/builtin/packages/muparser/package.py diff --git a/var/spack/repos/builtin/packages/muparser/package.py b/var/spack/repos/builtin/packages/muparser/package.py new file mode 100644 index 00000000000..a1a9ff90e53 --- /dev/null +++ b/var/spack/repos/builtin/packages/muparser/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Muparser(Package): + """C++ math expression parser library.""" + homepage = "http://muparser.beltoforion.de/" + url = "https://github.com/beltoforion/muparser/archive/v2.2.5.tar.gz" + + version('2.2.5', '02dae671aa5ad955fdcbcd3fee313fb7') + + def install(self, spec, prefix): + options = ['--disable-debug', + '--disable-dependency-tracking', + '--prefix=%s' % prefix] + + configure(*options) + + make() + make("install") From d05e4581c9d249c0736d8d58b394860ca05f8402 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 22 Mar 2016 12:31:26 -0700 Subject: [PATCH 668/910] Added the '+silex' variant to the Silo package. Added a number of different versions to the Silo package. --- .../repos/builtin/packages/silo/package.py | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/silo/package.py b/var/spack/repos/builtin/packages/silo/package.py index d1aed78e0e8..b7894e4d2bd 100644 --- a/var/spack/repos/builtin/packages/silo/package.py +++ b/var/spack/repos/builtin/packages/silo/package.py @@ -5,24 +5,35 @@ class Silo(Package): data to binary, disk files.""" homepage = "http://wci.llnl.gov/simulation/computer-codes/silo" - url = "https://wci.llnl.gov/content/assets/docs/simulation/computer-codes/silo/silo-4.8/silo-4.8.tar.gz" + base_url = "https://wci.llnl.gov/content/assets/docs/simulation/computer-codes/silo" + version('4.10.2', '9ceac777a2f2469ac8cef40f4fab49c8') + version('4.9', 'a83eda4f06761a86726e918fc55e782a') version('4.8', 'b1cbc0e7ec435eb656dc4b53a23663c9') variant('fortran', default=True, description='Enable Fortran support') + variant('silex', default=False, description='Builds Silex, a GUI for viewing Silo files') - depends_on("hdf5") + depends_on('hdf5') + depends_on('qt', when='+silex') def install(self, spec, prefix): config_args = [ '--enable-fortran' if '+fortran' in spec else '--disable-fortran', + '--enable-silex' if '+silex' in spec else '--disable-silex', ] + if '+silex' in spec: + config_args.append('--with-Qt-dir=%s' % spec['qt'].prefix) + configure( - "--prefix=%s" % prefix, - "--with-hdf5=%s,%s" % (spec['hdf5'].prefix.include, spec['hdf5'].prefix.lib), - "--with-zlib=%s,%s" % (spec['zlib'].prefix.include, spec['zlib'].prefix.lib), + '--prefix=%s' % prefix, + '--with-hdf5=%s,%s' % (spec['hdf5'].prefix.include, spec['hdf5'].prefix.lib), + '--with-zlib=%s,%s' % (spec['zlib'].prefix.include, spec['zlib'].prefix.lib), *config_args) make() - make("install") + make('install') + + def url_for_version(self, version): + return '%s/silo-%s/silo-%s.tar.gz' % (Silo.base_url, version, version) From 1728a0c3e60d55d1181f14ef5a677b1652a0680b Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 22 Mar 2016 12:33:11 -0700 Subject: [PATCH 669/910] Updated the 'crypto++' package to use an improved URL formation scheme (inspired by changes made to the 'Silo' package). --- var/spack/repos/builtin/packages/cryptopp/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/cryptopp/package.py b/var/spack/repos/builtin/packages/cryptopp/package.py index 1693c4b160f..743a3ac70f4 100644 --- a/var/spack/repos/builtin/packages/cryptopp/package.py +++ b/var/spack/repos/builtin/packages/cryptopp/package.py @@ -8,8 +8,8 @@ class Cryptopp(Package): public-key encryption (RSA, DSA), and a few obsolete/historical encryption algorithms (MD5, Panama).""" - homepage = "http://www.cryptopp.com/" - url = "http://www.cryptopp.com/cryptopp563.zip" + homepage = "http://www.cryptopp.com" + base_url = "http://www.cryptopp.com" version('5.6.3', '3c5b70e2ec98b7a24988734446242d07') version('5.6.2', '7ed022585698df48e65ce9218f6c6a67') @@ -28,4 +28,4 @@ def url_for_version(self, version): version_tuple = tuple(v for v in iter(version)) version_string = reduce(lambda vs, nv: vs + str(nv), version_tuple, "") - return "%scryptopp%s.zip" % (Cryptopp.homepage, version_string) + return '%s/cryptopp%s.zip' % (Cryptopp.base_url, version_string) From 27f337aa11104a6e625db6e40f8d9414954123ee Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 11:44:38 +0100 Subject: [PATCH 670/910] petsc: set PETSC_DIR in dependent_env --- var/spack/repos/builtin/packages/petsc/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 7239baaf7f5..3dd117eed11 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -100,3 +100,7 @@ def install(self, spec, prefix): # PETSc has its own way of doing parallel make. make('MAKE_NP=%s' % make_jobs, parallel=False) make("install") + + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + # set up PETSC_DIR for everyone using PETSc package + spack_env.set('PETSC_DIR', self.prefix) From 24264eb00e4f4e3661ea1a28e9cbe3a97cd69580 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 11:45:18 +0100 Subject: [PATCH 671/910] add slepc package --- .../repos/builtin/packages/slepc/package.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 var/spack/repos/builtin/packages/slepc/package.py diff --git a/var/spack/repos/builtin/packages/slepc/package.py b/var/spack/repos/builtin/packages/slepc/package.py new file mode 100644 index 00000000000..8b5f24394f6 --- /dev/null +++ b/var/spack/repos/builtin/packages/slepc/package.py @@ -0,0 +1,49 @@ +import os +from spack import * + + +class Slepc(Package): + """ + Scalable Library for Eigenvalue Computations. + """ + + homepage = "http://www.grycap.upv.es/slepc" + url = "http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz" + + version('3.6.2', '2ab4311bed26ccf7771818665991b2ea3a9b15f97e29fd13911ab1293e8e65df') + + variant('arpack', default=False, description='Enables Arpack wrappers') + + depends_on('petsc') + depends_on('arpack-ng~mpi',when='+arpack^petsc~mpi') + depends_on('arpack-ng+mpi',when='+arpack^petsc+mpi') + + def install(self, spec, prefix): + # set SLEPC_DIR for installation + os.environ['SLEPC_DIR'] = self.stage.source_path + + options = [] + + if '+arpack' in spec: + options.extend([ + '--with-arpack-dir=%s' % spec['arpack-ng'].prefix.lib, + ]) + if 'arpack-ng~mpi' in spec: + options.extend([ + '--with-arpack-flags=-larpack' + ]) + else: + options.extend([ + '--with-arpack-flags=-lparpack,-larpack' + ]) + + configure('--prefix=%s' % prefix, *options) + + make('MAKE_NP=%s' % make_jobs, parallel=False) + #FIXME: + # make('test') + make('install') + + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + # set up SLEPC_DIR for everyone using SLEPc package + spack_env.set('SLEPC_DIR', self.prefix) From 6c8d51f403213cf3a49373d4dca03fac565a73b4 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 22:58:14 +0100 Subject: [PATCH 672/910] add p4est package --- .../repos/builtin/packages/p4est/package.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 var/spack/repos/builtin/packages/p4est/package.py diff --git a/var/spack/repos/builtin/packages/p4est/package.py b/var/spack/repos/builtin/packages/p4est/package.py new file mode 100644 index 00000000000..1e2969fe642 --- /dev/null +++ b/var/spack/repos/builtin/packages/p4est/package.py @@ -0,0 +1,34 @@ +from spack import * + +class P4est(Package): + """Dynamic management of a collection (a forest) of adaptive octrees in parallel""" + homepage = "http://www.p4est.org" + url = "http://p4est.github.io/release/p4est-1.1.tar.gz" + + version('1.1', '37ba7f4410958cfb38a2140339dbf64f') + + # disable by default to make it work on frontend of clusters + variant('tests', default=False, description='Run small tests') + + depends_on('mpi') + + def install(self, spec, prefix): + options = ['--enable-mpi', + '--enable-shared', + '--disable-vtk-binary', + '--without-blas', + 'CPPFLAGS=-DSC_LOG_PRIORITY=SC_LP_ESSENTIAL', + 'CFLAGS=-O2', + 'CC=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), # TODO: use ENV variables or MPI class wrappers + 'CXX=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpic++'), + 'FC=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + 'F77=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif77'), + ] + + configure('--prefix=%s' % prefix, *options) + + make() + if '+tests' in self.spec: + make("check") + + make("install") From 0ee993569f8f47f5ca94873bc779d98639e9a1b4 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 23:04:59 +0100 Subject: [PATCH 673/910] add deal.ii package --- .../repos/builtin/packages/dealii/package.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 var/spack/repos/builtin/packages/dealii/package.py diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py new file mode 100644 index 00000000000..cddf5cd98ba --- /dev/null +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -0,0 +1,67 @@ +from spack import * + +class Dealii(Package): + """C++ software library providing well-documented tools to build finite element codes for a broad variety of PDEs.""" + homepage = "https://www.dealii.org" + url = "https://github.com/dealii/dealii/releases/download/v8.4.0/dealii-8.4.0.tar.gz" + + version('8.4.0', 'ac5dbf676096ff61e092ce98c80c2b00') + + depends_on ("cmake") + depends_on ("blas") + depends_on ("lapack") + depends_on ("mpi") + + #depends_on ("arpack") + depends_on ("boost") + #depends_on ("doxygen") + depends_on ("hdf5+mpi") + depends_on ("metis") + #depends_on "muparser" + depends_on ("netcdf") + #depends_on ("numdiff") + #depends_on ("oce") + depends_on ("p4est") + depends_on ("parmetis") + depends_on ("petsc") + #depends_on ("slepc") + depends_on ("SuiteSparse") + #depends_on "tbb" + depends_on ("trilinos") + + def install(self, spec, prefix): + options = [] + options.extend(std_cmake_args) + + # CMAKE_BUILD_TYPE should be DebugRelease | Debug | Release + for word in options[:]: + if word.startswith('-DCMAKE_BUILD_TYPE'): + options.remove(word) + + options.extend([ + '-DCMAKE_BUILD_TYPE=DebugRelease', + '-DDEAL_II_WITH_THREADS:BOOL=ON' + '-DDEAL_II_WITH_MPI:BOOL=ON', + '-DCMAKE_C_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), + '-DCMAKE_CXX_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpic++'), + '-DCMAKE_Fortran_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + #'-DARPACK_DIR=%s' % , + '-DBOOST_DIR=%s' % spec['boost'].prefix, + '-DHDF5_DIR=%s' % spec['hdf5'].prefix, + '-DMETIS_DIR=%s' % spec['metis'].prefix, + #'-DMUPARSER_DIR=%s ' + '-DNETCDF_DIR=%s' % spec['netcdf'].prefix, + #'-DOPENCASCADE_DIR= + '-DP4EST_DIR=%s' % spec['p4est'].prefix, + '-DPETSC_DIR=%s' % spec['petsc'].prefix, + #'-DSLEPC_DIR= + '-DUMFPACK_DIR=%s' % spec['SuiteSparse'].prefix, + #'-DTBB_DIR=%s' + '-DTRILINOS_DIR=%s' % spec['trilinos'].prefix + ]) + + cmake('.', *options) + + make() + make("test") + make("install") From 474db2085e223b8226a6ae799232bb2fede3efde Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 22 Mar 2016 15:55:54 -0700 Subject: [PATCH 674/910] Simplified the 'url_for_version' function for the 'Crypto++' package. --- var/spack/repos/builtin/packages/cryptopp/package.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/cryptopp/package.py b/var/spack/repos/builtin/packages/cryptopp/package.py index 743a3ac70f4..bc83cb2b651 100644 --- a/var/spack/repos/builtin/packages/cryptopp/package.py +++ b/var/spack/repos/builtin/packages/cryptopp/package.py @@ -25,7 +25,5 @@ def install(self, spec, prefix): install('libcryptopp.a', prefix.lib) def url_for_version(self, version): - version_tuple = tuple(v for v in iter(version)) - version_string = reduce(lambda vs, nv: vs + str(nv), version_tuple, "") - + version_string = str(version).replace('.', '') return '%s/cryptopp%s.zip' % (Cryptopp.base_url, version_string) From 5b3c96a9bb0ba6501512c93dcdb26510e796314e Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 08:07:45 +0100 Subject: [PATCH 675/910] fix superlu-dist lib installation --- var/spack/repos/builtin/packages/superlu-dist/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/superlu-dist/package.py b/var/spack/repos/builtin/packages/superlu-dist/package.py index c4c76909b35..9a94de8ba50 100644 --- a/var/spack/repos/builtin/packages/superlu-dist/package.py +++ b/var/spack/repos/builtin/packages/superlu-dist/package.py @@ -54,6 +54,7 @@ def install(self, spec, prefix): # need to install by hand headers_location = join_path(self.prefix.include,'superlu_dist') mkdirp(headers_location) + mkdirp(prefix.lib) # FIXME: fetch all headers in the folder automatically for header in ['Cnames.h','cublas_utils.h','dcomplex.h','html_mainpage.h','machines.h','old_colamd.h','psymbfact.h','superlu_ddefs.h','superlu_defs.h','superlu_enum_consts.h','superlu_zdefs.h','supermatrix.h','util_dist.h']: superludist_header = join_path(self.stage.source_path, 'SRC/',header) From c4134ee71e43a6a2a38822e25cc72ca73efdcfea Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 23 Mar 2016 00:36:32 -0700 Subject: [PATCH 676/910] Fix #608: broken numpy build. - Failed to catch all instances of modify_module when it was renamed to setup_dependent_package. - Refactored remaining modify_module calls. - Also modified Python's setup_dependent_package slightly: only creates empty site-packages directory for Python extensions now, not for all dependents. --- lib/spack/spack/modules.py | 4 ++-- lib/spack/spack/preferred_packages.py | 4 +++- var/spack/repos/builtin/packages/python/package.py | 11 ++++++----- var/spack/repos/builtin/packages/ruby/package.py | 6 +++--- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 05c93cd3e65..8ed98e5d38f 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -160,8 +160,8 @@ def write(self): # package-specific modifications for extendee in self.pkg.extendees: extendee_spec = self.spec[extendee] - extendee_spec.package.modify_module( - self.pkg.module, extendee_spec, self.spec) + extendee_spec.package.setup_dependent_package( + self.pkg.module, self.spec) # Package-specific environment modifications spack_env = EnvironmentModifications() diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index 4d8526c75fa..f0a5382dc91 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -150,7 +150,9 @@ def spec_has_preferred_provider(self, pkgname, provider_str): def version_compare(self, pkgname, a, b): """Return less-than-0, 0, or greater than 0 if version a of pkgname is respecively less-than, equal-to, or greater-than version b of pkgname. - One version is less-than another if it is preferred over the other.""" + Versions marked 'preferred=True' in package.py take precedence over any + versions not marked preferred. + """ return self._spec_compare(pkgname, 'version', a, b, True, None) diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 4f55bc803ea..6d9030805b1 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -108,7 +108,7 @@ def setup_dependent_environment(self, spack_env, run_env, extension_spec): run_env.set('PYTHONPATH', pythonpath) - def modify_module(self, module, spec, ext_spec): + def setup_dependent_package(self, module, ext_spec): """ Called before python modules' install() methods. @@ -118,17 +118,18 @@ def modify_module(self, module, spec, ext_spec): """ # Python extension builds can have a global python executable function if self.version >= Version("3.0.0") and self.version < Version("4.0.0"): - module.python = Executable(join_path(spec.prefix.bin, 'python3')) + module.python = Executable(join_path(self.spec.prefix.bin, 'python3')) else: - module.python = Executable(join_path(spec.prefix.bin, 'python')) + module.python = Executable(join_path(self.spec.prefix.bin, 'python')) # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) module.python_include_dir = os.path.join(ext_spec.prefix, self.python_include_dir) module.site_packages_dir = os.path.join(ext_spec.prefix, self.site_packages_dir) - # Make the site packages directory if it does not exist already. - mkdirp(module.site_packages_dir) + # Make the site packages directory for extensions, if it does not exist already. + if ext_spec.package.is_extension: + mkdirp(module.site_packages_dir) # ======================================================================== # Handle specifics of activating and deactivating python modules. diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py index 7ff1898ce91..e13677e4d22 100644 --- a/var/spack/repos/builtin/packages/ruby/package.py +++ b/var/spack/repos/builtin/packages/ruby/package.py @@ -30,7 +30,7 @@ def setup_dependent_environment(self, spack_env, run_env, extension_spec): # The actual installation path for this gem spack_env.set('GEM_HOME', extension_spec.prefix) - def modify_module(self, module, spec, ext_spec): + def setup_dependent_package(self, module, ext_spec): """Called before ruby modules' install() methods. Sets GEM_HOME and GEM_PATH to values appropriate for the package being built. @@ -39,5 +39,5 @@ def modify_module(self, module, spec, ext_spec): gem('install', '.gem') """ # Ruby extension builds have global ruby and gem functions - module.ruby = Executable(join_path(spec.prefix.bin, 'ruby')) - module.gem = Executable(join_path(spec.prefix.bin, 'gem')) + module.ruby = Executable(join_path(self.spec.prefix.bin, 'ruby')) + module.gem = Executable(join_path(self.spec.prefix.bin, 'gem')) From 01918a6f4891ab985b8619daaadc1bf484e4db17 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 05:26:51 +0100 Subject: [PATCH 677/910] fix MUMPS build on OSX (set parallel=False) --- var/spack/repos/builtin/packages/mumps/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index 5c120c37df8..5a254dfd00e 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -135,7 +135,8 @@ def install(self, spec, prefix): self.write_makefile_inc() - make(*make_libs) + # Build fails in parallel, at least on OS-X + make(*make_libs, parallel=False) install_tree('lib', prefix.lib) install_tree('include', prefix.include) From 38350ae33d29e108803cebdf13b90b6898947328 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 23 Mar 2016 01:32:54 -0700 Subject: [PATCH 678/910] resurrect preferred=True option for packages lost in merge of externals support. - Pyton 2.7.11 is preferred again. --- lib/spack/spack/concretize.py | 4 ++++ lib/spack/spack/preferred_packages.py | 4 +--- lib/spack/spack/test/concretize.py | 9 +++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 2e576743ec6..ed9bf798682 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -159,6 +159,10 @@ def concretize_version(self, spec): if any(v.satisfies(sv) for sv in spec.versions)], cmp=cmp_versions) + def prefer_key(v): + return pkg.versions.get(Version(v)).get('preferred', False) + valid_versions.sort(key=prefer_key, reverse=True) + if valid_versions: spec.versions = ver([valid_versions[0]]) else: diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index f0a5382dc91..4d8526c75fa 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -150,9 +150,7 @@ def spec_has_preferred_provider(self, pkgname, provider_str): def version_compare(self, pkgname, a, b): """Return less-than-0, 0, or greater than 0 if version a of pkgname is respecively less-than, equal-to, or greater-than version b of pkgname. - Versions marked 'preferred=True' in package.py take precedence over any - versions not marked preferred. - """ + One version is less-than another if it is preferred over the other.""" return self._spec_compare(pkgname, 'version', a, b, True, None) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 08cce096747..9cd8c969aed 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -24,6 +24,7 @@ ############################################################################## import spack from spack.spec import Spec, CompilerSpec +from spack.version import ver from spack.concretize import find_spec from spack.test.mock_packages_test import * @@ -77,6 +78,14 @@ def test_concretize_variant(self): self.check_concretize('mpich') + def test_concretize_preferred_version(self): + spec = self.check_concretize('python') + self.assertEqual(spec.versions, ver('2.7.11')) + + spec = self.check_concretize('python@3.5.1') + self.assertEqual(spec.versions, ver('3.5.1')) + + def test_concretize_with_virtual(self): self.check_concretize('mpileaks ^mpi') self.check_concretize('mpileaks ^mpi@:1.1') From cc582dd4b435ba06dc140b1ca96b688871e36abb Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 23 Mar 2016 02:02:49 -0700 Subject: [PATCH 679/910] Add mock python package. --- .../builtin.mock/packages/python/package.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 var/spack/repos/builtin.mock/packages/python/package.py diff --git a/var/spack/repos/builtin.mock/packages/python/package.py b/var/spack/repos/builtin.mock/packages/python/package.py new file mode 100644 index 00000000000..c5fed52f534 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/python/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Python(Package): + """Dummy Python package to demonstrate preferred versions.""" + homepage = "http://www.python.org" + url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz" + + extendable = True + + version('3.5.1', 'be78e48cdfc1a7ad90efff146dce6cfe') + version('3.5.0', 'a56c0c0b45d75a0ec9c6dee933c41c36') + version('2.7.11', '6b6076ec9e93f05dd63e47eb9c15728b', preferred=True) + version('2.7.10', 'd7547558fd673bd9d38e2108c6b42521') + version('2.7.9', '5eebcaa0030dc4061156d3429657fb83') + version('2.7.8', 'd4bca0159acb0b44a781292b5231936f') + + def install(self, spec, prefix): + pass + From 0fbdb3f65d234d6ed8f77d3732c134962ad47b35 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 23 Mar 2016 15:43:16 +0100 Subject: [PATCH 680/910] modules : added configuration file with disable keyword --- lib/spack/spack/config.py | 28 +++++++++++++++++++++++++--- lib/spack/spack/modules.py | 12 ++++++++++-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 6afd69b3ac7..6ef79c70b15 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -237,7 +237,29 @@ 'type' : 'object', 'default' : {}, } - },},},},},} + },},},},},}, + 'modules': { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack module file configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + r'modules:?': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'properties': { + 'disable': { + 'type': 'array', + 'default': [], + 'items': { + 'type': 'string' + } + } + } + }, + }, + }, } """OrderedDict of config scopes keyed by name. @@ -405,11 +427,11 @@ def _read_config_file(filename, schema): validate_section(data, schema) return data - except MarkedYAMLError, e: + except MarkedYAMLError as e: raise ConfigFileError( "Error parsing yaml%s: %s" % (str(e.context_mark), e.problem)) - except IOError, e: + except IOError as e: raise ConfigFileError( "Error reading configuration file %s: %s" % (filename, str(e))) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 8ed98e5d38f..639f1101b82 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -48,6 +48,7 @@ import llnl.util.tty as tty import spack +import spack.config from llnl.util.filesystem import join_path, mkdirp from spack.environment import * @@ -57,6 +58,13 @@ module_types = {} +def read_configuration_file(): + f = spack.config.get_config('modules') + f.setdefault('disable', []) # Default : disable nothing + return f + +CONFIGURATION = read_configuration_file() + def print_help(): """For use by commands to tell user how to activate shell support.""" @@ -115,8 +123,8 @@ class EnvModule(object): class __metaclass__(type): def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) - if cls.name != 'env_module': - module_types[cls.name] = cls + if cls.name != 'env_module' and cls.name not in CONFIGURATION['disable']: + module_types[cls.name] = cls def __init__(self, spec=None): self.spec = spec From d93f2b335d5a5198b943cc6293a1d24d614c979b Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 23 Mar 2016 16:04:36 +0100 Subject: [PATCH 681/910] modules : fixed annoying indent --- lib/spack/spack/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 639f1101b82..d354c8bb715 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -124,7 +124,7 @@ class __metaclass__(type): def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) if cls.name != 'env_module' and cls.name not in CONFIGURATION['disable']: - module_types[cls.name] = cls + module_types[cls.name] = cls def __init__(self, spec=None): self.spec = spec From 34a8f0c96bc50870f20f094d95b403b2bb19487d Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 23 Mar 2016 17:01:52 +0100 Subject: [PATCH 682/910] espresso : fixes #454 --- var/spack/repos/builtin/packages/espresso/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index a2bf58f585b..59f362ab46c 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -32,6 +32,10 @@ def check_variants(self, spec): if '+elpa' in spec and ('~mpi' in spec or '~scalapack' in spec): raise RuntimeError(error.format(variant='elpa')) + def setup_environment(self, spack_env, run_env): + # Espresso copies every executable in prefix without creating sub-folders + run_env.prepend_path('PATH', self.prefix) + def install(self, spec, prefix): self.check_variants(spec) From 4e5dfc8b18ed30e261fe6b6dd014c8991937934d Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 19:40:16 +0100 Subject: [PATCH 683/910] fix missing (depricated) _dggsvd_ and _sggsvd_ in Openblas 0.2.16 --- var/spack/repos/builtin/packages/openblas/package.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 781a1e2ec8f..1d10f217c45 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -6,6 +6,7 @@ class Openblas(Package): homepage = "http://www.openblas.net" url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + version('0.2.17', '664a12807f2a2a7cda4781e3ab2ae0e1') version('0.2.16', 'fef46ab92463bdbb1479dcec594ef6dc') version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') @@ -14,7 +15,14 @@ class Openblas(Package): provides('lapack') def install(self, spec, prefix): - make('libs', 'netlib', 'shared', 'CC=cc', 'FC=f77') + extra=[] + if spec.satisfies('@0.2.16'): + extra.extend([ + 'BUILD_LAPACK_DEPRECATED=1' # fix missing _dggsvd_ and _sggsvd_ + ]) + + make('libs', 'netlib', 'shared', 'CC=cc', 'FC=f77',*extra) + make("tests") make('install', "PREFIX='%s'" % prefix) lib_dsuffix = 'dylib' if sys.platform == 'darwin' else 'so' From f095e619b985a9271ff96cd469086d4654edf489 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 24 Mar 2016 09:31:28 +0100 Subject: [PATCH 684/910] module files configuration : enable/disable logic is now positive --- lib/spack/spack/config.py | 2 +- lib/spack/spack/modules.py | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 6ef79c70b15..14e5aaf4fb6 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -249,7 +249,7 @@ 'default': {}, 'additionalProperties': False, 'properties': { - 'disable': { + 'enable': { 'type': 'array', 'default': [], 'items': { diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index d354c8bb715..6c32937c3cc 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -57,13 +57,8 @@ # Registry of all types of modules. Entries created by EnvModule's metaclass module_types = {} +CONFIGURATION = spack.config.get_config('modules') -def read_configuration_file(): - f = spack.config.get_config('modules') - f.setdefault('disable', []) # Default : disable nothing - return f - -CONFIGURATION = read_configuration_file() def print_help(): """For use by commands to tell user how to activate shell support.""" @@ -123,7 +118,7 @@ class EnvModule(object): class __metaclass__(type): def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) - if cls.name != 'env_module' and cls.name not in CONFIGURATION['disable']: + if cls.name != 'env_module' and cls.name in CONFIGURATION['enable']: module_types[cls.name] = cls def __init__(self, spec=None): From e4adc675e58222f3082f167e709e3819beafd474 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 24 Mar 2016 10:04:02 +0100 Subject: [PATCH 685/910] module files configuration : added default configuration --- etc/spack/modules.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 etc/spack/modules.yaml diff --git a/etc/spack/modules.yaml b/etc/spack/modules.yaml new file mode 100644 index 00000000000..aa2a2c3fe29 --- /dev/null +++ b/etc/spack/modules.yaml @@ -0,0 +1,8 @@ +# ------------------------------------------------------------------------- +# This is the default spack module files generation configuration. +# +# Changes to this file will affect all users of this spack install, +# although users can override these settings in their ~/.spack/modules.yaml. +# ------------------------------------------------------------------------- +modules: + enable: ['tcl', 'dotkit'] From 9b8b17b6f1ec22ce4ac2115e5f7687beba21dd19 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 24 Mar 2016 10:36:02 +0100 Subject: [PATCH 686/910] fix parmetis on Ubuntu with GNU compilers by linking to lm --- .../builtin/packages/parmetis/link-to-lm.patch | 14 ++++++++++++++ .../repos/builtin/packages/parmetis/package.py | 2 ++ 2 files changed, 16 insertions(+) create mode 100644 var/spack/repos/builtin/packages/parmetis/link-to-lm.patch diff --git a/var/spack/repos/builtin/packages/parmetis/link-to-lm.patch b/var/spack/repos/builtin/packages/parmetis/link-to-lm.patch new file mode 100644 index 00000000000..faa809231e1 --- /dev/null +++ b/var/spack/repos/builtin/packages/parmetis/link-to-lm.patch @@ -0,0 +1,14 @@ +diff --git a/libparmetis/CMakeLists.txt b/libparmetis/CMakeLists.txt +index 9cfc8a7..dfc0125 100644 +--- a/libparmetis/CMakeLists.txt ++++ b/libparmetis/CMakeLists.txt +@@ -5,7 +5,7 @@ file(GLOB parmetis_sources *.c) + # Create libparmetis + add_library(parmetis ${ParMETIS_LIBRARY_TYPE} ${parmetis_sources}) + # Link with metis and MPI libraries. +-target_link_libraries(parmetis metis ${MPI_LIBRARIES}) ++target_link_libraries(parmetis metis ${MPI_LIBRARIES} "-lm") + set_target_properties(parmetis PROPERTIES LINK_FLAGS "${MPI_LINK_FLAGS}") + + install(TARGETS parmetis + diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py index c691cf4191d..bc71fb72991 100644 --- a/var/spack/repos/builtin/packages/parmetis/package.py +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -52,6 +52,8 @@ class Parmetis(Package): # https://bitbucket.org/petsc/pkg-parmetis/commits/82409d68aa1d6cbc70740d0f35024aae17f7d5cb/raw/ patch('pkg-parmetis-82409d68aa1d6cbc70740d0f35024aae17f7d5cb.patch') + patch('link-to-lm.patch') + depends_on('gdb', when='+gdb') def install(self, spec, prefix): From 758a9c9096bd6c306f2dba0e40b46544dcec1992 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 23 Mar 2016 11:18:11 +0100 Subject: [PATCH 687/910] python extensions : create PYTHONPATH in module files --- lib/spack/spack/modules.py | 10 +++++----- var/spack/repos/builtin/packages/python/package.py | 5 ++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 6c32937c3cc..d45fdde7035 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -161,13 +161,13 @@ def write(self): # Let the extendee modify their extensions before asking for # package-specific modifications - for extendee in self.pkg.extendees: - extendee_spec = self.spec[extendee] - extendee_spec.package.setup_dependent_package( - self.pkg.module, self.spec) + spack_env = EnvironmentModifications() + for item in self.pkg.extendees: + package = self.spec[item].package + package.setup_dependent_package(self.pkg.module, self.spec) + package.setup_dependent_environment(spack_env, env, self.spec) # Package-specific environment modifications - spack_env = EnvironmentModifications() self.spec.package.setup_environment(spack_env, env) # TODO : implement site-specific modifications and filters diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 6d9030805b1..f5237c3b578 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -105,7 +105,10 @@ def setup_dependent_environment(self, spack_env, run_env, extension_spec): pythonpath = ':'.join(python_paths) spack_env.set('PYTHONPATH', pythonpath) - run_env.set('PYTHONPATH', pythonpath) + + # For run time environment set only the path for extension_spec and prepend it to PYTHONPATH + if extension_spec.package.extends(self.spec): + run_env.prepend_path('PYTHONPATH', os.path.join(extension_spec.prefix, self.site_packages_dir)) def setup_dependent_package(self, module, ext_spec): From 8643e2191357dd6f53bfafdfd51f43b770e2137c Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 24 Mar 2016 12:34:58 -0500 Subject: [PATCH 688/910] Simplified NumPy dependencies --- .../builtin/packages/py-numpy/package.py | 37 ++++++++++++++----- .../builtin/packages/py-scipy/package.py | 5 ++- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-numpy/package.py b/var/spack/repos/builtin/packages/py-numpy/package.py index 03548111863..a08e612df6c 100644 --- a/var/spack/repos/builtin/packages/py-numpy/package.py +++ b/var/spack/repos/builtin/packages/py-numpy/package.py @@ -1,24 +1,43 @@ from spack import * class PyNumpy(Package): - """array processing for numbers, strings, records, and objects.""" - homepage = "https://pypi.python.org/pypi/numpy" + """NumPy is the fundamental package for scientific computing with Python. + It contains among other things: a powerful N-dimensional array object, + sophisticated (broadcasting) functions, tools for integrating C/C++ and + Fortran code, and useful linear algebra, Fourier transform, and random + number capabilities""" + homepage = "http://www.numpy.org/" url = "https://pypi.python.org/packages/source/n/numpy/numpy-1.9.1.tar.gz" - version('1.9.1', '78842b73560ec378142665e712ae4ad9') - version('1.9.2', 'a1ed53432dbcd256398898d35bc8e645') + version('1.10.4', 'aed294de0aa1ac7bd3f9745f4f1968ad') + version('1.9.2', 'a1ed53432dbcd256398898d35bc8e645') + version('1.9.1', '78842b73560ec378142665e712ae4ad9') - variant('blas', default=True) + + variant('blas', default=True) + variant('lapack', default=True) extends('python') depends_on('py-nose') - depends_on('netlib-blas+fpic', when='+blas') - depends_on('netlib-lapack+shared', when='+blas') + depends_on('blas', when='+blas') + depends_on('lapack', when='+lapack') def install(self, spec, prefix): + libraries = [] + library_dirs = [] + if '+blas' in spec: + libraries.append('blas') + library_dirs.append(spec['blas'].prefix.lib) + if '+lapack' in spec: + libraries.append('lapack') + library_dirs.append(spec['lapack'].prefix.lib) + + if '+blas' in spec or '+lapack' in spec: with open('site.cfg', 'w') as f: f.write('[DEFAULT]\n') - f.write('libraries=lapack,blas\n') - f.write('library_dirs=%s/lib:%s/lib\n' % (spec['blas'].prefix, spec['lapack'].prefix)) + f.write('libraries=%s\n' % ','.join(libraries)) + f.write('library_dirs=%s\n' % ':'.join(library_dirs)) + python('setup.py', 'install', '--prefix=%s' % prefix) + diff --git a/var/spack/repos/builtin/packages/py-scipy/package.py b/var/spack/repos/builtin/packages/py-scipy/package.py index 3a1124cc15a..c2161c90c43 100644 --- a/var/spack/repos/builtin/packages/py-scipy/package.py +++ b/var/spack/repos/builtin/packages/py-scipy/package.py @@ -2,11 +2,12 @@ class PyScipy(Package): """Scientific Library for Python.""" - homepage = "https://pypi.python.org/pypi/scipy" + homepage = "http://www.scipy.org/" url = "https://pypi.python.org/packages/source/s/scipy/scipy-0.15.0.tar.gz" - version('0.15.0', '639112f077f0aeb6d80718dc5019dc7a') + version('0.17.0', '5ff2971e1ce90e762c59d2cd84837224') version('0.15.1', 'be56cd8e60591d6332aac792a5880110') + version('0.15.0', '639112f077f0aeb6d80718dc5019dc7a') extends('python') depends_on('py-nose') From 254f65f2994de32da8681c24838beb32bbbc5fe1 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Thu, 24 Mar 2016 10:48:38 -0700 Subject: [PATCH 689/910] Remove incorrect sanity check for libelf --- var/spack/repos/builtin/packages/libelf/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/libelf/package.py b/var/spack/repos/builtin/packages/libelf/package.py index 9f16708af5b..29bc21b65c4 100644 --- a/var/spack/repos/builtin/packages/libelf/package.py +++ b/var/spack/repos/builtin/packages/libelf/package.py @@ -38,8 +38,6 @@ class Libelf(Package): provides('elf') - sanity_check_is_file = 'include/libelf.h' - def install(self, spec, prefix): configure("--prefix=" + prefix, "--enable-shared", From 9985816642bad10008f45583a7d715761a6ae03d Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 24 Mar 2016 14:49:41 -0500 Subject: [PATCH 690/910] Fix spack info indentation --- lib/spack/spack/cmd/info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index e7abe7f4a59..c93db55c63c 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -52,7 +52,7 @@ def print_text_info(pkg): print "Safe versions: " if not pkg.versions: - print("None") + print(" None") else: pad = padder(pkg.versions, 4) for v in reversed(sorted(pkg.versions)): @@ -62,7 +62,7 @@ def print_text_info(pkg): print print "Variants:" if not pkg.variants: - print "None" + print " None" else: pad = padder(pkg.variants, 4) From 7fb463abb9b912f9a5bcb6e810b733f0b8bc1672 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 24 Mar 2016 20:23:13 +0100 Subject: [PATCH 691/910] install GKlib headers along with METIS --- .../metis/install_gklib_defs_rename.patch | 22 +++++++++++++++++++ .../repos/builtin/packages/metis/package.py | 13 +++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 var/spack/repos/builtin/packages/metis/install_gklib_defs_rename.patch diff --git a/var/spack/repos/builtin/packages/metis/install_gklib_defs_rename.patch b/var/spack/repos/builtin/packages/metis/install_gklib_defs_rename.patch new file mode 100644 index 00000000000..b182b167b90 --- /dev/null +++ b/var/spack/repos/builtin/packages/metis/install_gklib_defs_rename.patch @@ -0,0 +1,22 @@ +# HG changeset patch +# User Sean Farley +# Date 1332269671 18000 +# Tue Mar 20 13:54:31 2012 -0500 +# Node ID b95c0c2e1d8bf8e3273f7d45e856f0c0127d998e +# Parent 88049269953c67c3fdcc4309bf901508a875f0dc +cmake: add gklib headers to install into include + +diff -r 88049269953c -r b95c0c2e1d8b libmetis/CMakeLists.txt +Index: libmetis/CMakeLists.txt +=================================================================== +--- a/libmetis/CMakeLists.txt Tue Mar 20 13:54:29 2012 -0500 ++++ b/libmetis/CMakeLists.txt Tue Mar 20 13:54:31 2012 -0500 +@@ -12,6 +12,8 @@ endif() + if(METIS_INSTALL) + install(TARGETS metis + LIBRARY DESTINATION lib + RUNTIME DESTINATION lib + ARCHIVE DESTINATION lib) ++ install(FILES gklib_defs.h DESTINATION include) ++ install(FILES gklib_rename.h DESTINATION include) + endif() diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py index bbfc4de7d1d..68b9f6fd306 100644 --- a/var/spack/repos/builtin/packages/metis/package.py +++ b/var/spack/repos/builtin/packages/metis/package.py @@ -24,7 +24,7 @@ ############################################################################## from spack import * - +import glob class Metis(Package): """ @@ -49,6 +49,8 @@ class Metis(Package): depends_on('gdb', when='+gdb') + patch('install_gklib_defs_rename.patch') + def install(self, spec, prefix): options = [] @@ -80,4 +82,11 @@ def install(self, spec, prefix): with working_dir(build_directory, create=True): cmake(source_directory, *options) make() - make("install") \ No newline at end of file + make("install") + + # install GKlib headers, which will be needed for ParMETIS + GKlib_dist = join_path(prefix.include,'GKlib') + mkdirp(GKlib_dist) + fs = glob.glob(join_path(source_directory,'GKlib',"*.h")) + for f in fs: + install(f, GKlib_dist) From 0d4d201b70c57359f041807df0a2f58b47be78d6 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 24 Mar 2016 21:22:58 +0100 Subject: [PATCH 692/910] patch parmetis to use external GKlib --- .../parmetis/enable_external_metis.patch | 48 +++++++++++++++++-- .../builtin/packages/parmetis/package.py | 2 +- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch b/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch index 514781b8b88..8b05ecc872f 100644 --- a/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch +++ b/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch @@ -1,13 +1,55 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt -index ca945dd..1bf94e9 100644 +index ca945dd..5995e42 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt +@@ -23,7 +23,7 @@ else() + set(ParMETIS_LIBRARY_TYPE STATIC) + endif() + +-include(${GKLIB_PATH}/GKlibSystem.cmake) ++include_directories(${GKLIB_PATH}) + + # List of paths that the compiler will search for header files. + # i.e., the -I equivalent @@ -33,7 +33,7 @@ include_directories(${GKLIB_PATH}) include_directories(${METIS_PATH}/include) - + # List of directories that cmake will look for CMakeLists.txt -add_subdirectory(${METIS_PATH}/libmetis ${CMAKE_BINARY_DIR}/libmetis) -+#add_subdirectory(${METIS_PATH}/libmetis ${CMAKE_BINARY_DIR}/libmetis) ++# add_subdirectory(${METIS_PATH}/libmetis ${CMAKE_BINARY_DIR}/libmetis) add_subdirectory(include) add_subdirectory(libparmetis) add_subdirectory(programs) +diff --git a/libparmetis/parmetislib.h b/libparmetis/parmetislib.h +index c1daeeb..07511f6 100644 +--- a/libparmetis/parmetislib.h ++++ b/libparmetis/parmetislib.h +@@ -20,13 +20,12 @@ + + #include + +-#include "../metis/libmetis/gklib_defs.h" ++#include + +-#include ++#include + + #include + #include + #include + #include + #include +- +diff --git a/programs/parmetisbin.h b/programs/parmetisbin.h +index e26cd2d..d156480 100644 +--- a/programs/parmetisbin.h ++++ b/programs/parmetisbin.h +@@ -19,7 +19,7 @@ + #include + #include + +-#include "../metis/libmetis/gklib_defs.h" ++#include + #include "../libparmetis/rename.h" + #include "../libparmetis/defs.h" + #include "../libparmetis/struct.h" diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py index bc71fb72991..5c97a836f9b 100644 --- a/var/spack/repos/builtin/packages/parmetis/package.py +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -66,7 +66,7 @@ def install(self, spec, prefix): # FIXME : Once a contract is defined, MPI compilers should be retrieved indirectly via spec['mpi'] in case # FIXME : they use a non-standard name - options.extend(['-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=metis_source), # still need headers from METIS source, and they are not installed with METIS. shame... + options.extend(['-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=spec['metis'].prefix.include), '-DMETIS_PATH:PATH={metis_source}'.format(metis_source=spec['metis'].prefix), '-DCMAKE_C_COMPILER:STRING=mpicc', '-DCMAKE_CXX_COMPILER:STRING=mpicxx']) From 03c8a91f6a647849f0251496f361da9574a98c22 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 24 Mar 2016 22:33:10 +0100 Subject: [PATCH 693/910] parmetis: avoid hardcoding -lm; use same approach as in METIS --- .../parmetis/enable_external_metis.patch | 20 +++++++++++++++++-- .../packages/parmetis/link-to-lm.patch | 14 ------------- .../builtin/packages/parmetis/package.py | 2 -- 3 files changed, 18 insertions(+), 18 deletions(-) delete mode 100644 var/spack/repos/builtin/packages/parmetis/link-to-lm.patch diff --git a/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch b/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch index 8b05ecc872f..e4f27294836 100644 --- a/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch +++ b/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch @@ -1,5 +1,5 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt -index ca945dd..5995e42 100644 +index ca945dd..aff8b5f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ else() @@ -16,10 +16,26 @@ index ca945dd..5995e42 100644 # List of directories that cmake will look for CMakeLists.txt -add_subdirectory(${METIS_PATH}/libmetis ${CMAKE_BINARY_DIR}/libmetis) -+# add_subdirectory(${METIS_PATH}/libmetis ${CMAKE_BINARY_DIR}/libmetis) ++find_library(METIS_LIBRARY metis PATHS ${METIS_PATH}/lib) add_subdirectory(include) add_subdirectory(libparmetis) add_subdirectory(programs) +diff --git a/libparmetis/CMakeLists.txt b/libparmetis/CMakeLists.txt +index 9cfc8a7..e0c4de7 100644 +--- a/libparmetis/CMakeLists.txt ++++ b/libparmetis/CMakeLists.txt +@@ -5,7 +5,10 @@ file(GLOB parmetis_sources *.c) + # Create libparmetis + add_library(parmetis ${ParMETIS_LIBRARY_TYPE} ${parmetis_sources}) + # Link with metis and MPI libraries. +-target_link_libraries(parmetis metis ${MPI_LIBRARIES}) ++target_link_libraries(parmetis ${METIS_LIBRARY} ${MPI_LIBRARIES}) ++if(UNIX) ++ target_link_libraries(parmetis m) ++endif() + set_target_properties(parmetis PROPERTIES LINK_FLAGS "${MPI_LINK_FLAGS}") + + install(TARGETS parmetis diff --git a/libparmetis/parmetislib.h b/libparmetis/parmetislib.h index c1daeeb..07511f6 100644 --- a/libparmetis/parmetislib.h diff --git a/var/spack/repos/builtin/packages/parmetis/link-to-lm.patch b/var/spack/repos/builtin/packages/parmetis/link-to-lm.patch deleted file mode 100644 index faa809231e1..00000000000 --- a/var/spack/repos/builtin/packages/parmetis/link-to-lm.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/libparmetis/CMakeLists.txt b/libparmetis/CMakeLists.txt -index 9cfc8a7..dfc0125 100644 ---- a/libparmetis/CMakeLists.txt -+++ b/libparmetis/CMakeLists.txt -@@ -5,7 +5,7 @@ file(GLOB parmetis_sources *.c) - # Create libparmetis - add_library(parmetis ${ParMETIS_LIBRARY_TYPE} ${parmetis_sources}) - # Link with metis and MPI libraries. --target_link_libraries(parmetis metis ${MPI_LIBRARIES}) -+target_link_libraries(parmetis metis ${MPI_LIBRARIES} "-lm") - set_target_properties(parmetis PROPERTIES LINK_FLAGS "${MPI_LINK_FLAGS}") - - install(TARGETS parmetis - diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py index 5c97a836f9b..f5b8b6de919 100644 --- a/var/spack/repos/builtin/packages/parmetis/package.py +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -52,8 +52,6 @@ class Parmetis(Package): # https://bitbucket.org/petsc/pkg-parmetis/commits/82409d68aa1d6cbc70740d0f35024aae17f7d5cb/raw/ patch('pkg-parmetis-82409d68aa1d6cbc70740d0f35024aae17f7d5cb.patch') - patch('link-to-lm.patch') - depends_on('gdb', when='+gdb') def install(self, spec, prefix): From fdd7e91ba078bea4f0fec1859124baaa8261fa2c Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 24 Mar 2016 12:49:56 +0100 Subject: [PATCH 694/910] add shared variant to mumps (needed for Trilinos) plus tests --- .../repos/builtin/packages/mumps/Makefile.inc | 5 +-- .../repos/builtin/packages/mumps/package.py | 45 ++++++++++++++++--- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/mumps/Makefile.inc b/var/spack/repos/builtin/packages/mumps/Makefile.inc index 2e6a041878c..22d8f5518a5 100644 --- a/var/spack/repos/builtin/packages/mumps/Makefile.inc +++ b/var/spack/repos/builtin/packages/mumps/Makefile.inc @@ -8,12 +8,9 @@ IORDERINGSF = $(ISCOTCH) IORDERINGSC = $(IMETIS) $(IPORD) $(ISCOTCH) PLAT = -LIBEXT = .a -OUTC = -o +OUTC = -o OUTF = -o RM = /bin/rm -f -AR = ar vr -RANLIB = ranlib INCSEQ = -I$(topdir)/libseq LIBSEQ = -L$(topdir)/libseq -lmpiseq diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index 5a254dfd00e..2801d4194dc 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -1,6 +1,5 @@ from spack import * -import os - +import os, sys class Mumps(Package): """MUMPS: a MUltifrontal Massively Parallel sparse direct Solver""" @@ -19,6 +18,7 @@ class Mumps(Package): variant('float', default=True, description='Activate the compilation of smumps') variant('complex', default=True, description='Activate the compilation of cmumps and/or zmumps') variant('idx64', default=False, description='Use int64_t/integer*8 as default index type') + variant('shared', default=True, description='Build shared libraries') depends_on('scotch + esmumps', when='~ptscotch+scotch') @@ -105,6 +105,27 @@ def write_makefile_inc(self): # compiler possible values are -DAdd_, -DAdd__ and/or -DUPPER makefile_conf.append("CDEFS = -DAdd_") + if '+shared' in self.spec: + if sys.platform == 'darwin': + # Building dylibs with mpif90 causes segfaults on 10.8 and 10.10. Use gfortran. (Homebrew) + makefile_conf.extend([ + 'LIBEXT=.dylib', + 'AR=%s -dynamiclib -undefined dynamic_lookup -o ' % os.environ['FC'], + 'RANLIB=echo' + ]) + else: + makefile_conf.extend([ + 'LIBEXT=.so', + 'AR=$(FL) -shared -o', + 'RANLIB=echo' + ]) + else: + makefile_conf.extend([ + 'LIBEXT = .a', + 'AR = ar vr', + 'RANLIB = ranlib' + ]) + makefile_inc_template = join_path(os.path.dirname(self.module.__file__), 'Makefile.inc') @@ -121,7 +142,7 @@ def write_makefile_inc(self): def install(self, spec, prefix): make_libs = [] - # the coice to compile ?examples is to have kind of a sanity + # the choice to compile ?examples is to have kind of a sanity # check on the libraries generated. if '+float' in spec: make_libs.append('sexamples') @@ -135,10 +156,24 @@ def install(self, spec, prefix): self.write_makefile_inc() - # Build fails in parallel, at least on OS-X + # Build fails in parallel make(*make_libs, parallel=False) install_tree('lib', prefix.lib) install_tree('include', prefix.include) if '~mpi' in spec: - install('libseq/libmpiseq.a', prefix.lib) + lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' + lib_suffix = lib_dsuffix if '+shared' in spec else '.a' + install('libseq/libmpiseq%s' % lib_suffix, prefix.lib) + + # FIXME: extend the tests to mpirun -np 2 (or alike) when build with MPI + # FIXME: use something like numdiff to compare blessed output with the current + with working_dir('examples'): + if '+float' in spec: + os.system('./ssimpletest < input_simpletest_real') + if '+complex' in spec: + os.system('./csimpletest < input_simpletest_real') + if '+double' in spec: + os.system('./dsimpletest < input_simpletest_real') + if '+complex' in spec: + os.system('./zsimpletest < input_simpletest_cmplx') From 540e57f02686bad7125a951d14784e8fcc896e26 Mon Sep 17 00:00:00 2001 From: Josh Sixsmith Date: Fri, 25 Mar 2016 18:27:02 +1100 Subject: [PATCH 695/910] Upload of GDAL, kealib, openjpeg, py-tuiview package builds. --- .../repos/builtin/packages/gdal/package.py | 69 +++++++++++++++++++ .../repos/builtin/packages/kealib/package.py | 35 ++++++++++ .../builtin/packages/openjpeg/package.py | 26 +++++++ .../builtin/packages/py-tuiview/package.py | 19 +++++ 4 files changed, 149 insertions(+) create mode 100644 var/spack/repos/builtin/packages/gdal/package.py create mode 100644 var/spack/repos/builtin/packages/kealib/package.py create mode 100644 var/spack/repos/builtin/packages/openjpeg/package.py create mode 100644 var/spack/repos/builtin/packages/py-tuiview/package.py diff --git a/var/spack/repos/builtin/packages/gdal/package.py b/var/spack/repos/builtin/packages/gdal/package.py new file mode 100644 index 00000000000..4f1f1ec2dd1 --- /dev/null +++ b/var/spack/repos/builtin/packages/gdal/package.py @@ -0,0 +1,69 @@ +from spack import * + +class Gdal(Package): + """ + GDAL is a translator library for raster and vector geospatial + data formats that is released under an X/MIT style Open Source + license by the Open Source Geospatial Foundation. As a library, + it presents a single raster abstract data model and vector + abstract data model to the calling application for all supported + formats. It also comes with a variety of useful command line + utilities for data translation and processing + """ + + homepage = "http://www.gdal.org/" + url = "http://download.osgeo.org/gdal/2.0.2/gdal-2.0.2.tar.gz" + list_url = "http://download.osgeo.org/gdal/" + list_depth = 2 + + version('2.0.2', '573865f3f59ba7b4f8f4cddf223b52a5') + + extends('python') + + variant('hdf5', default=False, description='Enable HDF5 support') + variant('hdf', default=False, description='Enable HDF4 support') + variant('openjpeg', default=False, description='Enable JPEG2000 support') + variant('geos', default=False, description='Enable GEOS support') + variant('kea', default=False, description='Enable KEA support') + variant('netcdf', default=False, description='Enable netcdf support') + + depends_on('swig') + depends_on("hdf5", when='+hdf5') + depends_on("hdf", when='+hdf') + depends_on("openjpeg", when='+openjpeg') + depends_on("geos", when='+geos') + depends_on("kealib", when='+kea') + depends_on("netcdf", when='+netcdf') + depends_on("libtiff") + depends_on("libpng") + depends_on("zlib") + depends_on("proj") + depends_on("py-numpy") + + parallel = False + + def install(self, spec, prefix): + args = [] + args.append("--prefix=%s" % prefix) + args.append("--with-liblzma=yes") + args.append("--with-zlib=%s" % spec['zlib'].prefix) + args.append("--with-python=%s" % spec['python'].prefix.bin + "/python") + args.append("--without-libtool") + + if '+geos' in spec: + args.append('--with-geos=yes') + if '+hdf' in spec: + args.append('--with-hdf4=%s' % spec['hdf'].prefix) + if '+hdf5' in spec: + args.append('--with-hdf5=%s' % spec['hdf5'].prefix) + if '+openjpeg' in spec: + args.append('--with-openjpeg=%s' % spec['openjpeg'].prefix) + if '+kea' in spec: + args.append('--with-kea=yes') + if '+netcdf' in spec: + args.append('--with-netcdf=%s' % spec['netcdf'].prefix) + + configure(*args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/kealib/package.py b/var/spack/repos/builtin/packages/kealib/package.py new file mode 100644 index 00000000000..475d21e1d80 --- /dev/null +++ b/var/spack/repos/builtin/packages/kealib/package.py @@ -0,0 +1,35 @@ +from spack import * + +class Kealib(Package): + """An HDF5 Based Raster File Format + + KEALib provides an implementation of the GDAL data model. + The format supports raster attribute tables, image pyramids, + meta-data and in-built statistics while also handling very + large files and compression throughout. + + Based on the HDF5 standard, it also provides a base from which + other formats can be derived and is a good choice for long + term data archiving. An independent software library (libkea) + provides complete access to the KEA image format and a GDAL + driver allowing KEA images to be used from any GDAL supported software. + + Development work on this project has been funded by Landcare Research. + """ + homepage = "http://kealib.org/" + url = "https://bitbucket.org/chchrsc/kealib/get/kealib-1.4.5.tar.gz" + + version('1.4.5', '112e9c42d980b2d2987a3c15d0833a5d') + + depends_on("hdf5") + + def install(self, spec, prefix): + with working_dir('trunk', create=False): + cmake_args = [] + cmake_args.append("-DCMAKE_INSTALL_PREFIX=%s" % prefix) + cmake_args.append("-DHDF5_INCLUDE_DIR=%s" % spec['hdf5'].prefix.include) + cmake_args.append("-DHDF5_LIB_PATH=%s" % spec['hdf5'].prefix.lib) + cmake('.', *cmake_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/openjpeg/package.py b/var/spack/repos/builtin/packages/openjpeg/package.py new file mode 100644 index 00000000000..afec197d112 --- /dev/null +++ b/var/spack/repos/builtin/packages/openjpeg/package.py @@ -0,0 +1,26 @@ +from spack import * + +class Openjpeg(Package): + """ + OpenJPEG is an open-source JPEG 2000 codec written in C language. + It has been developed in order to promote the use of JPEG 2000, a + still-image compression standard from the Joint Photographic + Experts Group (JPEG). + Since April 2015, it is officially recognized by ISO/IEC and + ITU-T as a JPEG 2000 Reference Software. + """ + homepage = "https://github.com/uclouvain/openjpeg" + url = "https://github.com/uclouvain/openjpeg/archive/version.2.1.tar.gz" + + version('2.1' , '3e1c451c087f8462955426da38aa3b3d') + version('2.0.1', '105876ed43ff7dbb2f90b41b5a43cfa5') + version('2.0' , 'cdf266530fee8af87454f15feb619609') + version('1.5.2', '545f98923430369a6b046ef3632ef95c') + version('1.5.1', 'd774e4b5a0db5f0f171c4fc0aabfa14e') + + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/py-tuiview/package.py b/var/spack/repos/builtin/packages/py-tuiview/package.py new file mode 100644 index 00000000000..984b4196b19 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-tuiview/package.py @@ -0,0 +1,19 @@ +from spack import * + +class PyTuiview(Package): + """ + TuiView is a lightweight raster GIS with powerful raster attribute + table manipulation abilities. + """ + homepage = "https://bitbucket.org/chchrsc/tuiview" + url = "https://bitbucket.org/chchrsc/tuiview/get/tuiview-1.1.7.tar.gz" + + version('1.1.7', '4b3b38a820cc239c8ab4a181ac5d4c30') + + extends("python") + depends_on("py-pyqt") + depends_on("py-numpy") + depends_on("gdal") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From fd67da607e5476b767071dc37af66af6fd492452 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 25 Mar 2016 08:32:14 +0100 Subject: [PATCH 696/910] make netcdf link against Spack's curl --- var/spack/repos/builtin/packages/netcdf/package.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 227362399a9..b60a2c4e9a3 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -43,6 +43,13 @@ def install(self, spec, prefix): "--enable-dap" ] + # Make sure Netcdf links against Spack's curl + # Otherwise it may pick up system's curl, which could lead to link errors: + # /usr/lib/x86_64-linux-gnu/libcurl.so: undefined reference to `SSL_CTX_use_certificate_chain_file@OPENSSL_1.0.0' + LIBS.append("-lcurl") + CPPFLAGS.append("-I%s" % spec['curl'].prefix.include) + LDFLAGS.append ("-L%s" % spec['curl'].prefix.lib) + if '+mpi' in spec: config_args.append('--enable-parallel4') From 3e39daeb12aa13a3f3f975ff315143bc3f13a2b4 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 25 Mar 2016 09:56:24 +0100 Subject: [PATCH 697/910] add fPIC to MUMPS when building shared libs --- var/spack/repos/builtin/packages/mumps/package.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index 2801d4194dc..b295857ab59 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -70,6 +70,9 @@ def write_makefile_inc(self): makefile_conf.append("ORDERINGSF = %s" % (' '.join(orderings))) + # when building shared libs need -fPIC, otherwise + # /usr/bin/ld: graph.o: relocation R_X86_64_32 against `.rodata.str1.1' can not be used when making a shared object; recompile with -fPIC + fpic = '-fPIC' if '+shared' in self.spec else '' # TODO: test this part, it needs a full blas, scalapack and # partitionning environment with 64bit integers if '+idx64' in self.spec: @@ -77,14 +80,14 @@ def write_makefile_inc(self): # the fortran compilation flags most probably are # working only for intel and gnu compilers this is # perhaps something the compiler should provide - ['OPTF = -O -DALLOW_NON_INIT %s' % '-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8', - 'OPTL = -O ', - 'OPTC = -O -DINTSIZE64']) + ['OPTF = %s -O -DALLOW_NON_INIT %s' % (fpic,'-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8'), + 'OPTL = %s -O ' % fpic, + 'OPTC = %s -O -DINTSIZE64' % fpic]) else: makefile_conf.extend( - ['OPTF = -O -DALLOW_NON_INIT', - 'OPTL = -O ', - 'OPTC = -O ']) + ['OPTF = %s -O -DALLOW_NON_INIT' % fpic, + 'OPTL = %s -O ' % fpic, + 'OPTC = %s -O ' % fpic]) if '+mpi' in self.spec: From 83fea631f1765d4641cde8af2c5c931b22e4ee33 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 23:04:17 +0100 Subject: [PATCH 698/910] extend trilinos --- .../builtin/packages/trilinos/package.py | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index edc40476e30..22c7ab6d61c 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -1,5 +1,5 @@ from spack import * - +import os class Trilinos(Package): """ @@ -10,6 +10,7 @@ class Trilinos(Package): homepage = "https://trilinos.org/" url = "http://trilinos.csbsju.edu/download/files/trilinos-12.2.1-Source.tar.gz" + version('12.6.1', 'adcf2d3aab74cdda98f88fee19cd1442604199b0515ee3da4d80cbe8f37d00e4') version('12.4.2', '7c830f7f0f68b8ad324690603baf404e') version('12.2.1', '6161926ea247863c690e927687f83be9') version('12.0.1', 'bd99741d047471e127b8296b2ec08017') @@ -39,15 +40,38 @@ def install(self, spec, prefix): options.extend(std_cmake_args) options.extend(['-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', + '-DTrilinos_ENABLE_ALL_OPTIONAL_PACKAGES:BOOL=ON', '-DTrilinos_ENABLE_TESTS:BOOL=OFF', '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), - '-DTPL_ENABLE_MPI:STRING=ON', - '-DBLAS_LIBRARY_DIRS:PATH=%s' % spec['blas'].prefix, - '-DLAPACK_LIBRARY_DIRS:PATH=%s' % spec['lapack'].prefix + '-DTPL_ENABLE_MPI:BOOL=ON', + '-DMPI_BASE_DIR:PATH=%s' % spec['mpi'].prefix, + '-DTPL_ENABLE_BLAS=ON', + '-DBLAS_LIBRARY_NAMES=blas', + '-DBLAS_LIBRARY_DIRS=/usr/lib', # % spec['blas'].prefix, + '-DTPL_ENABLE_LAPACK=ON', + '-DLAPACK_LIBRARY_NAMES=lapack', + '-DLAPACK_LIBRARY_DIRS=/usr/lib', # % spec['lapack'].prefix, + '-DTPL_ENABLE_Boost:BOOL=ON', + '-DBOOST_BASE_DIR:PATH=%s' % spec['boost'].prefix, + '-DTrilinos_ENABLE_Fortran=OFF', + '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', + '-DTrilinos_ENABLE_CXX11:BOOL=ON', + '-DTrilinos_CXX11_FLAGS=-std=c++11' ]) + # disable due to compiler / config errors: + options.extend(['-DTrilinos_ENABLE_SEACAS=OFF', + '-DTrilinos_ENABLE_Pike=OFF', + '-DTrilinos_ENABLE_STK=OFF' + ]) + + if self.compiler.name == "clang": + os.environ['CPPFLAGS']="-Qunused-arguments" + + #os.environ['LDFLAGS']="lgfortran" + with working_dir('spack-build', create=True): cmake('..', *options) make() From 30ba96e58fe699e92902ce53a87b5e1cfb756d6f Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 23:56:18 +0100 Subject: [PATCH 699/910] same --- .../builtin/packages/trilinos/package.py | 84 ++++++++++++++++--- 1 file changed, 74 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 22c7ab6d61c..7b913031e3d 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -2,8 +2,7 @@ import os class Trilinos(Package): - """ - The Trilinos Project is an effort to develop algorithms and enabling technologies within an object-oriented + """The Trilinos Project is an effort to develop algorithms and enabling technologies within an object-oriented software framework for the solution of large-scale, complex multi-physics engineering and scientific problems. A unique design feature of Trilinos is its focus on packages. """ @@ -28,10 +27,19 @@ class Trilinos(Package): depends_on('matio') depends_on('glm') depends_on('swig') + depends_on('metis') + depends_on('suite-sparse') + depends_on('tbb') # MPI related dependencies depends_on('mpi') depends_on('netcdf+mpi') + depends_on('parmetis') + depends_on('mumps+metis+parmetis') + depends_on('scalapack') + depends_on('superlu-dist') + depends_on('hypre') + depends_on('hdf5+mpi') depends_on('python') # Needs py-numpy activated @@ -49,23 +57,79 @@ def install(self, spec, prefix): '-DMPI_BASE_DIR:PATH=%s' % spec['mpi'].prefix, '-DTPL_ENABLE_BLAS=ON', '-DBLAS_LIBRARY_NAMES=blas', - '-DBLAS_LIBRARY_DIRS=/usr/lib', # % spec['blas'].prefix, + '-DBLAS_LIBRARY_DIRS=/usr/lib', # % spec['blas'].prefix, #FIXME '-DTPL_ENABLE_LAPACK=ON', '-DLAPACK_LIBRARY_NAMES=lapack', - '-DLAPACK_LIBRARY_DIRS=/usr/lib', # % spec['lapack'].prefix, + '-DLAPACK_LIBRARY_DIRS=/usr/lib', # % spec['lapack'].prefix, #FIXME '-DTPL_ENABLE_Boost:BOOL=ON', '-DBOOST_BASE_DIR:PATH=%s' % spec['boost'].prefix, - '-DTrilinos_ENABLE_Fortran=OFF', + '-DTrilinos_ENABLE_Fortran=OFF', # FIXME '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', '-DTrilinos_ENABLE_CXX11:BOOL=ON', - '-DTrilinos_CXX11_FLAGS=-std=c++11' + '-DTrilinos_CXX11_FLAGS=-std=c++11', + '-DTPL_ENABLE_Netcdf:BOOL=ON', + '-DTPL_ENABLE_HYPRE:BOOL=ON', + '-DTPL_ENABLE_HDF5:BOOL=ON', + '-DTPL_ENABLE_TBB:BOOL=ON' ]) + # Amesos, conflicting types of double and complex SLU_D + # see https://trilinos.org/pipermail/trilinos-users/2015-March/004731.html + # and https://trilinos.org/pipermail/trilinos-users/2015-March/004802.html + options.extend([ + '-DTeuchos_ENABLE_COMPLEX:BOOL=OFF', + '-DKokkosTSQR_ENABLE_Complex:BOOL=OFF' + ]) + + # suite-sparse related + options.extend([ + '-DTPL_ENABLE_Cholmod:BOOL=ON', + '-DTPL_ENABLE_UMFPACK:BOOL=ON', + '-DUMFPACK_LIBRARY_NAMES=umfpack;amd;colamd;cholmod;suitesparseconfig' + ]) + + # metis / parmetis + options.extend([ + '-DTPL_ENABLE_METIS:BOOL=ON', + '-DMETIS_LIBRARY_DIRS=%s' % spec['metis'].prefix.lib, + '-DMETIS_LIBRARY_NAMES=metis', + '-DTPL_METIS_INCLUDE_DIRS=%s' % spec['metis'].prefix.include, + '-DTPL_ENABLE_ParMETIS:BOOL=ON', + '-DParMETIS_LIBRARY_DIRS=%s;%s' % (spec['parmetis'].prefix.lib,spec['metis'].prefix.lib), + '-DParMETIS_LIBRARY_NAMES=parmetis;metis', + '-DTPL_ParMETIS_INCLUDE_DIRS=%s' % spec['parmetis'].prefix.include + ]) + + # mumps + options.extend([ + '-DTPL_ENABLE_MUMPS:BOOL=ON"', + '-DMUMPS_LIBRARY_DIRS=%s' % spec['mumps'].prefix.lib, + '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord' # order is important! + ]) + + # scalapack + options.extend([ + '-DTPL_ENABLE_SCALAPACK:BOOL=ON', + '-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 + ]) + + # superlu_dist + options.extend([ + '-DTPL_ENABLE_SuperLUDist:BOOL=ON', + '-DSuperLUDist_INCLUDE_DIRS=%s' % spec['superlu_dist'].prefix.include + ]) + if spec.satisfies('^superlu-dist@4.0:'): + options.extend([ + '-DHAVE_SUPERLUDIST_LUSTRUCTINIT_2ARG:BOOL=ON' + ]) + + # disable due to compiler / config errors: - options.extend(['-DTrilinos_ENABLE_SEACAS=OFF', - '-DTrilinos_ENABLE_Pike=OFF', - '-DTrilinos_ENABLE_STK=OFF' - ]) + options.extend([ + '-DTrilinos_ENABLE_SEACAS=OFF', + '-DTrilinos_ENABLE_Pike=OFF', + '-DTrilinos_ENABLE_STK=OFF' + ]) if self.compiler.name == "clang": os.environ['CPPFLAGS']="-Qunused-arguments" From 76006fc346c7880b03356c1cf9d00bab2fdeac45 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 05:30:48 +0100 Subject: [PATCH 700/910] fix typo --- var/spack/repos/builtin/packages/trilinos/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 7b913031e3d..7d7613d2b53 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -116,7 +116,7 @@ def install(self, spec, prefix): # superlu_dist options.extend([ '-DTPL_ENABLE_SuperLUDist:BOOL=ON', - '-DSuperLUDist_INCLUDE_DIRS=%s' % spec['superlu_dist'].prefix.include + '-DSuperLUDist_INCLUDE_DIRS=%s' % spec['superlu-dist'].prefix.include ]) if spec.satisfies('^superlu-dist@4.0:'): options.extend([ From 18db930866b2024b242ea7c825851d7a56b27247 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 07:09:58 +0100 Subject: [PATCH 701/910] rearrange --- .../repos/builtin/packages/trilinos/package.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 7d7613d2b53..063d179c7eb 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -73,14 +73,6 @@ def install(self, spec, prefix): '-DTPL_ENABLE_TBB:BOOL=ON' ]) - # Amesos, conflicting types of double and complex SLU_D - # see https://trilinos.org/pipermail/trilinos-users/2015-March/004731.html - # and https://trilinos.org/pipermail/trilinos-users/2015-March/004802.html - options.extend([ - '-DTeuchos_ENABLE_COMPLEX:BOOL=OFF', - '-DKokkosTSQR_ENABLE_Complex:BOOL=OFF' - ]) - # suite-sparse related options.extend([ '-DTPL_ENABLE_Cholmod:BOOL=ON', @@ -113,7 +105,14 @@ def install(self, spec, prefix): '-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 ]) - # superlu_dist + # superlu_dist: + # Amesos, conflicting types of double and complex SLU_D + # see https://trilinos.org/pipermail/trilinos-users/2015-March/004731.html + # and https://trilinos.org/pipermail/trilinos-users/2015-March/004802.html + options.extend([ + '-DTeuchos_ENABLE_COMPLEX:BOOL=OFF', + '-DKokkosTSQR_ENABLE_Complex:BOOL=OFF' + ]) options.extend([ '-DTPL_ENABLE_SuperLUDist:BOOL=ON', '-DSuperLUDist_INCLUDE_DIRS=%s' % spec['superlu-dist'].prefix.include @@ -123,7 +122,6 @@ def install(self, spec, prefix): '-DHAVE_SUPERLUDIST_LUSTRUCTINIT_2ARG:BOOL=ON' ]) - # disable due to compiler / config errors: options.extend([ '-DTrilinos_ENABLE_SEACAS=OFF', From b91a155e8d996c9e51da7f75d4e106469ba488e6 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 07:10:31 +0100 Subject: [PATCH 702/910] more specific about dependencies --- var/spack/repos/builtin/packages/trilinos/package.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 063d179c7eb..2937ed4c381 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -62,7 +62,7 @@ def install(self, spec, prefix): '-DLAPACK_LIBRARY_NAMES=lapack', '-DLAPACK_LIBRARY_DIRS=/usr/lib', # % spec['lapack'].prefix, #FIXME '-DTPL_ENABLE_Boost:BOOL=ON', - '-DBOOST_BASE_DIR:PATH=%s' % spec['boost'].prefix, + '-DBoost_BASE_DIR:PATH=%s' % spec['boost'].prefix, '-DTrilinos_ENABLE_Fortran=OFF', # FIXME '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', '-DTrilinos_ENABLE_CXX11:BOOL=ON', @@ -76,7 +76,9 @@ def install(self, spec, prefix): # suite-sparse related options.extend([ '-DTPL_ENABLE_Cholmod:BOOL=ON', + '-DCholmod_BASE_DIR:PATH=%s' % spec['suite-sparse'].prefix, '-DTPL_ENABLE_UMFPACK:BOOL=ON', + '-D_UMFPACK_BASE_DIR:PATH=%s' % spec['suite-sparse'].prefix, '-DUMFPACK_LIBRARY_NAMES=umfpack;amd;colamd;cholmod;suitesparseconfig' ]) @@ -94,7 +96,7 @@ def install(self, spec, prefix): # mumps options.extend([ - '-DTPL_ENABLE_MUMPS:BOOL=ON"', + '-DTPL_ENABLE_MUMPS:BOOL=ON', '-DMUMPS_LIBRARY_DIRS=%s' % spec['mumps'].prefix.lib, '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord' # order is important! ]) @@ -115,6 +117,7 @@ def install(self, spec, prefix): ]) options.extend([ '-DTPL_ENABLE_SuperLUDist:BOOL=ON', + '-DSuperLUDist_BASE_DIR:PATH=%s' % spec['superlu-dist'].prefix, '-DSuperLUDist_INCLUDE_DIRS=%s' % spec['superlu-dist'].prefix.include ]) if spec.satisfies('^superlu-dist@4.0:'): From b6a35716ce3e02e1a3661e2433986ad637b65d0c Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 07:33:28 +0100 Subject: [PATCH 703/910] same --- var/spack/repos/builtin/packages/trilinos/package.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 2937ed4c381..b31cf880bca 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -43,6 +43,8 @@ class Trilinos(Package): depends_on('python') # Needs py-numpy activated + patch('umfpack_from_suitesparse.patch') + def install(self, spec, prefix): options = [] options.extend(std_cmake_args) @@ -76,9 +78,11 @@ def install(self, spec, prefix): # suite-sparse related options.extend([ '-DTPL_ENABLE_Cholmod:BOOL=ON', - '-DCholmod_BASE_DIR:PATH=%s' % spec['suite-sparse'].prefix, + '-DCholmod_LIBRARY_DIRS=%s' % spec['suite-sparse'].prefix.lib, + '-DCholmod_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, '-DTPL_ENABLE_UMFPACK:BOOL=ON', - '-D_UMFPACK_BASE_DIR:PATH=%s' % spec['suite-sparse'].prefix, + '-D_UMFPACK_LIBRARY_DIRS:PATH=%s' % spec['suite-sparse'].prefix.lib, + '-D_UMFPACK_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, '-DUMFPACK_LIBRARY_NAMES=umfpack;amd;colamd;cholmod;suitesparseconfig' ]) @@ -117,7 +121,7 @@ def install(self, spec, prefix): ]) options.extend([ '-DTPL_ENABLE_SuperLUDist:BOOL=ON', - '-DSuperLUDist_BASE_DIR:PATH=%s' % spec['superlu-dist'].prefix, + '-DSuperLUDist_LIBRARY_DIRS=%s' % spec['superlu-dist'].prefix.lib, '-DSuperLUDist_INCLUDE_DIRS=%s' % spec['superlu-dist'].prefix.include ]) if spec.satisfies('^superlu-dist@4.0:'): From 45b4895888a2abb3cf11ecf2b26b7940a9f89b5a Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 07:34:22 +0100 Subject: [PATCH 704/910] patch --- .../packages/trilinos/umfpack_from_suitesparse.patch | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 var/spack/repos/builtin/packages/trilinos/umfpack_from_suitesparse.patch diff --git a/var/spack/repos/builtin/packages/trilinos/umfpack_from_suitesparse.patch b/var/spack/repos/builtin/packages/trilinos/umfpack_from_suitesparse.patch new file mode 100644 index 00000000000..9defc555276 --- /dev/null +++ b/var/spack/repos/builtin/packages/trilinos/umfpack_from_suitesparse.patch @@ -0,0 +1,12 @@ +diff --git a/cmake/TPLs/FindTPLUMFPACK.cmake b/cmake/TPLs/FindTPLUMFPACK.cmake +index 963eb71..998cd02 100644 +--- a/cmake/TPLs/FindTPLUMFPACK.cmake ++++ b/cmake/TPLs/FindTPLUMFPACK.cmake +@@ -55,6 +55,6 @@ + + + TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( UMFPACK +- REQUIRED_HEADERS umfpack.h amd.h UFconfig.h ++ REQUIRED_HEADERS umfpack.h amd.h SuiteSparse_config.h + REQUIRED_LIBS_NAMES umfpack amd + ) From bc3314a57d43963416f709a94967ae8e1758c458 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 08:05:18 +0100 Subject: [PATCH 705/910] more and more --- .../repos/builtin/packages/trilinos/package.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index b31cf880bca..77ae6f818da 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -64,7 +64,8 @@ def install(self, spec, prefix): '-DLAPACK_LIBRARY_NAMES=lapack', '-DLAPACK_LIBRARY_DIRS=/usr/lib', # % spec['lapack'].prefix, #FIXME '-DTPL_ENABLE_Boost:BOOL=ON', - '-DBoost_BASE_DIR:PATH=%s' % spec['boost'].prefix, + '-DBoost_INCLUDE_DIRS:PATH=%s' % spec['boost'].prefix.include, + '-DBoost_LIBRARY_DIRS:PATH=%s' % spec['boost'].prefix.lib, '-DTrilinos_ENABLE_Fortran=OFF', # FIXME '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', '-DTrilinos_ENABLE_CXX11:BOOL=ON', @@ -77,12 +78,13 @@ def install(self, spec, prefix): # suite-sparse related options.extend([ - '-DTPL_ENABLE_Cholmod:BOOL=ON', - '-DCholmod_LIBRARY_DIRS=%s' % spec['suite-sparse'].prefix.lib, - '-DCholmod_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, + '-DTPL_ENABLE_Cholmod:BOOL=OFF', # FIXME: Trilinos seems to be looking for static libs only, patch CMake TPL file? + #'-DTPL_ENABLE_Cholmod:BOOL=ON', + #'-DCholmod_LIBRARY_DIRS:PATH=%s' % spec['suite-sparse'].prefix.lib, + #'-DCholmod_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, '-DTPL_ENABLE_UMFPACK:BOOL=ON', - '-D_UMFPACK_LIBRARY_DIRS:PATH=%s' % spec['suite-sparse'].prefix.lib, - '-D_UMFPACK_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, + '-DUMFPACK_LIBRARY_DIRS:PATH=%s' % spec['suite-sparse'].prefix.lib, + '-DUMFPACK_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, '-DUMFPACK_LIBRARY_NAMES=umfpack;amd;colamd;cholmod;suitesparseconfig' ]) From 2162627f35f98dc7f07cd34a6593f71d74095955 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 08:13:19 +0100 Subject: [PATCH 706/910] more --- var/spack/repos/builtin/packages/trilinos/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 77ae6f818da..532db69cc24 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -124,7 +124,7 @@ def install(self, spec, prefix): options.extend([ '-DTPL_ENABLE_SuperLUDist:BOOL=ON', '-DSuperLUDist_LIBRARY_DIRS=%s' % spec['superlu-dist'].prefix.lib, - '-DSuperLUDist_INCLUDE_DIRS=%s' % spec['superlu-dist'].prefix.include + '-DSuperLUDist_INCLUDE_DIRS=%s/superlu_dist' % spec['superlu-dist'].prefix.include # superlu_dist and superlu have the same header names :-( In order to avoid conflicts, try to keep "dist" version headers in a subfolder ]) if spec.satisfies('^superlu-dist@4.0:'): options.extend([ From 88a7a23132a379e526220324b8493d3be870f7e6 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 08:46:23 +0100 Subject: [PATCH 707/910] even more --- .../repos/builtin/packages/trilinos/package.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 532db69cc24..13794ae930a 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -36,7 +36,7 @@ class Trilinos(Package): depends_on('netcdf+mpi') depends_on('parmetis') depends_on('mumps+metis+parmetis') - depends_on('scalapack') + # depends_on('scalapack') # see FIXME below depends_on('superlu-dist') depends_on('hypre') depends_on('hdf5+mpi') @@ -49,6 +49,7 @@ def install(self, spec, prefix): options = [] options.extend(std_cmake_args) + mpi_bin = spec['mpi'].prefix.bin options.extend(['-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', '-DTrilinos_ENABLE_ALL_OPTIONAL_PACKAGES:BOOL=ON', '-DTrilinos_ENABLE_TESTS:BOOL=OFF', @@ -73,7 +74,12 @@ def install(self, spec, prefix): '-DTPL_ENABLE_Netcdf:BOOL=ON', '-DTPL_ENABLE_HYPRE:BOOL=ON', '-DTPL_ENABLE_HDF5:BOOL=ON', - '-DTPL_ENABLE_TBB:BOOL=ON' + '-DTPL_ENABLE_TBB:BOOL=ON', + # Need to use MPI wrappers, otherwise: Undefined symbols for architecture x86_64: "_mpi_abort_","_mpi_allgatherv_", etc from MUMPS + '-DCMAKE_C_COMPILER=%s' % join_path(mpi_bin,'mpicc'), # FIXME: dont hardcode compiler name + '-DCMAKE_CXX_COMPILER=%s' % join_path(mpi_bin,'mpicxx'), + '-DCMAKE_Fortran_COMPILER=%s' % join_path(mpi_bin,'mpicxx'), + '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-lgfortran' ]) # suite-sparse related @@ -109,8 +115,8 @@ def install(self, spec, prefix): # scalapack options.extend([ - '-DTPL_ENABLE_SCALAPACK:BOOL=ON', - '-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 + '-DTPL_ENABLE_SCALAPACK:BOOL=OFF', #FIXME: Undefined symbols for architecture x86_64: "_blacs_gridinfo__", referenced from: Amesos_Scalapack::RedistributeA() in Amesos_Scalapack.cpp.o + #'-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 ]) # superlu_dist: @@ -141,8 +147,6 @@ def install(self, spec, prefix): if self.compiler.name == "clang": os.environ['CPPFLAGS']="-Qunused-arguments" - #os.environ['LDFLAGS']="lgfortran" - with working_dir('spack-build', create=True): cmake('..', *options) make() From 7e4e2c988ab4e5163469395ace204d9f332a9675 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 08:48:25 +0100 Subject: [PATCH 708/910] minor --- var/spack/repos/builtin/packages/trilinos/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 13794ae930a..c915d8eb1a6 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -78,7 +78,7 @@ def install(self, spec, prefix): # Need to use MPI wrappers, otherwise: Undefined symbols for architecture x86_64: "_mpi_abort_","_mpi_allgatherv_", etc from MUMPS '-DCMAKE_C_COMPILER=%s' % join_path(mpi_bin,'mpicc'), # FIXME: dont hardcode compiler name '-DCMAKE_CXX_COMPILER=%s' % join_path(mpi_bin,'mpicxx'), - '-DCMAKE_Fortran_COMPILER=%s' % join_path(mpi_bin,'mpicxx'), + '-DCMAKE_Fortran_COMPILER=%s' % join_path(mpi_bin,'mpif90'), '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-lgfortran' ]) From 50e4b609c688b573ef882a079e2d45a83966f94c Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 09:11:26 +0100 Subject: [PATCH 709/910] minor --- var/spack/repos/builtin/packages/trilinos/package.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index c915d8eb1a6..3622548ecea 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -52,6 +52,7 @@ def install(self, spec, prefix): mpi_bin = spec['mpi'].prefix.bin options.extend(['-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', '-DTrilinos_ENABLE_ALL_OPTIONAL_PACKAGES:BOOL=ON', + '-DTrilinos_VERBOSE_CONFIGURE:BOOL=OFF', '-DTrilinos_ENABLE_TESTS:BOOL=OFF', '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), @@ -145,7 +146,9 @@ def install(self, spec, prefix): ]) if self.compiler.name == "clang": - os.environ['CPPFLAGS']="-Qunused-arguments" + options.extend([ + '-DCMAKE_EXE_LINKER_FLAGS:STRING=-Qunused-arguments' + ]) with working_dir('spack-build', create=True): cmake('..', *options) From 2fb599ea86c82e3395d8402db116a3a25fed0c73 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 09:16:51 +0100 Subject: [PATCH 710/910] minor --- var/spack/repos/builtin/packages/trilinos/package.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 3622548ecea..82681e004c0 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -55,7 +55,7 @@ def install(self, spec, prefix): '-DTrilinos_VERBOSE_CONFIGURE:BOOL=OFF', '-DTrilinos_ENABLE_TESTS:BOOL=OFF', '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', - '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), + '-DCMAKE_BUILD_TYPE:STRING=%s' % ('DEBUG' if '+debug' in spec else 'RELEASE'), '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), '-DTPL_ENABLE_MPI:BOOL=ON', '-DMPI_BASE_DIR:PATH=%s' % spec['mpi'].prefix, @@ -83,6 +83,11 @@ def install(self, spec, prefix): '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-lgfortran' ]) + # for build-debug only: + options.extend([ + '-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE' + ]) + # suite-sparse related options.extend([ '-DTPL_ENABLE_Cholmod:BOOL=OFF', # FIXME: Trilinos seems to be looking for static libs only, patch CMake TPL file? From eb5b0767aa20b46932c36203595c124b7463a444 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 09:45:33 +0100 Subject: [PATCH 711/910] remove gfortran as we have libnotfound... --- var/spack/repos/builtin/packages/trilinos/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 82681e004c0..bb6f0951909 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -80,7 +80,7 @@ def install(self, spec, prefix): '-DCMAKE_C_COMPILER=%s' % join_path(mpi_bin,'mpicc'), # FIXME: dont hardcode compiler name '-DCMAKE_CXX_COMPILER=%s' % join_path(mpi_bin,'mpicxx'), '-DCMAKE_Fortran_COMPILER=%s' % join_path(mpi_bin,'mpif90'), - '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-lgfortran' + # '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-lgfortran' # FIXME ]) # for build-debug only: From 7897f10126f0f8389326c9e723d73021f654ea6a Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 13:04:43 +0100 Subject: [PATCH 712/910] fiddle with fortran --- .../builtin/packages/trilinos/package.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index bb6f0951909..f6f24e967e9 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -68,7 +68,7 @@ def install(self, spec, prefix): '-DTPL_ENABLE_Boost:BOOL=ON', '-DBoost_INCLUDE_DIRS:PATH=%s' % spec['boost'].prefix.include, '-DBoost_LIBRARY_DIRS:PATH=%s' % spec['boost'].prefix.lib, - '-DTrilinos_ENABLE_Fortran=OFF', # FIXME + # '-DTrilinos_ENABLE_Fortran=OFF', # FIXME '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', '-DTrilinos_ENABLE_CXX11:BOOL=ON', '-DTrilinos_CXX11_FLAGS=-std=c++11', @@ -79,10 +79,17 @@ def install(self, spec, prefix): # Need to use MPI wrappers, otherwise: Undefined symbols for architecture x86_64: "_mpi_abort_","_mpi_allgatherv_", etc from MUMPS '-DCMAKE_C_COMPILER=%s' % join_path(mpi_bin,'mpicc'), # FIXME: dont hardcode compiler name '-DCMAKE_CXX_COMPILER=%s' % join_path(mpi_bin,'mpicxx'), - '-DCMAKE_Fortran_COMPILER=%s' % join_path(mpi_bin,'mpif90'), - # '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-lgfortran' # FIXME + '-DCMAKE_Fortran_COMPILER=%s' % join_path(mpi_bin,'mpif90') ]) + # Fortran lib + libgfortran = os.path.dirname (os.popen('%s --print-file-name libgfortran.a' % join_path(mpi_bin,'mpif90') ).read()) + #os.environ['LDFLAGS'] = '-L%s -lgfortran' % libgfortran + options.extend([ + '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-lgfortran', + '-DCMAKE_EXE_LINKER_FLAGS:STRING=-L%s -lgfortran' % libgfortran + ]) + # for build-debug only: options.extend([ '-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE' @@ -150,10 +157,10 @@ def install(self, spec, prefix): '-DTrilinos_ENABLE_STK=OFF' ]) - if self.compiler.name == "clang": - options.extend([ - '-DCMAKE_EXE_LINKER_FLAGS:STRING=-Qunused-arguments' - ]) + #if self.compiler.name == "clang": + # options.extend([ + # '-DCMAKE_EXE_LINKER_FLAGS:STRING=-Qunused-arguments' + # ]) with working_dir('spack-build', create=True): cmake('..', *options) From fade526ae4e748a9454fd56c07ef7bc1fca38423 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 16:58:59 +0100 Subject: [PATCH 713/910] remove tbb --- var/spack/repos/builtin/packages/trilinos/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index f6f24e967e9..12212360a0d 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -29,7 +29,6 @@ class Trilinos(Package): depends_on('swig') depends_on('metis') depends_on('suite-sparse') - depends_on('tbb') # MPI related dependencies depends_on('mpi') @@ -75,7 +74,6 @@ def install(self, spec, prefix): '-DTPL_ENABLE_Netcdf:BOOL=ON', '-DTPL_ENABLE_HYPRE:BOOL=ON', '-DTPL_ENABLE_HDF5:BOOL=ON', - '-DTPL_ENABLE_TBB:BOOL=ON', # Need to use MPI wrappers, otherwise: Undefined symbols for architecture x86_64: "_mpi_abort_","_mpi_allgatherv_", etc from MUMPS '-DCMAKE_C_COMPILER=%s' % join_path(mpi_bin,'mpicc'), # FIXME: dont hardcode compiler name '-DCMAKE_CXX_COMPILER=%s' % join_path(mpi_bin,'mpicxx'), From e51fe2934e0189aa5c33c8663819d23e7706f809 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 17:38:36 +0100 Subject: [PATCH 714/910] fiddle more --- var/spack/repos/builtin/packages/trilinos/package.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 12212360a0d..fb14e1f36cb 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -67,7 +67,6 @@ def install(self, spec, prefix): '-DTPL_ENABLE_Boost:BOOL=ON', '-DBoost_INCLUDE_DIRS:PATH=%s' % spec['boost'].prefix.include, '-DBoost_LIBRARY_DIRS:PATH=%s' % spec['boost'].prefix.lib, - # '-DTrilinos_ENABLE_Fortran=OFF', # FIXME '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', '-DTrilinos_ENABLE_CXX11:BOOL=ON', '-DTrilinos_CXX11_FLAGS=-std=c++11', @@ -82,10 +81,9 @@ def install(self, spec, prefix): # Fortran lib libgfortran = os.path.dirname (os.popen('%s --print-file-name libgfortran.a' % join_path(mpi_bin,'mpif90') ).read()) - #os.environ['LDFLAGS'] = '-L%s -lgfortran' % libgfortran options.extend([ - '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-lgfortran', - '-DCMAKE_EXE_LINKER_FLAGS:STRING=-L%s -lgfortran' % libgfortran + '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-L%s/ -lgfortran' % libgfortran, + '-DTrilinos_ENABLE_Fortran=OFF' # FIXME: otherwise VerifyFortranC fails as it does not contain -lgfortran , issues with IMPLICIT_LINK_LIBRARIES in CMakeFiles/CMake(C|CXX|Fortran)Compiler.cmake? ]) # for build-debug only: @@ -155,11 +153,6 @@ def install(self, spec, prefix): '-DTrilinos_ENABLE_STK=OFF' ]) - #if self.compiler.name == "clang": - # options.extend([ - # '-DCMAKE_EXE_LINKER_FLAGS:STRING=-Qunused-arguments' - # ]) - with working_dir('spack-build', create=True): cmake('..', *options) make() From 3f12a51759fcdd3b11867cb849fc2aa81a4941ef Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 23 Mar 2016 19:35:49 +0100 Subject: [PATCH 715/910] keep fiddling --- .../repos/builtin/packages/trilinos/package.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index fb14e1f36cb..cdc2bf2379f 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -1,6 +1,11 @@ from spack import * import os +# Trilinos is complicated to build, as an inspiration a couple of links to other repositories which build it: +# https://github.com/hpcugent/easybuild-easyblocks/blob/master/easybuild/easyblocks/t/trilinos.py#L111 +# https://github.com/koecher/candi/blob/master/deal.II-toolchain/packages/trilinos.package +# https://gitlab.com/configurations/cluster-config/blob/master/trilinos.sh +# https://github.com/Homebrew/homebrew-science/blob/master/trilinos.rb class Trilinos(Package): """The Trilinos Project is an effort to develop algorithms and enabling technologies within an object-oriented software framework for the solution of large-scale, complex multi-physics engineering and scientific problems. @@ -59,11 +64,11 @@ def install(self, spec, prefix): '-DTPL_ENABLE_MPI:BOOL=ON', '-DMPI_BASE_DIR:PATH=%s' % spec['mpi'].prefix, '-DTPL_ENABLE_BLAS=ON', - '-DBLAS_LIBRARY_NAMES=blas', - '-DBLAS_LIBRARY_DIRS=/usr/lib', # % spec['blas'].prefix, #FIXME + '-DBLAS_LIBRARY_NAMES=blas', # FIXME: for Intel, Clang+GNU, GNU; easybuild add gfortran here... + '-DBLAS_LIBRARY_DIRS=%s' % spec['blas'].prefix.lib, '-DTPL_ENABLE_LAPACK=ON', '-DLAPACK_LIBRARY_NAMES=lapack', - '-DLAPACK_LIBRARY_DIRS=/usr/lib', # % spec['lapack'].prefix, #FIXME + '-DLAPACK_LIBRARY_DIRS=%s' % spec['lapack'].prefix, '-DTPL_ENABLE_Boost:BOOL=ON', '-DBoost_INCLUDE_DIRS:PATH=%s' % spec['boost'].prefix.include, '-DBoost_LIBRARY_DIRS:PATH=%s' % spec['boost'].prefix.lib, @@ -74,9 +79,9 @@ def install(self, spec, prefix): '-DTPL_ENABLE_HYPRE:BOOL=ON', '-DTPL_ENABLE_HDF5:BOOL=ON', # Need to use MPI wrappers, otherwise: Undefined symbols for architecture x86_64: "_mpi_abort_","_mpi_allgatherv_", etc from MUMPS - '-DCMAKE_C_COMPILER=%s' % join_path(mpi_bin,'mpicc'), # FIXME: dont hardcode compiler name - '-DCMAKE_CXX_COMPILER=%s' % join_path(mpi_bin,'mpicxx'), - '-DCMAKE_Fortran_COMPILER=%s' % join_path(mpi_bin,'mpif90') + #'-DCMAKE_C_COMPILER=%s' % join_path(mpi_bin,'mpicc'), # FIXME: dont hardcode compiler name + #'-DCMAKE_CXX_COMPILER=%s' % join_path(mpi_bin,'mpicxx'), + #'-DCMAKE_Fortran_COMPILER=%s' % join_path(mpi_bin,'mpif90') ]) # Fortran lib From 8c9f88761384d16c77aa9cbee55cc501712c7b12 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 24 Mar 2016 12:45:43 +0100 Subject: [PATCH 716/910] finally builds --- var/spack/repos/builtin/packages/trilinos/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index cdc2bf2379f..b80123838ac 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -39,7 +39,7 @@ class Trilinos(Package): depends_on('mpi') depends_on('netcdf+mpi') depends_on('parmetis') - depends_on('mumps+metis+parmetis') + depends_on('mumps+metis+parmetis+shared') # build errors with static libs # depends_on('scalapack') # see FIXME below depends_on('superlu-dist') depends_on('hypre') From df995c4ef850338823b3a5a5c2ee3b12e05a33b1 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 24 Mar 2016 12:57:22 +0100 Subject: [PATCH 717/910] minor cleanup --- .../repos/builtin/packages/trilinos/package.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index b80123838ac..f2e9c37caac 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -77,24 +77,20 @@ def install(self, spec, prefix): '-DTrilinos_CXX11_FLAGS=-std=c++11', '-DTPL_ENABLE_Netcdf:BOOL=ON', '-DTPL_ENABLE_HYPRE:BOOL=ON', - '-DTPL_ENABLE_HDF5:BOOL=ON', - # Need to use MPI wrappers, otherwise: Undefined symbols for architecture x86_64: "_mpi_abort_","_mpi_allgatherv_", etc from MUMPS - #'-DCMAKE_C_COMPILER=%s' % join_path(mpi_bin,'mpicc'), # FIXME: dont hardcode compiler name - #'-DCMAKE_CXX_COMPILER=%s' % join_path(mpi_bin,'mpicxx'), - #'-DCMAKE_Fortran_COMPILER=%s' % join_path(mpi_bin,'mpif90') + '-DTPL_ENABLE_HDF5:BOOL=ON' ]) # Fortran lib libgfortran = os.path.dirname (os.popen('%s --print-file-name libgfortran.a' % join_path(mpi_bin,'mpif90') ).read()) options.extend([ '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-L%s/ -lgfortran' % libgfortran, - '-DTrilinos_ENABLE_Fortran=OFF' # FIXME: otherwise VerifyFortranC fails as it does not contain -lgfortran , issues with IMPLICIT_LINK_LIBRARIES in CMakeFiles/CMake(C|CXX|Fortran)Compiler.cmake? + '-DTrilinos_ENABLE_Fortran=OFF' # FIXME: otherwise CMake's VerifyFortranC fails as it does not contain -lgfortran ]) # for build-debug only: - options.extend([ - '-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE' - ]) + # options.extend([ + # '-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE' + # ]) # suite-sparse related options.extend([ From 0fc9326a4cc742e7a06cc1bb4192d0dfd008880d Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 24 Mar 2016 13:23:56 +0100 Subject: [PATCH 718/910] add variants --- .../builtin/packages/trilinos/package.py | 111 +++++++++++------- 1 file changed, 69 insertions(+), 42 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index f2e9c37caac..2e119581e11 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -22,8 +22,15 @@ class Trilinos(Package): version('11.14.2', 'a43590cf896c677890d75bfe75bc6254') version('11.14.1', '40febc57f76668be8b6a77b7607bb67f') - variant('shared', default=True, description='Enables the build of shared libraries') - variant('debug', default=False, description='Builds a debug version of the libraries') + variant('metis', default=True, description='Compile with METIS') + variant('parmetis', default=True, description='Compile with ParMETIS') + variant('mumps', default=True, description='Compile with support for MUMPS solvers') + variant('superlu-dist', default=True, description='Compile with SuperluDist solvers') + variant('hypre', default=True, description='Compile with Hypre preconditioner') + variant('hdf5', default=True, description='Compile with HDF5') + variant('suite-sparse', default=True, description='Compile with SuiteSparse solvers') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Builds a debug version of the libraries') # Everything should be compiled with -fpic depends_on('blas') @@ -49,7 +56,23 @@ class Trilinos(Package): patch('umfpack_from_suitesparse.patch') + # check that the combination of variants makes sense + def variants_check(self): + if '+parmetis' in self.spec and '+metis' not in self.spec: + raise RuntimeError('You cannot use the variant parmetis without metis') + + if '+mumps' in self.spec and '+parmetis' not in self.spec: + raise RuntimeError('You cannot use the variant mumps without parmetis') + + if '+superlu-dist' in self.spec and self.spec.satisfies('@:11.4.3'): + # For Trilinos v11 we need to force SuperLUDist=OFF, + # since only the deprecated SuperLUDist v3.3 together with an Amesos patch + # is working. + raise RuntimeError('The superlu-dist variant can only be used with Trilinos @12.0.1:') + def install(self, spec, prefix): + self.variants_check() + options = [] options.extend(std_cmake_args) @@ -93,35 +116,38 @@ def install(self, spec, prefix): # ]) # suite-sparse related - options.extend([ - '-DTPL_ENABLE_Cholmod:BOOL=OFF', # FIXME: Trilinos seems to be looking for static libs only, patch CMake TPL file? - #'-DTPL_ENABLE_Cholmod:BOOL=ON', - #'-DCholmod_LIBRARY_DIRS:PATH=%s' % spec['suite-sparse'].prefix.lib, - #'-DCholmod_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, - '-DTPL_ENABLE_UMFPACK:BOOL=ON', - '-DUMFPACK_LIBRARY_DIRS:PATH=%s' % spec['suite-sparse'].prefix.lib, - '-DUMFPACK_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, - '-DUMFPACK_LIBRARY_NAMES=umfpack;amd;colamd;cholmod;suitesparseconfig' - ]) + if '+suite-sparse' in spec: + options.extend([ + '-DTPL_ENABLE_Cholmod:BOOL=OFF', # FIXME: Trilinos seems to be looking for static libs only, patch CMake TPL file? + #'-DTPL_ENABLE_Cholmod:BOOL=ON', + #'-DCholmod_LIBRARY_DIRS:PATH=%s' % spec['suite-sparse'].prefix.lib, + #'-DCholmod_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, + '-DTPL_ENABLE_UMFPACK:BOOL=ON', + '-DUMFPACK_LIBRARY_DIRS:PATH=%s' % spec['suite-sparse'].prefix.lib, + '-DUMFPACK_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, + '-DUMFPACK_LIBRARY_NAMES=umfpack;amd;colamd;cholmod;suitesparseconfig' + ]) # metis / parmetis - options.extend([ - '-DTPL_ENABLE_METIS:BOOL=ON', - '-DMETIS_LIBRARY_DIRS=%s' % spec['metis'].prefix.lib, - '-DMETIS_LIBRARY_NAMES=metis', - '-DTPL_METIS_INCLUDE_DIRS=%s' % spec['metis'].prefix.include, - '-DTPL_ENABLE_ParMETIS:BOOL=ON', - '-DParMETIS_LIBRARY_DIRS=%s;%s' % (spec['parmetis'].prefix.lib,spec['metis'].prefix.lib), - '-DParMETIS_LIBRARY_NAMES=parmetis;metis', - '-DTPL_ParMETIS_INCLUDE_DIRS=%s' % spec['parmetis'].prefix.include - ]) + if '+parmetis' in spec: # metis is required, see variants_check() + options.extend([ + '-DTPL_ENABLE_METIS:BOOL=ON', + '-DMETIS_LIBRARY_DIRS=%s' % spec['metis'].prefix.lib, + '-DMETIS_LIBRARY_NAMES=metis', + '-DTPL_METIS_INCLUDE_DIRS=%s' % spec['metis'].prefix.include, + '-DTPL_ENABLE_ParMETIS:BOOL=ON', + '-DParMETIS_LIBRARY_DIRS=%s;%s' % (spec['parmetis'].prefix.lib,spec['metis'].prefix.lib), + '-DParMETIS_LIBRARY_NAMES=parmetis;metis', + '-DTPL_ParMETIS_INCLUDE_DIRS=%s' % spec['parmetis'].prefix.include + ]) # mumps - options.extend([ - '-DTPL_ENABLE_MUMPS:BOOL=ON', - '-DMUMPS_LIBRARY_DIRS=%s' % spec['mumps'].prefix.lib, - '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord' # order is important! - ]) + if '+mumps' in spec: + options.extend([ + '-DTPL_ENABLE_MUMPS:BOOL=ON', + '-DMUMPS_LIBRARY_DIRS=%s' % spec['mumps'].prefix.lib, + '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord' # order is important! + ]) # scalapack options.extend([ @@ -129,23 +155,24 @@ def install(self, spec, prefix): #'-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 ]) - # superlu_dist: - # Amesos, conflicting types of double and complex SLU_D - # see https://trilinos.org/pipermail/trilinos-users/2015-March/004731.html - # and https://trilinos.org/pipermail/trilinos-users/2015-March/004802.html - options.extend([ - '-DTeuchos_ENABLE_COMPLEX:BOOL=OFF', - '-DKokkosTSQR_ENABLE_Complex:BOOL=OFF' - ]) - options.extend([ - '-DTPL_ENABLE_SuperLUDist:BOOL=ON', - '-DSuperLUDist_LIBRARY_DIRS=%s' % spec['superlu-dist'].prefix.lib, - '-DSuperLUDist_INCLUDE_DIRS=%s/superlu_dist' % spec['superlu-dist'].prefix.include # superlu_dist and superlu have the same header names :-( In order to avoid conflicts, try to keep "dist" version headers in a subfolder - ]) - if spec.satisfies('^superlu-dist@4.0:'): + # superlu-dist: + if '+superlu-dist' in spec: + # Amesos, conflicting types of double and complex SLU_D + # see https://trilinos.org/pipermail/trilinos-users/2015-March/004731.html + # and https://trilinos.org/pipermail/trilinos-users/2015-March/004802.html options.extend([ - '-DHAVE_SUPERLUDIST_LUSTRUCTINIT_2ARG:BOOL=ON' + '-DTeuchos_ENABLE_COMPLEX:BOOL=OFF', + '-DKokkosTSQR_ENABLE_Complex:BOOL=OFF' ]) + options.extend([ + '-DTPL_ENABLE_SuperLUDist:BOOL=ON', + '-DSuperLUDist_LIBRARY_DIRS=%s' % spec['superlu-dist'].prefix.lib, + '-DSuperLUDist_INCLUDE_DIRS=%s/superlu_dist' % spec['superlu-dist'].prefix.include # superlu_dist and superlu have the same header names :-( In order to avoid conflicts, try to keep "dist" version headers in a subfolder + ]) + if spec.satisfies('^superlu-dist@4.0:'): + options.extend([ + '-DHAVE_SUPERLUDIST_LUSTRUCTINIT_2ARG:BOOL=ON' + ]) # disable due to compiler / config errors: options.extend([ From 416fcd081013afa8349d311909d50b21d9685674 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 24 Mar 2016 13:33:00 +0100 Subject: [PATCH 719/910] same --- .../repos/builtin/packages/trilinos/package.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 2e119581e11..3beadeb7980 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -29,6 +29,7 @@ class Trilinos(Package): variant('hypre', default=True, description='Compile with Hypre preconditioner') variant('hdf5', default=True, description='Compile with HDF5') variant('suite-sparse', default=True, description='Compile with SuiteSparse solvers') + variant('python', default=True, description='Build python wrappers') variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds a debug version of the libraries') @@ -39,20 +40,20 @@ class Trilinos(Package): depends_on('matio') depends_on('glm') depends_on('swig') - depends_on('metis') - depends_on('suite-sparse') + depends_on('metis',when='+metis') + depends_on('suite-sparse',when='+suite-sparse') # MPI related dependencies depends_on('mpi') depends_on('netcdf+mpi') - depends_on('parmetis') - depends_on('mumps+metis+parmetis+shared') # build errors with static libs + depends_on('parmetis',when='+parmetis') + depends_on('mumps+metis+parmetis+shared',when='+mumps') # build errors with static libs # depends_on('scalapack') # see FIXME below - depends_on('superlu-dist') - depends_on('hypre') - depends_on('hdf5+mpi') + depends_on('superlu-dist',when='+superlu-dist') + depends_on('hypre',when='+hypre') + depends_on('hdf5+mpi',when='+hdf5') - depends_on('python') # Needs py-numpy activated + depends_on('python',when='+python') # Needs py-numpy activated patch('umfpack_from_suitesparse.patch') From c33a63cabc8dcd8c71fc87de60a883cef4a4dae2 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 24 Mar 2016 13:47:07 +0100 Subject: [PATCH 720/910] fix python installation --- var/spack/repos/builtin/packages/trilinos/package.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 3beadeb7980..886893b639c 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -186,3 +186,14 @@ def install(self, spec, prefix): cmake('..', *options) make() make('install') + + # When trilinos is built with Python, libpytrilinos is included through + # cmake configure files. Namely, Trilinos_LIBRARIES in TrilinosConfig.cmake + # contains pytrilinos. This leads to a run-time error: + # Symbol not found: _PyBool_Type + # and prevents Trilinos to be used in any C++ code, which links executable + # against the libraries listed in Trilinos_LIBRARIES. + # See https://github.com/Homebrew/homebrew-science/issues/2148#issuecomment-103614509 + # A workaround it to remove PyTrilinos from the COMPONENTS_LIST : + if '+python' in self.spec: + filter_file(r'(?:SET\(COMPONENTS_LIST.*)(PyTrilinos;)(?:.*)', '', '%s/cmake/Trilinos/TrilinosConfig.cmake' % prefix.lib) From d096b155f45d97775affa2bdda640a22823cb73e Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 24 Mar 2016 19:41:37 +0100 Subject: [PATCH 721/910] fix filter_file --- var/spack/repos/builtin/packages/trilinos/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 886893b639c..3967d0ef0ed 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -196,4 +196,4 @@ def install(self, spec, prefix): # See https://github.com/Homebrew/homebrew-science/issues/2148#issuecomment-103614509 # A workaround it to remove PyTrilinos from the COMPONENTS_LIST : if '+python' in self.spec: - filter_file(r'(?:SET\(COMPONENTS_LIST.*)(PyTrilinos;)(?:.*)', '', '%s/cmake/Trilinos/TrilinosConfig.cmake' % prefix.lib) + filter_file(r'(SET\(COMPONENTS_LIST.*)(PyTrilinos;)(.*)', (r'\1\3'), '%s/cmake/Trilinos/TrilinosConfig.cmake' % prefix.lib) From 7f2db8c26793abb1977dbccd2ae8a3a1b5a46568 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 25 Mar 2016 11:54:51 +0100 Subject: [PATCH 722/910] fix w GCC 4.8 on Ubuntu but broke w GCC 5.3/Clang on OSX --- .../builtin/packages/trilinos/package.py | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 3967d0ef0ed..1f1601e5376 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -48,7 +48,7 @@ class Trilinos(Package): depends_on('netcdf+mpi') depends_on('parmetis',when='+parmetis') depends_on('mumps+metis+parmetis+shared',when='+mumps') # build errors with static libs - # depends_on('scalapack') # see FIXME below + depends_on('scalapack') depends_on('superlu-dist',when='+superlu-dist') depends_on('hypre',when='+hypre') depends_on('hdf5+mpi',when='+hdf5') @@ -108,13 +108,13 @@ def install(self, spec, prefix): libgfortran = os.path.dirname (os.popen('%s --print-file-name libgfortran.a' % join_path(mpi_bin,'mpif90') ).read()) options.extend([ '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-L%s/ -lgfortran' % libgfortran, - '-DTrilinos_ENABLE_Fortran=OFF' # FIXME: otherwise CMake's VerifyFortranC fails as it does not contain -lgfortran + '-DTrilinos_ENABLE_Fortran=ON' # FIXME: otherwise CMake's VerifyFortranC fails as it does not contain -lgfortran ]) # for build-debug only: - # options.extend([ - # '-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE' - # ]) + options.extend([ + '-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE' + ]) # suite-sparse related if '+suite-sparse' in spec: @@ -144,16 +144,22 @@ def install(self, spec, prefix): # mumps if '+mumps' in spec: + # FIXME: + # since we use mumps with MPI, it will certainly be build against Scalapack. + # Add scalapack lib here as well. + # This likely won't be need if Trilinos would compile with Scalapack options.extend([ '-DTPL_ENABLE_MUMPS:BOOL=ON', '-DMUMPS_LIBRARY_DIRS=%s' % spec['mumps'].prefix.lib, '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord' # order is important! +# '-DMUMPS_LIBRARY_DIRS=%s;%s' % (spec['mumps'].prefix.lib,spec['scalapack'].prefix.lib), +# '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord;scalapack' # order is important! ]) # scalapack options.extend([ - '-DTPL_ENABLE_SCALAPACK:BOOL=OFF', #FIXME: Undefined symbols for architecture x86_64: "_blacs_gridinfo__", referenced from: Amesos_Scalapack::RedistributeA() in Amesos_Scalapack.cpp.o - #'-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 + '-DTPL_ENABLE_SCALAPACK:BOOL=ON', #FIXME: Undefined symbols for architecture x86_64: "_blacs_gridinfo__", referenced from: Amesos_Scalapack::RedistributeA() in Amesos_Scalapack.cpp.o + '-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 ]) # superlu-dist: @@ -175,6 +181,12 @@ def install(self, spec, prefix): '-DHAVE_SUPERLUDIST_LUSTRUCTINIT_2ARG:BOOL=ON' ]) + # python + if '~python' in spec: + options.extend([ + '-DTrilinos_ENABLE_PyTrilinos:BOOL=OFF' + ]) + # disable due to compiler / config errors: options.extend([ '-DTrilinos_ENABLE_SEACAS=OFF', From 828aeefb1d4e09deb0694c61397a3097bda85ccb Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 25 Mar 2016 12:01:00 +0100 Subject: [PATCH 723/910] minor cleanup --- var/spack/repos/builtin/packages/trilinos/package.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 1f1601e5376..1dba2b8db39 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -88,7 +88,7 @@ def install(self, spec, prefix): '-DTPL_ENABLE_MPI:BOOL=ON', '-DMPI_BASE_DIR:PATH=%s' % spec['mpi'].prefix, '-DTPL_ENABLE_BLAS=ON', - '-DBLAS_LIBRARY_NAMES=blas', # FIXME: for Intel, Clang+GNU, GNU; easybuild add gfortran here... + '-DBLAS_LIBRARY_NAMES=blas', # FIXME: don't hardcode names '-DBLAS_LIBRARY_DIRS=%s' % spec['blas'].prefix.lib, '-DTPL_ENABLE_LAPACK=ON', '-DLAPACK_LIBRARY_NAMES=lapack', @@ -108,7 +108,7 @@ def install(self, spec, prefix): libgfortran = os.path.dirname (os.popen('%s --print-file-name libgfortran.a' % join_path(mpi_bin,'mpif90') ).read()) options.extend([ '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-L%s/ -lgfortran' % libgfortran, - '-DTrilinos_ENABLE_Fortran=ON' # FIXME: otherwise CMake's VerifyFortranC fails as it does not contain -lgfortran + '-DTrilinos_ENABLE_Fortran=ON' ]) # for build-debug only: @@ -144,21 +144,15 @@ def install(self, spec, prefix): # mumps if '+mumps' in spec: - # FIXME: - # since we use mumps with MPI, it will certainly be build against Scalapack. - # Add scalapack lib here as well. - # This likely won't be need if Trilinos would compile with Scalapack options.extend([ '-DTPL_ENABLE_MUMPS:BOOL=ON', '-DMUMPS_LIBRARY_DIRS=%s' % spec['mumps'].prefix.lib, '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord' # order is important! -# '-DMUMPS_LIBRARY_DIRS=%s;%s' % (spec['mumps'].prefix.lib,spec['scalapack'].prefix.lib), -# '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord;scalapack' # order is important! ]) # scalapack options.extend([ - '-DTPL_ENABLE_SCALAPACK:BOOL=ON', #FIXME: Undefined symbols for architecture x86_64: "_blacs_gridinfo__", referenced from: Amesos_Scalapack::RedistributeA() in Amesos_Scalapack.cpp.o + '-DTPL_ENABLE_SCALAPACK:BOOL=ON', '-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 ]) From 21e81e92a9ac88bd498bd65aa9b77a43c27da881 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 25 Mar 2016 12:05:48 +0100 Subject: [PATCH 724/910] try to disable VerifyFortranC tests on Darwin --- var/spack/repos/builtin/packages/trilinos/package.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 1dba2b8db39..0c7d8a79673 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -110,6 +110,13 @@ def install(self, spec, prefix): '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-L%s/ -lgfortran' % libgfortran, '-DTrilinos_ENABLE_Fortran=ON' ]) + # FIXME: + # VerifyFortranC test of CMake does not contain -lgfortran and thus fails. + # This could be GCC/CMake related, but appears at least on OSX with Clang@7.0.2 and GNU Fortran@5.3.0 + if sys.platform == 'darwin': + options.extend([ + '-DTrilinos_SKIP_FORTRANCINTERFACE_VERIFY_TEST:BOOL=TRUE' + ]) # for build-debug only: options.extend([ From 26a72619aed8684019ab331889b276b8883b76c8 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 25 Mar 2016 12:42:26 +0100 Subject: [PATCH 725/910] add missing sys --- var/spack/repos/builtin/packages/trilinos/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 0c7d8a79673..e077dfbbfd1 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -1,5 +1,5 @@ from spack import * -import os +import os, sys # Trilinos is complicated to build, as an inspiration a couple of links to other repositories which build it: # https://github.com/hpcugent/easybuild-easyblocks/blob/master/easybuild/easyblocks/t/trilinos.py#L111 From 5172f09b9f966a3f1be438bab972a0267cccc73b Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 25 Mar 2016 16:47:48 +0100 Subject: [PATCH 726/910] builds on OSX again (modulo CC wrappers are solved) --- .../builtin/packages/trilinos/package.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index e077dfbbfd1..3003b4ab53f 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -113,15 +113,15 @@ def install(self, spec, prefix): # FIXME: # VerifyFortranC test of CMake does not contain -lgfortran and thus fails. # This could be GCC/CMake related, but appears at least on OSX with Clang@7.0.2 and GNU Fortran@5.3.0 - if sys.platform == 'darwin': - options.extend([ - '-DTrilinos_SKIP_FORTRANCINTERFACE_VERIFY_TEST:BOOL=TRUE' - ]) + #if sys.platform == 'darwin': + # options.extend([ + # '-DTrilinos_SKIP_FORTRANCINTERFACE_VERIFY_TEST:BOOL=TRUE' + # ]) # for build-debug only: - options.extend([ - '-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE' - ]) + #options.extend([ + # '-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE' + #]) # suite-sparse related if '+suite-sparse' in spec: @@ -194,6 +194,11 @@ def install(self, spec, prefix): '-DTrilinos_ENABLE_Pike=OFF', '-DTrilinos_ENABLE_STK=OFF' ]) + if sys.platform == 'darwin': + options.extend([ + '-DTrilinos_ENABLE_FEI=OFF' + ]) + with working_dir('spack-build', create=True): cmake('..', *options) From eec50b524bdf9e04f9e3536bc389ce633133373b Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 25 Mar 2016 16:58:14 +0100 Subject: [PATCH 727/910] minor cleanup --- .../builtin/packages/trilinos/package.py | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 3003b4ab53f..b1caca96ff7 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -28,6 +28,7 @@ class Trilinos(Package): variant('superlu-dist', default=True, description='Compile with SuperluDist solvers') variant('hypre', default=True, description='Compile with Hypre preconditioner') variant('hdf5', default=True, description='Compile with HDF5') + variant('scalapack', default=True, description='Compile with Scalapack') variant('suite-sparse', default=True, description='Compile with SuiteSparse solvers') variant('python', default=True, description='Build python wrappers') variant('shared', default=True, description='Enables the build of shared libraries') @@ -110,13 +111,6 @@ def install(self, spec, prefix): '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-L%s/ -lgfortran' % libgfortran, '-DTrilinos_ENABLE_Fortran=ON' ]) - # FIXME: - # VerifyFortranC test of CMake does not contain -lgfortran and thus fails. - # This could be GCC/CMake related, but appears at least on OSX with Clang@7.0.2 and GNU Fortran@5.3.0 - #if sys.platform == 'darwin': - # options.extend([ - # '-DTrilinos_SKIP_FORTRANCINTERFACE_VERIFY_TEST:BOOL=TRUE' - # ]) # for build-debug only: #options.extend([ @@ -135,6 +129,11 @@ def install(self, spec, prefix): '-DUMFPACK_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, '-DUMFPACK_LIBRARY_NAMES=umfpack;amd;colamd;cholmod;suitesparseconfig' ]) + else: + options.extend([ + '-DTPL_ENABLE_Cholmod:BOOL=OFF', + '-DTPL_ENABLE_UMFPACK:BOOL=OFF', + ]) # metis / parmetis if '+parmetis' in spec: # metis is required, see variants_check() @@ -148,6 +147,11 @@ def install(self, spec, prefix): '-DParMETIS_LIBRARY_NAMES=parmetis;metis', '-DTPL_ParMETIS_INCLUDE_DIRS=%s' % spec['parmetis'].prefix.include ]) + else: + options.extend([ + '-DTPL_ENABLE_METIS:BOOL=OFF', + '-DTPL_ENABLE_ParMETIS:BOOL=OFF', + ]) # mumps if '+mumps' in spec: @@ -156,12 +160,22 @@ def install(self, spec, prefix): '-DMUMPS_LIBRARY_DIRS=%s' % spec['mumps'].prefix.lib, '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord' # order is important! ]) + else: + options.extend([ + '-DTPL_ENABLE_MUMPS:BOOL=OFF', + ]) # scalapack - options.extend([ - '-DTPL_ENABLE_SCALAPACK:BOOL=ON', - '-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 - ]) + if '+scalapack' in spec: + options.extend([ + '-DTPL_ENABLE_SCALAPACK:BOOL=ON', + '-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 + ]) + else: + options.extend([ + '-DTPL_ENABLE_SCALAPACK:BOOL=OFF', + ]) + # superlu-dist: if '+superlu-dist' in spec: @@ -181,12 +195,22 @@ def install(self, spec, prefix): options.extend([ '-DHAVE_SUPERLUDIST_LUSTRUCTINIT_2ARG:BOOL=ON' ]) + else: + options.extend([ + '-DTPL_ENABLE_SuperLUDist:BOOL=OFF', + ]) + # python if '~python' in spec: options.extend([ '-DTrilinos_ENABLE_PyTrilinos:BOOL=OFF' ]) + else: + options.extend([ + '-DTrilinos_ENABLE_PyTrilinos:BOOL=ON' + ]) + # disable due to compiler / config errors: options.extend([ From 6699399e5191004ed924b9a8916ec1ae61223000 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 25 Mar 2016 17:00:42 +0100 Subject: [PATCH 728/910] add when to depends_on scalapack --- var/spack/repos/builtin/packages/trilinos/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index b1caca96ff7..8b664051371 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -49,7 +49,7 @@ class Trilinos(Package): depends_on('netcdf+mpi') depends_on('parmetis',when='+parmetis') depends_on('mumps+metis+parmetis+shared',when='+mumps') # build errors with static libs - depends_on('scalapack') + depends_on('scalapack',when='+scalapack') depends_on('superlu-dist',when='+superlu-dist') depends_on('hypre',when='+hypre') depends_on('hdf5+mpi',when='+hdf5') From c7011d21aa95145c7b3978f4220b211e39c9ecf2 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 25 Mar 2016 22:15:39 +0100 Subject: [PATCH 729/910] update dependencies. Builds fine on OSX --- .../repos/builtin/packages/dealii/package.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index cddf5cd98ba..5a50481b9b8 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -12,21 +12,21 @@ class Dealii(Package): depends_on ("lapack") depends_on ("mpi") - #depends_on ("arpack") + depends_on ("arpack-ng") depends_on ("boost") - #depends_on ("doxygen") - depends_on ("hdf5+mpi") + depends_on ("doxygen") + depends_on ("hdf5+mpi~cxx") #FIXME NetCDF declares dependency with ~cxx, why? depends_on ("metis") - #depends_on "muparser" + depends_on ("muparser") depends_on ("netcdf") - #depends_on ("numdiff") - #depends_on ("oce") + #depends_on ("numdiff") #FIXME + depends_on ("oce") depends_on ("p4est") depends_on ("parmetis") depends_on ("petsc") - #depends_on ("slepc") - depends_on ("SuiteSparse") - #depends_on "tbb" + depends_on ("slepc") + depends_on ("suite-sparse") + depends_on ("tbb") depends_on ("trilinos") def install(self, spec, prefix): @@ -42,21 +42,21 @@ def install(self, spec, prefix): '-DCMAKE_BUILD_TYPE=DebugRelease', '-DDEAL_II_WITH_THREADS:BOOL=ON' '-DDEAL_II_WITH_MPI:BOOL=ON', - '-DCMAKE_C_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), + '-DCMAKE_C_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), # FIXME: avoid hardcoding mpi wrappers names '-DCMAKE_CXX_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpic++'), '-DCMAKE_Fortran_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), - #'-DARPACK_DIR=%s' % , + '-DARPACK_DIR=%s' % spec['arpack-ng'].prefix, '-DBOOST_DIR=%s' % spec['boost'].prefix, '-DHDF5_DIR=%s' % spec['hdf5'].prefix, '-DMETIS_DIR=%s' % spec['metis'].prefix, - #'-DMUPARSER_DIR=%s ' + '-DMUPARSER_DIR=%s ' % spec['muparser'].prefix, '-DNETCDF_DIR=%s' % spec['netcdf'].prefix, - #'-DOPENCASCADE_DIR= + '-DOPENCASCADE_DIR=%s' % spec['oce'].prefix, '-DP4EST_DIR=%s' % spec['p4est'].prefix, '-DPETSC_DIR=%s' % spec['petsc'].prefix, - #'-DSLEPC_DIR= - '-DUMFPACK_DIR=%s' % spec['SuiteSparse'].prefix, - #'-DTBB_DIR=%s' + '-DSLEPC_DIR=%s' % spec['slepc'].prefix, + '-DUMFPACK_DIR=%s' % spec['suite-sparse'].prefix, + '-DTBB_DIR=%s' % spec['tbb'].prefix, '-DTRILINOS_DIR=%s' % spec['trilinos'].prefix ]) From 497adc3b42191c000323c55aae5da32fab901265 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 25 Mar 2016 22:45:19 -0400 Subject: [PATCH 730/910] Added new version; old versions don't work with Python3. --- var/spack/repos/builtin/packages/py-cython/package.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-cython/package.py b/var/spack/repos/builtin/packages/py-cython/package.py index 68eb735ad93..4d728fa8fbd 100644 --- a/var/spack/repos/builtin/packages/py-cython/package.py +++ b/var/spack/repos/builtin/packages/py-cython/package.py @@ -3,10 +3,13 @@ class PyCython(Package): """The Cython compiler for writing C extensions for the Python language.""" homepage = "https://pypi.python.org/pypi/cython" - url = "https://pypi.python.org/packages/source/C/Cython/cython-0.22.tar.gz" + url = "https://pypi.python.org/packages/source/C/Cython/Cython-0.22.tar.gz" - version('0.21.2', 'd21adb870c75680dc857cd05d41046a4') + version('0.23.4', '157df1f69bcec6b56fd97e0f2e057f6e') + + # These versions contain illegal Python3 code... version('0.22', '1ae25add4ef7b63ee9b4af697300d6b6') + version('0.21.2', 'd21adb870c75680dc857cd05d41046a4') extends('python') From f603c82e814ebc59ba03b0b83be86f1a4bfcbd94 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sat, 26 Mar 2016 18:43:55 +0100 Subject: [PATCH 731/910] qualify Amesos link errors --- var/spack/repos/builtin/packages/trilinos/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 8b664051371..0ba7a51b8b9 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -48,7 +48,7 @@ class Trilinos(Package): depends_on('mpi') depends_on('netcdf+mpi') depends_on('parmetis',when='+parmetis') - depends_on('mumps+metis+parmetis+shared',when='+mumps') # build errors with static libs + depends_on('mumps+metis+parmetis+shared',when='+mumps') # Amesos link errors with static: "__gfortran_adjustl", referenced from: _dmumps_ in libdmumps.a(dmumps_driver.o) "_mpi_abort_", referenced from: _mumps_abort_ in libmumps_common.a(tools_common.o) depends_on('scalapack',when='+scalapack') depends_on('superlu-dist',when='+superlu-dist') depends_on('hypre',when='+hypre') From 965ce633c311e2bcb765f5e9900f0da6205f9c23 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sat, 26 Mar 2016 18:51:59 +0100 Subject: [PATCH 732/910] cleanup variants --- .../builtin/packages/trilinos/package.py | 40 ++++++------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 0ba7a51b8b9..48bffc65e0d 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -22,13 +22,11 @@ class Trilinos(Package): version('11.14.2', 'a43590cf896c677890d75bfe75bc6254') version('11.14.1', '40febc57f76668be8b6a77b7607bb67f') - variant('metis', default=True, description='Compile with METIS') - variant('parmetis', default=True, description='Compile with ParMETIS') + variant('metis', default=True, description='Compile with METIS and ParMETIS') variant('mumps', default=True, description='Compile with support for MUMPS solvers') variant('superlu-dist', default=True, description='Compile with SuperluDist solvers') variant('hypre', default=True, description='Compile with Hypre preconditioner') variant('hdf5', default=True, description='Compile with HDF5') - variant('scalapack', default=True, description='Compile with Scalapack') variant('suite-sparse', default=True, description='Compile with SuiteSparse solvers') variant('python', default=True, description='Build python wrappers') variant('shared', default=True, description='Enables the build of shared libraries') @@ -47,9 +45,9 @@ class Trilinos(Package): # MPI related dependencies depends_on('mpi') depends_on('netcdf+mpi') - depends_on('parmetis',when='+parmetis') - depends_on('mumps+metis+parmetis+shared',when='+mumps') # Amesos link errors with static: "__gfortran_adjustl", referenced from: _dmumps_ in libdmumps.a(dmumps_driver.o) "_mpi_abort_", referenced from: _mumps_abort_ in libmumps_common.a(tools_common.o) - depends_on('scalapack',when='+scalapack') + depends_on('parmetis',when='+metis') + depends_on('mumps+mpi+metis+parmetis+shared',when='+mumps') # Amesos link errors with static: "__gfortran_adjustl", referenced from: _dmumps_ in libdmumps.a(dmumps_driver.o) "_mpi_abort_", referenced from: _mumps_abort_ in libmumps_common.a(tools_common.o) + depends_on('scalapack',when='+mumps') depends_on('superlu-dist',when='+superlu-dist') depends_on('hypre',when='+hypre') depends_on('hdf5+mpi',when='+hdf5') @@ -60,11 +58,8 @@ class Trilinos(Package): # check that the combination of variants makes sense def variants_check(self): - if '+parmetis' in self.spec and '+metis' not in self.spec: - raise RuntimeError('You cannot use the variant parmetis without metis') - - if '+mumps' in self.spec and '+parmetis' not in self.spec: - raise RuntimeError('You cannot use the variant mumps without parmetis') + if '+mumps' in self.spec and '+metis' not in self.spec: + raise RuntimeError('You cannot use the variant mumps without metis') if '+superlu-dist' in self.spec and self.spec.satisfies('@:11.4.3'): # For Trilinos v11 we need to force SuperLUDist=OFF, @@ -136,7 +131,7 @@ def install(self, spec, prefix): ]) # metis / parmetis - if '+parmetis' in spec: # metis is required, see variants_check() + if '+metis' in spec: options.extend([ '-DTPL_ENABLE_METIS:BOOL=ON', '-DMETIS_LIBRARY_DIRS=%s' % spec['metis'].prefix.lib, @@ -153,30 +148,21 @@ def install(self, spec, prefix): '-DTPL_ENABLE_ParMETIS:BOOL=OFF', ]) - # mumps + # mumps / scalapack if '+mumps' in spec: options.extend([ '-DTPL_ENABLE_MUMPS:BOOL=ON', '-DMUMPS_LIBRARY_DIRS=%s' % spec['mumps'].prefix.lib, - '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord' # order is important! - ]) - else: - options.extend([ - '-DTPL_ENABLE_MUMPS:BOOL=OFF', - ]) - - # scalapack - if '+scalapack' in spec: - options.extend([ + '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord', # order is important! '-DTPL_ENABLE_SCALAPACK:BOOL=ON', '-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 ]) else: options.extend([ + '-DTPL_ENABLE_MUMPS:BOOL=OFF', '-DTPL_ENABLE_SCALAPACK:BOOL=OFF', ]) - # superlu-dist: if '+superlu-dist' in spec: # Amesos, conflicting types of double and complex SLU_D @@ -202,13 +188,13 @@ def install(self, spec, prefix): # python - if '~python' in spec: + if '+python' in spec: options.extend([ - '-DTrilinos_ENABLE_PyTrilinos:BOOL=OFF' + '-DTrilinos_ENABLE_PyTrilinos:BOOL=ON' ]) else: options.extend([ - '-DTrilinos_ENABLE_PyTrilinos:BOOL=ON' + '-DTrilinos_ENABLE_PyTrilinos:BOOL=OFF' ]) From 9c391bbda1c8379dd60cf21d82779db704183f32 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sat, 26 Mar 2016 20:52:08 +0100 Subject: [PATCH 733/910] minor changes to variants --- var/spack/repos/builtin/packages/dealii/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 5a50481b9b8..19bb3814be5 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -12,7 +12,7 @@ class Dealii(Package): depends_on ("lapack") depends_on ("mpi") - depends_on ("arpack-ng") + depends_on ("arpack-ng+mpi") depends_on ("boost") depends_on ("doxygen") depends_on ("hdf5+mpi~cxx") #FIXME NetCDF declares dependency with ~cxx, why? @@ -23,7 +23,7 @@ class Dealii(Package): depends_on ("oce") depends_on ("p4est") depends_on ("parmetis") - depends_on ("petsc") + depends_on ("petsc+mpi") depends_on ("slepc") depends_on ("suite-sparse") depends_on ("tbb") From 11ca06b1045283e2c89dc84a91ceb88fcadb6163 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sat, 26 Mar 2016 20:35:34 -0400 Subject: [PATCH 734/910] Added new py-cython version --- var/spack/repos/builtin/packages/py-cython/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-cython/package.py b/var/spack/repos/builtin/packages/py-cython/package.py index 4d728fa8fbd..072355026e1 100644 --- a/var/spack/repos/builtin/packages/py-cython/package.py +++ b/var/spack/repos/builtin/packages/py-cython/package.py @@ -5,6 +5,7 @@ class PyCython(Package): homepage = "https://pypi.python.org/pypi/cython" url = "https://pypi.python.org/packages/source/C/Cython/Cython-0.22.tar.gz" + version('0.23.5', '66b62989a67c55af016c916da36e7514') version('0.23.4', '157df1f69bcec6b56fd97e0f2e057f6e') # These versions contain illegal Python3 code... From 9b130e1d199bcff10903e981bd7592ed941d0107 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sat, 26 Mar 2016 20:42:35 -0400 Subject: [PATCH 735/910] Added googletest package. --- .../builtin/packages/googletest/package.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 var/spack/repos/builtin/packages/googletest/package.py diff --git a/var/spack/repos/builtin/packages/googletest/package.py b/var/spack/repos/builtin/packages/googletest/package.py new file mode 100644 index 00000000000..ed27afa10ce --- /dev/null +++ b/var/spack/repos/builtin/packages/googletest/package.py @@ -0,0 +1,40 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install googletest +# +# You can always get back here to change things with: +# +# spack edit googletest +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class Googletest(Package): + """Google test framework for C++. Also called gtest.""" + homepage = "https://github.com/google/googletest" + url = "https://github.com/google/googletest/tarball/release-1.7.0" + + version('1.7.0', '5eaf03ed925a47b37c8e1d559eb19bc4') + + depends_on("cmake") + + def install(self, spec, prefix): + which('cmake')('.', *std_cmake_args) + + make() + + # Google Test doesn't have a make install + # We have to do our own install here. + install_tree('include', prefix.include) + + mkdirp(prefix.lib) + install('./libgtest.a', '%s' % prefix.lib) + install('./libgtest_main.a', '%s' % prefix.lib) + From a82a587b3298ffec0c11ddf1c9639c6de228fa6a Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sun, 27 Mar 2016 06:17:43 +0200 Subject: [PATCH 736/910] lower requirement on mumps --- var/spack/repos/builtin/packages/trilinos/package.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 48bffc65e0d..e8c4bcd7e21 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -46,7 +46,7 @@ class Trilinos(Package): depends_on('mpi') depends_on('netcdf+mpi') depends_on('parmetis',when='+metis') - depends_on('mumps+mpi+metis+parmetis+shared',when='+mumps') # Amesos link errors with static: "__gfortran_adjustl", referenced from: _dmumps_ in libdmumps.a(dmumps_driver.o) "_mpi_abort_", referenced from: _mumps_abort_ in libmumps_common.a(tools_common.o) + depends_on('mumps+mpi+shared',when='+mumps') # Amesos link errors with static: "__gfortran_adjustl", referenced from: _dmumps_ in libdmumps.a(dmumps_driver.o) "_mpi_abort_", referenced from: _mumps_abort_ in libmumps_common.a(tools_common.o) depends_on('scalapack',when='+mumps') depends_on('superlu-dist',when='+superlu-dist') depends_on('hypre',when='+hypre') @@ -58,9 +58,6 @@ class Trilinos(Package): # check that the combination of variants makes sense def variants_check(self): - if '+mumps' in self.spec and '+metis' not in self.spec: - raise RuntimeError('You cannot use the variant mumps without metis') - if '+superlu-dist' in self.spec and self.spec.satisfies('@:11.4.3'): # For Trilinos v11 we need to force SuperLUDist=OFF, # since only the deprecated SuperLUDist v3.3 together with an Amesos patch From 9cff241ad52ba628a0bdad47cee98bbbf6ba5511 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sun, 27 Mar 2016 10:39:09 +0200 Subject: [PATCH 737/910] fix install name on Darwin --- var/spack/repos/builtin/packages/trilinos/package.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index e8c4bcd7e21..72f904d8aa8 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -1,5 +1,5 @@ from spack import * -import os, sys +import os, sys, glob # Trilinos is complicated to build, as an inspiration a couple of links to other repositories which build it: # https://github.com/hpcugent/easybuild-easyblocks/blob/master/easybuild/easyblocks/t/trilinos.py#L111 @@ -222,3 +222,10 @@ def install(self, spec, prefix): # A workaround it to remove PyTrilinos from the COMPONENTS_LIST : if '+python' in self.spec: filter_file(r'(SET\(COMPONENTS_LIST.*)(PyTrilinos;)(.*)', (r'\1\3'), '%s/cmake/Trilinos/TrilinosConfig.cmake' % prefix.lib) + + # The shared libraries are not installed correctly on Darwin; correct this + if (sys.platform == 'darwin') and ('+shared' in spec): + fs = glob.glob(join_path(prefix.lib,"*.dylib")) + install_name_tool = which('install_name_tool') + for f in fs: + install_name_tool('-id',f,f) From 69ebc8173e1843ea570d78ac27465b0ea32c753d Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sun, 27 Mar 2016 16:04:13 +0200 Subject: [PATCH 738/910] use global tool to fix install_name --- var/spack/repos/builtin/packages/trilinos/package.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 72f904d8aa8..6608f64ea72 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -225,7 +225,4 @@ def install(self, spec, prefix): # The shared libraries are not installed correctly on Darwin; correct this if (sys.platform == 'darwin') and ('+shared' in spec): - fs = glob.glob(join_path(prefix.lib,"*.dylib")) - install_name_tool = which('install_name_tool') - for f in fs: - install_name_tool('-id',f,f) + fix_darwin_install_name(prefix.lib) From cc42391e0fa6eda490c4aeffcbfbae584f077db3 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sun, 27 Mar 2016 11:06:10 -0400 Subject: [PATCH 739/910] Added py-netcdf --- .../repos/builtin/packages/py-netcdf/package.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-netcdf/package.py diff --git a/var/spack/repos/builtin/packages/py-netcdf/package.py b/var/spack/repos/builtin/packages/py-netcdf/package.py new file mode 100644 index 00000000000..7faa15ad253 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-netcdf/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyNetcdf(Package): + """Python interface to the netCDF Library.""" + homepage = "http://unidata.github.io/netcdf4-python" + url = "https://github.com/Unidata/netcdf4-python/tarball/v1.2.3.1rel" + + version('1.2.3.1', '4fc4320d4f2a77b894ebf8da1c9895af') + + extends('python') + depends_on('py-numpy') + depends_on('py-cython') + depends_on('netcdf') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 439d3b3ddb4e53721b216bd19bec1e8fa16faf89 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sun, 27 Mar 2016 17:38:25 +0200 Subject: [PATCH 740/910] mumps: add install_name / soname --- var/spack/repos/builtin/packages/mumps/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index b295857ab59..26440ab7c81 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -113,13 +113,13 @@ def write_makefile_inc(self): # Building dylibs with mpif90 causes segfaults on 10.8 and 10.10. Use gfortran. (Homebrew) makefile_conf.extend([ 'LIBEXT=.dylib', - 'AR=%s -dynamiclib -undefined dynamic_lookup -o ' % os.environ['FC'], + 'AR=%s -dynamiclib -Wl,-install_name -Wl,%s/$(notdir $@) -undefined dynamic_lookup -o ' % (os.environ['FC'],prefix.lib), 'RANLIB=echo' ]) else: makefile_conf.extend([ 'LIBEXT=.so', - 'AR=$(FL) -shared -o', + 'AR=$(FL) -shared -Wl,-soname -Wl,%s/$(notdir $@) -o' % prefix.lib, 'RANLIB=echo' ]) else: From b0b882cbb3d903a85413acb0439a02d6ac9e26fc Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 26 Mar 2016 00:56:19 -0700 Subject: [PATCH 741/910] Speed up directives by skipping debug info in stack traversal. - `caller_locals()` doesn't need debug info, only frame locals. - `get_calling_module()` doesn't either. - Changed calls to `inspect.stack()` -> `inspect.stack(0)` --- lib/spack/llnl/util/lang.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 13d301f84e6..3b4e2c8352a 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -117,7 +117,8 @@ def caller_locals(): scope. Yes, this is some black magic, and yes it's useful for implementing things like depends_on and provides. """ - stack = inspect.stack() + # Passing zero here skips line context for speed. + stack = inspect.stack(0) try: return stack[2][0].f_locals finally: @@ -128,7 +129,8 @@ def get_calling_module_name(): """Make sure that the caller is a class definition, and return the enclosing module's name. """ - stack = inspect.stack() + # Passing zero here skips line context for speed. + stack = inspect.stack(0) try: # Make sure locals contain __module__ caller_locals = stack[2][0].f_locals From dce590fb21af844230727b283f6c8bf759ea805c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 26 Mar 2016 19:59:02 -0700 Subject: [PATCH 742/910] Add a dso_suffix variable to build_environment - Consolidate this in one place so that we don't have to do it in every build. - Will update further once better OS support is committed. Shoudl really be an attribute of the forthcoming `Platform` class. --- lib/spack/spack/build_environment.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 119a255a349..640db0c1d16 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -59,6 +59,11 @@ SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' +# Platform-specific library suffix. +dso_suffix = 'dylib' if sys.platform == 'darwin' else 'so' + + + class MakeExecutable(Executable): """Special callable executable object for make so the user can specify parallel or not on a per-invocation basis. Using @@ -246,6 +251,9 @@ def set_module_variables_for_package(pkg, module): # a Prefix object. m.prefix = pkg.prefix + # Platform-specific library suffix. + m.dso_suffix = dso_suffix + def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" From cde33205827edbed988ec4548c983ff93e9445c6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 26 Mar 2016 20:00:28 -0700 Subject: [PATCH 743/910] Add a method to find the containing directory of a library. --- lib/spack/llnl/util/filesystem.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index c4665c284cf..92f9ca29834 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -27,7 +27,7 @@ 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'set_executable', 'copy_mode', 'unset_executable_mode', - 'remove_dead_links', 'remove_linked_tree'] + 'remove_dead_links', 'remove_linked_tree', 'find_library_path'] import os import sys @@ -392,3 +392,18 @@ def remove_linked_tree(path): os.unlink(path) else: shutil.rmtree(path, True) + + +def find_library_path(libname, *paths): + """Searches for a file called in each path. + + Return: + directory where the library was found, if found. None otherwise. + + """ + for path in paths: + library = join_path(path, libname) + if os.path.exists(library): + return path + return None + From 66bb71534b3ab7eaf7b27743efc0ecf9763e4982 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 26 Mar 2016 20:02:08 -0700 Subject: [PATCH 744/910] Better blas/lapack and scipy packages: more robust, added `shared` variants. - py-scipy now builds with netlib-lapack, openblas, and atlas. - started a convention for passing lib info from blas/lapack implementations. - Improved netlib-lapack: - Also build static libs when `shared` variant is enabled. - Enable CBLAS build - needed minor patch to build correctly. - Added `shared` variant to OpenBLAS. - Made OpenBLAS build properly shared and static. --- .../builtin/packages/netlib-lapack/package.py | 40 ++++++++++++-- .../builtin/packages/openblas/package.py | 52 +++++++++++++++---- .../builtin/packages/py-scipy/package.py | 12 +++-- 3 files changed, 88 insertions(+), 16 deletions(-) diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py index c4b7ce3b048..05436332ac5 100644 --- a/var/spack/repos/builtin/packages/netlib-lapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -31,8 +31,16 @@ class NetlibLapack(Package): depends_on('cmake') depends_on('blas', when='+external-blas') - def install(self, spec, prefix): - cmake_args = ['-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), + + def patch(self): + # Fix cblas CMakeLists.txt -- has wrong case for subdirectory name. + filter_file('${CMAKE_CURRENT_SOURCE_DIR}/CMAKE/', + '${CMAKE_CURRENT_SOURCE_DIR}/cmake/', 'CBLAS/CMakeLists.txt', string=True) + + + def install_one(self, spec, prefix, shared): + cmake_args = ['-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if shared else 'OFF'), + '-DCBLAS=ON', # always build CBLAS '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), '-DLAPACKE:BOOL=%s' % ('ON' if '+lapacke' in spec else 'OFF')] if '+external-blas' in spec: @@ -45,7 +53,33 @@ def install(self, spec, prefix): cmake_args.extend(std_cmake_args) - with working_dir('spack-build', create=True): + build_dir = 'spack-build' + ('-shared' if shared else '-static') + with working_dir(build_dir, create=True): cmake('..', *cmake_args) make() make("install") + + + def install(self, spec, prefix): + # Always build static libraries. + self.install_one(spec, prefix, False) + + # Build shared libraries if requested. + if '+shared' in spec: + self.install_one(spec, prefix, True) + + + def setup_dependent_package(self, module, dspec): + # This is WIP for a prototype interface for virtual packages. + # We can update this as more builds start depending on BLAS/LAPACK. + libdir = find_library_path('libblas.a', self.prefix.lib64, self.prefix.lib) + + self.spec.blas_static_lib = join_path(libdir, 'libblas.a') + self.spec.lapack_static_lib = join_path(libdir, 'liblapack.a') + + if '+shared' in self.spec: + self.spec.blas_shared_lib = join_path(libdir, 'libblas.%s' % dso_suffix) + self.spec.lapack_shared_lib = join_path(libdir, 'liblapack.%s' % dso_suffix) + + + diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 1d10f217c45..4ec829a85b0 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -1,5 +1,6 @@ from spack import * import sys +import os class Openblas(Package): """OpenBLAS: An optimized BLAS library""" @@ -10,29 +11,60 @@ class Openblas(Package): version('0.2.16', 'fef46ab92463bdbb1479dcec594ef6dc') version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') + variant('shared', default=True, description="Build shared libraries as well as static libs.") + # virtual dependency provides('blas') provides('lapack') + def install(self, spec, prefix): - extra=[] + make_defs = ['CC=%s' % spack_cc, + 'FC=%s' % spack_fc] + + make_targets = ['libs', 'netlib'] + + # Build shared if variant is set. + if '+shared' in spec: + make_targets += ['shared'] + else: + make_defs += ['NO_SHARED=1'] + + # fix missing _dggsvd_ and _sggsvd_ if spec.satisfies('@0.2.16'): - extra.extend([ - 'BUILD_LAPACK_DEPRECATED=1' # fix missing _dggsvd_ and _sggsvd_ - ]) + make_defs += ['BUILD_LAPACK_DEPRECATED=1'] - make('libs', 'netlib', 'shared', 'CC=cc', 'FC=f77',*extra) - make("tests") - make('install', "PREFIX='%s'" % prefix) + make_args = make_defs + make_targets + make(*make_args) + + make("tests", *make_defs) + + # no quotes around prefix (spack doesn't use a shell) + make('install', "PREFIX=%s" % prefix, *make_defs) - lib_dsuffix = 'dylib' if sys.platform == 'darwin' else 'so' # Blas virtual package should provide blas.a and libblas.a with working_dir(prefix.lib): symlink('libopenblas.a', 'blas.a') symlink('libopenblas.a', 'libblas.a') - symlink('libopenblas.%s' % lib_dsuffix, 'libblas.%s' % lib_dsuffix) + if '+shared' in spec: + symlink('libopenblas.%s' % dso_suffix, 'libblas.%s' % dso_suffix) # Lapack virtual package should provide liblapack.a with working_dir(prefix.lib): symlink('libopenblas.a', 'liblapack.a') - symlink('libopenblas.%s' % lib_dsuffix, 'liblapack.%s' % lib_dsuffix) + if '+shared' in spec: + symlink('libopenblas.%s' % dso_suffix, 'liblapack.%s' % dso_suffix) + + + def setup_dependent_package(self, module, dspec): + # This is WIP for a prototype interface for virtual packages. + # We can update this as more builds start depending on BLAS/LAPACK. + libdir = find_library_path('libopenblas.a', self.prefix.lib64, self.prefix.lib) + + self.spec.blas_static_lib = join_path(libdir, 'libopenblas.a') + self.spec.lapack_static_lib = self.spec.blas_static_lib + + if '+shared' in self.spec: + self.spec.blas_shared_lib = join_path(libdir, 'libopenblas.%s' % dso_suffix) + self.spec.lapack_shared_lib = self.spec.blas_shared_lib + diff --git a/var/spack/repos/builtin/packages/py-scipy/package.py b/var/spack/repos/builtin/packages/py-scipy/package.py index c2161c90c43..4d47c641eec 100644 --- a/var/spack/repos/builtin/packages/py-scipy/package.py +++ b/var/spack/repos/builtin/packages/py-scipy/package.py @@ -11,9 +11,15 @@ class PyScipy(Package): extends('python') depends_on('py-nose') - depends_on('py-numpy') - depends_on('blas') - depends_on('lapack') + depends_on('py-numpy+blas+lapack') def install(self, spec, prefix): + if 'atlas' in spec: + # libatlas.so actually isn't always installed, but this + # seems to make the build autodetect things correctly. + env['ATLAS'] = join_path(spec['atlas'].prefix.lib, 'libatlas.' + dso_suffix) + else: + env['BLAS'] = spec['blas'].blas_shared_lib + env['LAPACK'] = spec['lapack'].lapack_shared_lib + python('setup.py', 'install', '--prefix=%s' % prefix) From e049fc2840ba81cdff2ca7edd798c5e961ae94e9 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 27 Mar 2016 11:47:20 -0700 Subject: [PATCH 745/910] Run post-install hoooks before build stage is removed. - Build will properly fail when post-install hoooks fail. - Post-install hooks have a proper working directory set now. --- lib/spack/spack/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 9af32218374..c17bec4a146 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -926,6 +926,9 @@ def build_process(): install(env_path, env_install_path) dump_packages(self.spec, packages_dir) + # Run post install hooks before build stage is removed. + spack.hooks.post_install(self) + # Stop timer. self._total_time = time.time() - start_time build_time = self._total_time - self._fetch_time @@ -954,9 +957,6 @@ def build_process(): # the database, so that we don't need to re-read from file. spack.installed_db.add(self.spec, self.prefix) - # Once everything else is done, run post install hooks - spack.hooks.post_install(self) - def sanity_check_prefix(self): """This function checks whether install succeeded.""" From 20600b8cac9488ff416397de374c2d3dacf4afe4 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sun, 27 Mar 2016 21:03:18 +0200 Subject: [PATCH 746/910] add tests and netcdf-cxx4 --- .../repos/builtin/packages/dealii/package.py | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 19bb3814be5..1d4c8002884 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -1,4 +1,5 @@ from spack import * +import shutil class Dealii(Package): """C++ software library providing well-documented tools to build finite element codes for a broad variety of PDEs.""" @@ -18,7 +19,7 @@ class Dealii(Package): depends_on ("hdf5+mpi~cxx") #FIXME NetCDF declares dependency with ~cxx, why? depends_on ("metis") depends_on ("muparser") - depends_on ("netcdf") + depends_on ("netcdf-cxx4") #depends_on ("numdiff") #FIXME depends_on ("oce") depends_on ("p4est") @@ -50,7 +51,7 @@ def install(self, spec, prefix): '-DHDF5_DIR=%s' % spec['hdf5'].prefix, '-DMETIS_DIR=%s' % spec['metis'].prefix, '-DMUPARSER_DIR=%s ' % spec['muparser'].prefix, - '-DNETCDF_DIR=%s' % spec['netcdf'].prefix, + '-DNETCDF_DIR=%s' % spec['netcdf-cxx4'].prefix, '-DOPENCASCADE_DIR=%s' % spec['oce'].prefix, '-DP4EST_DIR=%s' % spec['p4est'].prefix, '-DPETSC_DIR=%s' % spec['petsc'].prefix, @@ -65,3 +66,36 @@ def install(self, spec, prefix): make() make("test") make("install") + + # run some MPI examples with different solvers from PETSc and Trilinos + env['DEAL_II_DIR'] = prefix + # take bare-bones step-3 + with working_dir('examples/step-3'): + cmake('.') + make('release') + make('run',parallel=False) + + # take step-40 which can use both PETSc and Trilinos + # FIXME: switch step-40 to MPI run + with working_dir('examples/step-40'): + # list the number of cycles to speed up + filter_file(r'(const unsigned int n_cycles = 8;)', ('const unsigned int n_cycles = 2;'), 'step-40.cc') + cmake('.') + make('release') + make('run',parallel=False) + + # change Linear Algebra to Trilinos + filter_file(r'(#define USE_PETSC_LA.*)', (''), 'step-40.cc') + make('release') + make('run',parallel=False) + + with working_dir('examples/step-36'): + cmake('.') + make('release') + make('run',parallel=False) + + with working_dir('examples/step-54'): + cmake('.') + make('release') + # FIXME + # make('run',parallel=False) From 5695e4c03d6c98fc187ec405923cd1dcf2fbdff6 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 27 Mar 2016 15:37:52 -0500 Subject: [PATCH 747/910] Wrap the long description of an environment module so it is more readable. --- lib/spack/spack/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index d45fdde7035..f6a11c92e31 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -278,6 +278,6 @@ def write_header(self, module_file): # Long description if self.long_description: module_file.write('proc ModulesHelp { } {\n') - doc = re.sub(r'"', '\"', self.long_description) - module_file.write("puts stderr \"%s\"\n" % doc) + for line in textwrap.wrap(self.long_description, 72): + module_file.write("puts stderr \"%s\"\n" % line) module_file.write('}\n\n') From fc73e93b50a708ecfa2395a636bc32161c9d3051 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sun, 27 Mar 2016 23:47:43 +0200 Subject: [PATCH 748/910] rework netcdf --- var/spack/repos/builtin/packages/dealii/package.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 1d4c8002884..baedd155efa 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -1,5 +1,5 @@ from spack import * -import shutil +import sys class Dealii(Package): """C++ software library providing well-documented tools to build finite element codes for a broad variety of PDEs.""" @@ -39,6 +39,7 @@ def install(self, spec, prefix): if word.startswith('-DCMAKE_BUILD_TYPE'): options.remove(word) + dsuf = 'dylib' if sys.platform == 'darwin' else 'so' options.extend([ '-DCMAKE_BUILD_TYPE=DebugRelease', '-DDEAL_II_WITH_THREADS:BOOL=ON' @@ -51,7 +52,14 @@ def install(self, spec, prefix): '-DHDF5_DIR=%s' % spec['hdf5'].prefix, '-DMETIS_DIR=%s' % spec['metis'].prefix, '-DMUPARSER_DIR=%s ' % spec['muparser'].prefix, - '-DNETCDF_DIR=%s' % spec['netcdf-cxx4'].prefix, + # since Netcdf is spread among two, need to do it by hand: + '-DNETCDF_FOUND=true', + '-DNETCDF_LIBRARIES=%s;%s' % + (join_path(spec['netcdf-cxx4'].prefix.lib,'libnetcdf_c++4.%s' % dsuf), + join_path(spec['netcdf'].prefix.lib,'libnetcdf.%s' % dsuf)), + '-DNETCDF_INCLUDE_DIRS=%s;%s' % + (spec['netcdf-cxx4'].prefix.include, + spec['netcdf'].prefix.include), '-DOPENCASCADE_DIR=%s' % spec['oce'].prefix, '-DP4EST_DIR=%s' % spec['p4est'].prefix, '-DPETSC_DIR=%s' % spec['petsc'].prefix, From e0f463c4f99e4808b5647a2abae915ce0cb80561 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Sun, 27 Mar 2016 23:56:41 +0200 Subject: [PATCH 749/910] uninstall : added recursive option --- lib/spack/spack/cmd/uninstall.py | 151 +++++++++++++++++++++---------- 1 file changed, 101 insertions(+), 50 deletions(-) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 350ef372cb6..8da0fe1c4ae 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -23,19 +23,24 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from __future__ import print_function -import sys + import argparse +import sys import llnl.util.tty as tty -from llnl.util.tty.colify import colify - import spack import spack.cmd import spack.repository from spack.cmd.find import display_specs from spack.package import PackageStillNeededError -description="Remove an installed package" +description = "Remove an installed package" + +error_message = """You can either: + a) Use a more specific spec, or + b) use spack uninstall -a to uninstall ALL matching specs. +""" + def setup_parser(subparser): subparser.add_argument( @@ -44,10 +49,81 @@ def setup_parser(subparser): subparser.add_argument( '-a', '--all', action='store_true', dest='all', help="USE CAREFULLY. Remove ALL installed packages that match each " + - "supplied spec. i.e., if you say uninstall libelf, ALL versions of " + - "libelf are uninstalled. This is both useful and dangerous, like rm -r.") + "supplied spec. i.e., if you say uninstall libelf, ALL versions of " + + "libelf are uninstalled. This is both useful and dangerous, like rm -r.") subparser.add_argument( - 'packages', nargs=argparse.REMAINDER, help="specs of packages to uninstall") + '-r', '--recursive', action='store_true', dest='recursive', + help='Uninstall all the packages that depends on the ones for which we required explicit removal.' + + ) + subparser.add_argument('packages', nargs=argparse.REMAINDER, help="specs of packages to uninstall") + + +def concretize_specs(specs, allow_multiple_matches=False, force=False): + """ + Returns a list of specs matching the non necessarily concretized specs given from cli + + Args: + specs: list of specs to be matched against installed packages + allow_multiple_matches : boolean (if True multiple matches for each item in specs are admitted) + + Return: + list of specs + """ + specs_from_cli = [] # List of specs that match expressions given via command line + has_errors = False + for spec in specs: + matching = spack.installed_db.query(spec) + # For each spec provided, make sure it refers to only one package. + # Fail and ask user to be unambiguous if it doesn't + if not allow_multiple_matches and len(matching) > 1: + tty.error("%s matches multiple packages:" % spec) + print() + display_specs(matching, long=True) + print() + has_errors = True + + # No installed package matches the query + if len(matching) == 0 and not force: + tty.error("%s does not match any installed packages." % spec) + has_errors = True + + specs_from_cli.extend(matching) + if has_errors: + tty.die(error_message) + + return specs_from_cli + + +def installed_dependents(specs): + dependents = {} + for item in specs: + lst = [x for x in item.package.installed_dependents if x not in specs] + if lst: + dependents[item] = lst + return dependents + + +def do_uninstall(specs, force): + specs = list(set(specs)) # Make specs unique + packages = [] + for item in specs: + try: + # should work if package is known to spack + packages.append(item.package) + except spack.repository.UnknownPackageError as e: + # The package.py file has gone away -- but still + # want to uninstall. + spack.Package(item).do_uninstall(force=True) + + # Sort packages to be uninstalled by the number of installed dependents + # This ensures we do things in the right order + def num_installed_deps(pkg): + return len(pkg.installed_dependents) + + packages.sort(key=num_installed_deps) + for item in packages: + item.do_uninstall(force=force) def uninstall(parser, args): @@ -56,50 +132,25 @@ def uninstall(parser, args): with spack.installed_db.write_transaction(): specs = spack.cmd.parse_specs(args.packages) + # Gets the list of installed specs that match the ones give via cli + uninstall_list = concretize_specs(specs, args.all, args.force) # takes care of '-a' is given in the cli + dependent_list = installed_dependents(uninstall_list) # takes care of '-r' - # For each spec provided, make sure it refers to only one package. - # Fail and ask user to be unambiguous if it doesn't - pkgs = [] - for spec in specs: - matching_specs = spack.installed_db.query(spec) - if not args.all and len(matching_specs) > 1: - tty.error("%s matches multiple packages:" % spec) - print() - display_specs(matching_specs, long=True) - print() - print("You can either:") - print(" a) Use a more specific spec, or") - print(" b) use spack uninstall -a to uninstall ALL matching specs.") - sys.exit(1) - - if len(matching_specs) == 0: - if args.force: continue - tty.die("%s does not match any installed packages." % spec) - - for s in matching_specs: - try: - # should work if package is known to spack - pkgs.append(s.package) - except spack.repository.UnknownPackageError as e: - # The package.py file has gone away -- but still - # want to uninstall. - spack.Package(s).do_uninstall(force=True) - - # Sort packages to be uninstalled by the number of installed dependents - # This ensures we do things in the right order - def num_installed_deps(pkg): - return len(pkg.installed_dependents) - pkgs.sort(key=num_installed_deps) - - # Uninstall packages in order now. - for pkg in pkgs: - try: - pkg.do_uninstall(force=args.force) - except PackageStillNeededError as e: - tty.error("Will not uninstall %s" % e.spec.format("$_$@$%@$#", color=True)) + # There are dependents but recursive uninstall wasn't a requirement + has_error = False + if dependent_list and not args.recursive and not args.force: + for spec, lst in dependent_list.items(): + tty.error("Will not uninstall %s" % spec.format("$_$@$%@$#", color=True)) print('') print("The following packages depend on it:") - display_specs(e.dependents, long=True) + display_specs(lst, long=True) print('') - print("You can use spack uninstall -f to force this action.") - sys.exit(1) + has_error = True + elif args.recursive: + for key, lst in dependent_list.items(): + uninstall_list.extend(lst) + + if has_error: + tty.die('You can use spack uninstall -f to force this action') + # Uninstall everything on the list + do_uninstall(uninstall_list, args.force) From ccd155572fc8ef066b93424e90a74f9bb8fcf435 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sun, 27 Mar 2016 18:18:54 -0400 Subject: [PATCH 750/910] Removed redundant package py-libxml2. Use libxml2 instead. --- .../repos/builtin/packages/py-libxml2/package.py | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 var/spack/repos/builtin/packages/py-libxml2/package.py diff --git a/var/spack/repos/builtin/packages/py-libxml2/package.py b/var/spack/repos/builtin/packages/py-libxml2/package.py deleted file mode 100644 index 59005428e4d..00000000000 --- a/var/spack/repos/builtin/packages/py-libxml2/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class PyLibxml2(Package): - """A Python wrapper around libxml2.""" - homepage = "https://xmlsoft.org/python.html" - url = "ftp://xmlsoft.org/libxml2/python/libxml2-python-2.6.21.tar.gz" - - version('2.6.21', '229dd2b3d110a77defeeaa73af83f7f3') - - extends('python') - depends_on('libxml2') - depends_on('libxslt') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) From 7eca1284c81c3efc5a87b8a174a0974811656b3e Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sun, 27 Mar 2016 08:59:25 +0200 Subject: [PATCH 751/910] metis/parmetis/boost/oce/scalapack: correct install_name on Darwin via a global function. --- lib/spack/llnl/util/filesystem.py | 30 ++++++++++- .../repos/builtin/packages/boost/package.py | 50 ++++++++++--------- .../repos/builtin/packages/metis/package.py | 6 ++- .../packages/netlib-scalapack/package.py | 5 ++ .../repos/builtin/packages/oce/package.py | 6 ++- .../builtin/packages/parmetis/package.py | 6 ++- 6 files changed, 76 insertions(+), 27 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index c4665c284cf..46ca03bec40 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -27,9 +27,10 @@ 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'set_executable', 'copy_mode', 'unset_executable_mode', - 'remove_dead_links', 'remove_linked_tree'] + 'remove_dead_links', 'remove_linked_tree', 'fix_darwin_install_name'] import os +import glob import sys import re import shutil @@ -38,6 +39,7 @@ import getpass from contextlib import contextmanager, closing from tempfile import NamedTemporaryFile +import subprocess import llnl.util.tty as tty from spack.util.compression import ALLOWED_ARCHIVE_TYPES @@ -392,3 +394,29 @@ def remove_linked_tree(path): os.unlink(path) else: shutil.rmtree(path, True) + +def fix_darwin_install_name(path): + """ + Fix install name of dynamic libraries on Darwin to have full path. + There are two parts of this task: + (i) use install_name('-id',...) to change install name of a single lib; + (ii) use install_name('-change',...) to change the cross linking between libs. + The function assumes that all libraries are in one folder and currently won't + follow subfolders. + + Args: + path: directory in which .dylib files are alocated + + """ + libs = glob.glob(join_path(path,"*.dylib")) + for lib in libs: + # fix install name first: + subprocess.Popen(["install_name_tool", "-id",lib,lib], stdout=subprocess.PIPE).communicate()[0] + long_deps = subprocess.Popen(["otool", "-L",lib], stdout=subprocess.PIPE).communicate()[0].split('\n') + deps = [dep.partition(' ')[0][1::] for dep in long_deps[2:-1]] + # fix all dependencies: + for dep in deps: + for loc in libs: + if dep == os.path.basename(loc): + subprocess.Popen(["install_name_tool", "-change",dep,loc,lib], stdout=subprocess.PIPE).communicate()[0] + break diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index fb1f5daee7f..82ce6fbb744 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -1,5 +1,6 @@ from spack import * import spack +import sys class Boost(Package): """Boost provides free peer-reviewed portable C++ source @@ -45,34 +46,34 @@ class Boost(Package): version('1.34.1', '2d938467e8a448a2c9763e0a9f8ca7e5') version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') - default_install_libs = set(['atomic', - 'chrono', - 'date_time', - 'filesystem', + default_install_libs = set(['atomic', + 'chrono', + 'date_time', + 'filesystem', 'graph', 'iostreams', 'locale', 'log', - 'math', + 'math', 'program_options', - 'random', - 'regex', - 'serialization', - 'signals', - 'system', - 'test', - 'thread', + 'random', + 'regex', + 'serialization', + 'signals', + 'system', + 'test', + 'thread', 'wave']) - # mpi/python are not installed by default because they pull in many - # dependencies and/or because there is a great deal of customization + # mpi/python are not installed by default because they pull in many + # dependencies and/or because there is a great deal of customization # possible (and it would be difficult to choose sensible defaults) default_noinstall_libs = set(['mpi', 'python']) all_libs = default_install_libs | default_noinstall_libs for lib in all_libs: - variant(lib, default=(lib not in default_noinstall_libs), + variant(lib, default=(lib not in default_noinstall_libs), description="Compile with {0} library".format(lib)) variant('debug', default=False, description='Switch to the debug version of Boost') @@ -124,9 +125,9 @@ def determine_bootstrap_options(self, spec, withLibs, options): with open('user-config.jam', 'w') as f: compiler_wrapper = join_path(spack.build_env_path, 'c++') - f.write("using {0} : : {1} ;\n".format(boostToolsetId, + f.write("using {0} : : {1} ;\n".format(boostToolsetId, compiler_wrapper)) - + if '+mpi' in spec: f.write('using mpi : %s ;\n' % join_path(spec['mpi'].prefix.bin, 'mpicxx')) @@ -155,7 +156,7 @@ def determine_b2_options(self, spec, options): linkTypes = ['static'] if '+shared' in spec: linkTypes.append('shared') - + threadingOpts = [] if '+multithreaded' in spec: threadingOpts.append('multi') @@ -163,12 +164,12 @@ def determine_b2_options(self, spec, options): threadingOpts.append('single') if not threadingOpts: raise RuntimeError("At least one of {singlethreaded, multithreaded} must be enabled") - + options.extend([ 'toolset=%s' % self.determine_toolset(spec), 'link=%s' % ','.join(linkTypes), '--layout=tagged']) - + return threadingOpts def install(self, spec, prefix): @@ -177,14 +178,14 @@ def install(self, spec, prefix): if "+{0}".format(lib) in spec: withLibs.append(lib) if not withLibs: - # if no libraries are specified for compilation, then you dont have + # if no libraries are specified for compilation, then you dont have # to configure/build anything, just copy over to the prefix directory. src = join_path(self.stage.source_path, 'boost') mkdirp(join_path(prefix, 'include')) dst = join_path(prefix, 'include', 'boost') install_tree(src, dst) return - + # to make Boost find the user-config.jam env['BOOST_BUILD_PATH'] = './' @@ -207,4 +208,7 @@ def install(self, spec, prefix): # Boost.MPI if the threading options are not separated. for threadingOpt in threadingOpts: b2('install', 'threading=%s' % threadingOpt, *b2_options) - + + # The shared libraries are not installed correctly on Darwin; correct this + if (sys.platform == 'darwin') and ('+shared' in spec): + fix_darwin_install_name(prefix.lib) diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py index 68b9f6fd306..9301135f9f3 100644 --- a/var/spack/repos/builtin/packages/metis/package.py +++ b/var/spack/repos/builtin/packages/metis/package.py @@ -24,7 +24,7 @@ ############################################################################## from spack import * -import glob +import glob,sys class Metis(Package): """ @@ -90,3 +90,7 @@ def install(self, spec, prefix): fs = glob.glob(join_path(source_directory,'GKlib',"*.h")) for f in fs: install(f, GKlib_dist) + + # The shared library is not installed correctly on Darwin; correct this + if (sys.platform == 'darwin') and ('+shared' in spec): + fix_darwin_install_name(prefix.lib) diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index c3e6822cdfe..d59f8e41fec 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -41,6 +41,11 @@ def install(self, spec, prefix): make() make("install") + # The shared libraries are not installed correctly on Darwin; correct this + if (sys.platform == 'darwin') and ('+shared' in spec): + fix_darwin_install_name(prefix.lib) + + def setup_dependent_package(self, module, dependent_spec): spec = self.spec lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' diff --git a/var/spack/repos/builtin/packages/oce/package.py b/var/spack/repos/builtin/packages/oce/package.py index 06acb96736e..4d5081ac9d4 100644 --- a/var/spack/repos/builtin/packages/oce/package.py +++ b/var/spack/repos/builtin/packages/oce/package.py @@ -1,5 +1,5 @@ from spack import * -import platform +import platform, sys class Oce(Package): """ @@ -45,3 +45,7 @@ def install(self, spec, prefix): cmake('.', *options) make("install/strip") + + # The shared libraries are not installed correctly on Darwin; correct this + if (sys.platform == 'darwin'): + fix_darwin_install_name(prefix.lib) diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py index f5b8b6de919..ff4370aa4b4 100644 --- a/var/spack/repos/builtin/packages/parmetis/package.py +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -24,7 +24,7 @@ ############################################################################## from spack import * - +import sys class Parmetis(Package): """ @@ -83,3 +83,7 @@ def install(self, spec, prefix): cmake(source_directory, *options) make() make("install") + + # The shared library is not installed correctly on Darwin; correct this + if (sys.platform == 'darwin') and ('+shared' in spec): + fix_darwin_install_name(prefix.lib) From 3bd550d6a2ceb279743aae626c6a3a614519d2f5 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 27 Mar 2016 18:27:06 -0400 Subject: [PATCH 752/910] GEOS Python extension did not compile with Python3. --- .../repos/builtin/packages/geos/package.py | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/geos/package.py b/var/spack/repos/builtin/packages/geos/package.py index 4a2657e32f2..030703f286e 100644 --- a/var/spack/repos/builtin/packages/geos/package.py +++ b/var/spack/repos/builtin/packages/geos/package.py @@ -1,4 +1,5 @@ from spack import * +import os class Geos(Package): """GEOS (Geometry Engine - Open Source) is a C++ port of the Java @@ -10,6 +11,10 @@ class Geos(Package): homepage = "http://trac.osgeo.org/geos/" url = "http://download.osgeo.org/geos/geos-3.4.2.tar.bz2" + # Verison 3.5.0 supports Autotools and CMake + version('3.5.0', '136842690be7f504fba46b3c539438dd') + + # Versions through 3.4.2 have CMake, but only Autotools is supported version('3.4.2', 'fc5df2d926eb7e67f988a43a92683bae') version('3.4.1', '4c930dec44c45c49cd71f3e0931ded7e') version('3.4.0', 'e41318fc76b5dc764a69d43ac6b18488') @@ -21,11 +26,22 @@ class Geos(Package): version('3.3.4', '1bb9f14d57ef06ffa41cb1d67acb55a1') version('3.3.3', '8454e653d7ecca475153cc88fd1daa26') - extends('python') - depends_on('swig') +# # Python3 is not supported. +# variant('python', default=False, description='Enable Python support') + +# extends('python', when='+python') +# depends_on('python', when='+python') +# depends_on('swig', when='+python') def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--enable-python") + args = ["--prefix=%s" % prefix] +# if '+python' in spec: +# os.environ['PYTHON'] = join_path(spec['python'].prefix, 'bin', +# 'python' if spec['python'].version[:1][0] <= 2 else 'python3') +# os.environ['SWIG'] = join_path(spec['swig'].prefix, 'bin', 'swig') +# +# args.append("--enable-python") + + configure(*args) make() make("install") From 9c894f59354bafd9cd45e874be599dc8535e426e Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 28 Mar 2016 10:19:25 +0200 Subject: [PATCH 753/910] new package: netcdf-cxx --- .../repos/builtin/packages/netcdf-cxx/package.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 var/spack/repos/builtin/packages/netcdf-cxx/package.py diff --git a/var/spack/repos/builtin/packages/netcdf-cxx/package.py b/var/spack/repos/builtin/packages/netcdf-cxx/package.py new file mode 100644 index 00000000000..5334dfb853a --- /dev/null +++ b/var/spack/repos/builtin/packages/netcdf-cxx/package.py @@ -0,0 +1,15 @@ +from spack import * + +class NetcdfCxx(Package): + """C++ compatibility bindings for NetCDF""" + homepage = "http://www.unidata.ucar.edu/software/netcdf" + url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx-4.2.tar.gz" + + version('4.2', 'd32b20c00f144ae6565d9e98d9f6204c') + + depends_on('netcdf') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") From 97cdd7947894b3b631e2a209f0573ed839eb7b8a Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 28 Mar 2016 10:49:20 +0200 Subject: [PATCH 754/910] fix netcdf --- var/spack/repos/builtin/packages/dealii/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index baedd155efa..486aa6e4a4d 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -19,7 +19,7 @@ class Dealii(Package): depends_on ("hdf5+mpi~cxx") #FIXME NetCDF declares dependency with ~cxx, why? depends_on ("metis") depends_on ("muparser") - depends_on ("netcdf-cxx4") + depends_on ("netcdf-cxx") #depends_on ("numdiff") #FIXME depends_on ("oce") depends_on ("p4est") @@ -55,10 +55,10 @@ def install(self, spec, prefix): # since Netcdf is spread among two, need to do it by hand: '-DNETCDF_FOUND=true', '-DNETCDF_LIBRARIES=%s;%s' % - (join_path(spec['netcdf-cxx4'].prefix.lib,'libnetcdf_c++4.%s' % dsuf), + (join_path(spec['netcdf-cxx'].prefix.lib,'libnetcdf_c++.%s' % dsuf), join_path(spec['netcdf'].prefix.lib,'libnetcdf.%s' % dsuf)), '-DNETCDF_INCLUDE_DIRS=%s;%s' % - (spec['netcdf-cxx4'].prefix.include, + (spec['netcdf-cxx'].prefix.include, spec['netcdf'].prefix.include), '-DOPENCASCADE_DIR=%s' % spec['oce'].prefix, '-DP4EST_DIR=%s' % spec['p4est'].prefix, From e00cab75bd6721ac49bb6199d77ad99df80556a9 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 28 Mar 2016 12:12:55 +0200 Subject: [PATCH 755/910] more elaborated tests/direct solvers --- .../repos/builtin/packages/dealii/package.py | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 486aa6e4a4d..9ac1572b95a 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -77,7 +77,13 @@ def install(self, spec, prefix): # run some MPI examples with different solvers from PETSc and Trilinos env['DEAL_II_DIR'] = prefix + print('=====================================') + print('============ EXAMPLES ===============') + print('=====================================') # take bare-bones step-3 + print('=====================================') + print('============ Step-3 =================') + print('=====================================') with working_dir('examples/step-3'): cmake('.') make('release') @@ -86,22 +92,56 @@ def install(self, spec, prefix): # take step-40 which can use both PETSc and Trilinos # FIXME: switch step-40 to MPI run with working_dir('examples/step-40'): + print('=====================================') + print('========== Step-40 PETSc ============') + print('=====================================') # list the number of cycles to speed up filter_file(r'(const unsigned int n_cycles = 8;)', ('const unsigned int n_cycles = 2;'), 'step-40.cc') cmake('.') make('release') make('run',parallel=False) + print('=====================================') + print('========= Step-40 Trilinos ==========') + print('=====================================') # change Linear Algebra to Trilinos - filter_file(r'(#define USE_PETSC_LA.*)', (''), 'step-40.cc') + filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') make('release') make('run',parallel=False) + print('=====================================') + print('=== Step-40 Trilinos SuperluDist ====') + print('=====================================') + # change to direct solvers + filter_file(r'(LA::SolverCG solver\(solver_control\);)', ('TrilinosWrappers::SolverDirect::AdditionalData data(false,"Amesos_Superludist"); TrilinosWrappers::SolverDirect solver(solver_control,data);'), 'step-40.cc') + filter_file(r'(LA::MPI::PreconditionAMG preconditioner;)', (''), 'step-40.cc') + filter_file(r'(LA::MPI::PreconditionAMG::AdditionalData data;)', (''), 'step-40.cc') + filter_file(r'(preconditioner.initialize\(system_matrix, data\);)', (''), 'step-40.cc') + filter_file(r'(solver\.solve \(system_matrix, completely_distributed_solution, system_rhs,)', ('solver.solve (system_matrix, completely_distributed_solution, system_rhs);'), 'step-40.cc') + filter_file(r'(preconditioner\);)', (''), 'step-40.cc') + + make('release') + make('run',paralle=False) + + print('=====================================') + print('====== Step-40 Trilinos MUMPS =======') + print('=====================================') + # switch to Mumps + filter_file(r'(Amesos_Superludist)', ('Amesos_Mumps'), 'step-40.cc') + make('release') + make('run',parallel=False) + + print('=====================================') + print('============ Step-36 ================') + print('=====================================') with working_dir('examples/step-36'): cmake('.') make('release') make('run',parallel=False) + print('=====================================') + print('============ Step-54 ================') + print('=====================================') with working_dir('examples/step-54'): cmake('.') make('release') From f3dd889d4462d14c9f0233540bccbf6a9720bcf0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 28 Mar 2016 03:51:05 -0700 Subject: [PATCH 756/910] Fix bug with lib64 RPATH setting in cc. --- lib/spack/env/cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index c6a09724e9b..17740250d14 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -293,9 +293,9 @@ for dep in "${deps[@]}"; do if [[ -d $dep/lib64 ]]; then # libraries+=("$dep/lib64") if [[ $mode = ccld ]]; then - args=("-L$dep/lib" "-Wl,-rpath,$dep/lib" "${args[@]}") + args=("-L$dep/lib64" "-Wl,-rpath,$dep/lib64" "${args[@]}") elif [[ $mode = ld ]]; then - args=("-L$dep/lib" "-rpath" "$dep/lib" "${args[@]}") + args=("-L$dep/lib64" "-rpath" "$dep/lib64" "${args[@]}") fi fi done From d8579a5b80efd8b09e5332a922dee533f2a0a55e Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 28 Mar 2016 03:51:41 -0700 Subject: [PATCH 757/910] Simplify cc: Remove old logic and add better tests. - removed a lot of old logic that was only still needed for tests. - Added better unit tests for dependency RPATH, -L, and -I args - tests now check whether the compiler omits -I args in link mode. --- lib/spack/env/cc | 138 ++++--------------------------------- lib/spack/spack/test/cc.py | 127 +++++++++++++++++++++++++++++----- 2 files changed, 123 insertions(+), 142 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 17740250d14..4217159a25c 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -114,7 +114,9 @@ case "$command" in ;; esac -# If any of the arguments below is present then the mode is vcheck. In vcheck mode nothing is added in terms of extra search paths or libraries +# If any of the arguments below is present then the mode is vcheck. In +# vcheck mode nothing is added in terms of extra search paths or +# libraries if [ -z "$mode" ]; then for arg in "$@"; do if [ "$arg" = -v -o "$arg" = -V -o "$arg" = --version -o "$arg" = -dumpversion ]; then @@ -125,7 +127,6 @@ if [ -z "$mode" ]; then fi # Finish setting up the mode. - if [ -z "$mode" ]; then mode=ccld for arg in "$@"; do @@ -162,127 +163,18 @@ fi input_command="$@" args=("$@") -# Dump parsed values for unit testing if asked for -if [[ -n $SPACK_TEST_COMMAND ]]; then - - # - # Now do real parsing of the command line args, trying hard to keep - # non-rpath linker arguments in the proper order w.r.t. other command line - # arguments. This is important for things like groups. - # - includes=() - libraries=() - libs=() - rpaths=() - other_args=() - - while [ -n "$1" ]; do - case "$1" in - -I*) - arg="${1#-I}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - includes+=("$arg") - ;; - -L*) - arg="${1#-L}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - libraries+=("$arg") - ;; - -l*) - arg="${1#-l}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - libs+=("$arg") - ;; - -Wl,*) - arg="${1#-Wl,}" - # TODO: Handle multiple -Wl, continuations of -Wl,-rpath - if [[ $arg == -rpath=* ]]; then - arg="${arg#-rpath=}" - for rpath in ${arg//,/ }; do - rpaths+=("$rpath") - done - elif [[ $arg == -rpath,* ]]; then - arg="${arg#-rpath,}" - for rpath in ${arg//,/ }; do - rpaths+=("$rpath") - done - elif [[ $arg == -rpath ]]; then - shift; arg="$1" - if [[ $arg != '-Wl,'* ]]; then - die "-Wl,-rpath was not followed by -Wl,*" - fi - arg="${arg#-Wl,}" - for rpath in ${arg//,/ }; do - rpaths+=("$rpath") - done - else - other_args+=("-Wl,$arg") - fi - ;; - -Xlinker) - shift; arg="$1"; - if [[ $arg = -rpath=* ]]; then - rpaths+=("${arg#-rpath=}") - elif [[ $arg = -rpath ]]; then - shift; arg="$1" - if [[ $arg != -Xlinker ]]; then - die "-Xlinker -rpath was not followed by -Xlinker " - fi - shift; arg="$1" - rpaths+=("$arg") - else - other_args+=("-Xlinker") - other_args+=("$arg") - fi - ;; - *) - other_args+=("$1") - ;; - esac - shift - done - - IFS=$'\n' - case "$SPACK_TEST_COMMAND" in - dump-includes) echo "${includes[*]}";; - dump-libraries) echo "${libraries[*]}";; - dump-libs) echo "${libs[*]}";; - dump-rpaths) echo "${rpaths[*]}";; - dump-other-args) echo "${other_args[*]}";; - dump-all) - echo "INCLUDES:" - echo "${includes[*]}" - echo - echo "LIBRARIES:" - echo "${libraries[*]}" - echo - echo "LIBS:" - echo "${libs[*]}" - echo - echo "RPATHS:" - echo "${rpaths[*]}" - echo - echo "ARGS:" - echo "${other_args[*]}" - ;; - *) - die "ERROR: Unknown test command" - ;; - esac - exit -fi - # Read spack dependencies from the path environment variable IFS=':' read -ra deps <<< "$SPACK_DEPENDENCIES" for dep in "${deps[@]}"; do + # Prepend include directories if [[ -d $dep/include ]]; then if [[ $mode = cpp || $mode = cc || $mode = as || $mode = ccld ]]; then args=("-I$dep/include" "${args[@]}") fi fi + # Prepend lib and RPATH directories if [[ -d $dep/lib ]]; then - # libraries+=("$dep/lib") if [[ $mode = ccld ]]; then args=("-L$dep/lib" "-Wl,-rpath,$dep/lib" "${args[@]}") elif [[ $mode = ld ]]; then @@ -290,8 +182,8 @@ for dep in "${deps[@]}"; do fi fi + # Prepend lib64 and RPATH directories if [[ -d $dep/lib64 ]]; then - # libraries+=("$dep/lib64") if [[ $mode = ccld ]]; then args=("-L$dep/lib64" "-Wl,-rpath,$dep/lib64" "${args[@]}") elif [[ $mode = ld ]]; then @@ -302,18 +194,8 @@ done # Include all -L's and prefix/whatever dirs in rpath if [[ $mode = ccld ]]; then - # for dir in "${libraries[@]}"; do - # if [[ dir = $SPACK_INSTALL* ]]; then - # args=("-Wl,-rpath,$dir" "${args[@]}") - # fi - # done args=("-Wl,-rpath,$SPACK_PREFIX/lib" "-Wl,-rpath,$SPACK_PREFIX/lib64" "${args[@]}") elif [[ $mode = ld ]]; then - # for dir in "${libraries[@]}"; do - # if [[ dir = $SPACK_INSTALL* ]]; then - # args=("-rpath" "$dir" "${args[@]}") - # fi - # done args=("-rpath" "$SPACK_PREFIX/lib" "-rpath" "$SPACK_PREFIX/lib64" "${args[@]}") fi @@ -345,6 +227,14 @@ export PATH full_command=("$command" "${args[@]}") +# In test command mode, write out full command for Spack tests. +if [[ $SPACK_TEST_COMMAND = dump-args ]]; then + echo "${full_command[@]}" + exit +elif [[ -n $SPACK_TEST_COMMAND ]]; then + die "ERROR: Unknown test command" +fi + # # Write the input and output commands to debug logs if it's asked for. # diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index f3f6d4a22e6..0b1aeb2a8fe 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -28,6 +28,8 @@ """ import os import unittest +import tempfile +import shutil from llnl.util.filesystem import * import spack @@ -55,13 +57,40 @@ def setUp(self): self.ld = Executable(join_path(spack.build_env_path, "ld")) self.cpp = Executable(join_path(spack.build_env_path, "cpp")) - os.environ['SPACK_CC'] = "/bin/mycc" - os.environ['SPACK_PREFIX'] = "/usr" + self.realcc = "/bin/mycc" + self.prefix = "/spack-test-prefix" + + os.environ['SPACK_CC'] = self.realcc + os.environ['SPACK_PREFIX'] = self.prefix os.environ['SPACK_ENV_PATH']="test" os.environ['SPACK_DEBUG_LOG_DIR'] = "." os.environ['SPACK_COMPILER_SPEC'] = "gcc@4.4.7" os.environ['SPACK_SHORT_SPEC'] = "foo@1.2" + # Make some fake dependencies + self.tmp_deps = tempfile.mkdtemp() + self.dep1 = join_path(self.tmp_deps, 'dep1') + self.dep2 = join_path(self.tmp_deps, 'dep2') + self.dep3 = join_path(self.tmp_deps, 'dep3') + self.dep4 = join_path(self.tmp_deps, 'dep4') + + mkdirp(join_path(self.dep1, 'include')) + mkdirp(join_path(self.dep1, 'lib')) + + mkdirp(join_path(self.dep2, 'lib64')) + + mkdirp(join_path(self.dep3, 'include')) + mkdirp(join_path(self.dep3, 'lib64')) + + mkdirp(join_path(self.dep4, 'include')) + + if 'SPACK_DEPENDENCIES' in os.environ: + del os.environ['SPACK_DEPENDENCIES'] + + + def tearDown(self): + shutil.rmtree(self.tmp_deps, True) + def check_cc(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command @@ -92,6 +121,10 @@ def test_cpp_mode(self): self.check_cpp('dump-mode', [], "cpp") + def test_as_mode(self): + self.check_cc('dump-mode', ['-S'], "as") + + def test_ccld_mode(self): self.check_cc('dump-mode', [], "ccld") self.check_cc('dump-mode', ['foo.c', '-o', 'foo'], "ccld") @@ -104,27 +137,85 @@ def test_ld_mode(self): self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ld") - def test_includes(self): - self.check_cc('dump-includes', test_command, - "\n".join(["/test/include", "/other/include"])) + def test_dep_rpath(self): + """Ensure RPATHs for root package are added.""" + self.check_cc('dump-args', test_command, + self.realcc + ' ' + + '-Wl,-rpath,' + self.prefix + '/lib ' + + '-Wl,-rpath,' + self.prefix + '/lib64 ' + + ' '.join(test_command)) - def test_libraries(self): - self.check_cc('dump-libraries', test_command, - "\n".join(["/test/lib", "/other/lib"])) + def test_dep_include(self): + """Ensure a single dependency include directory is added.""" + os.environ['SPACK_DEPENDENCIES'] = self.dep4 + self.check_cc('dump-args', test_command, + self.realcc + ' ' + + '-Wl,-rpath,' + self.prefix + '/lib ' + + '-Wl,-rpath,' + self.prefix + '/lib64 ' + + '-I' + self.dep4 + '/include ' + + ' '.join(test_command)) - def test_libs(self): - self.check_cc('dump-libs', test_command, - "\n".join(["lib1", "lib2", "lib3", "lib4"])) + def test_dep_lib(self): + """Ensure a single dependency RPATH is added.""" + os.environ['SPACK_DEPENDENCIES'] = self.dep2 + self.check_cc('dump-args', test_command, + self.realcc + ' ' + + '-Wl,-rpath,' + self.prefix + '/lib ' + + '-Wl,-rpath,' + self.prefix + '/lib64 ' + + '-L' + self.dep2 + '/lib64 ' + + '-Wl,-rpath,' + self.dep2 + '/lib64 ' + + ' '.join(test_command)) - def test_rpaths(self): - self.check_cc('dump-rpaths', test_command, - "\n".join(["/first/rpath", "/second/rpath", "/third/rpath", "/fourth/rpath"])) + def test_all_deps(self): + """Ensure includes and RPATHs for all deps are added. """ + os.environ['SPACK_DEPENDENCIES'] = ':'.join([ + self.dep1, self.dep2, self.dep3, self.dep4]) + + # This is probably more constrained than it needs to be; it + # checks order within prepended args and doesn't strictly have + # to. We could loosen that if it becomes necessary + self.check_cc('dump-args', test_command, + self.realcc + ' ' + + '-Wl,-rpath,' + self.prefix + '/lib ' + + '-Wl,-rpath,' + self.prefix + '/lib64 ' + + + '-I' + self.dep4 + '/include ' + + + '-L' + self.dep3 + '/lib64 ' + + '-Wl,-rpath,' + self.dep3 + '/lib64 ' + + '-I' + self.dep3 + '/include ' + + + '-L' + self.dep2 + '/lib64 ' + + '-Wl,-rpath,' + self.dep2 + '/lib64 ' + + + '-L' + self.dep1 + '/lib ' + + '-Wl,-rpath,' + self.dep1 + '/lib ' + + '-I' + self.dep1 + '/include ' + + + ' '.join(test_command)) - def test_other_args(self): - self.check_cc('dump-other-args', test_command, - "\n".join(["arg1", "-Wl,--start-group", "arg2", "arg3", "arg4", - "-Wl,--end-group", "arg5", "arg6"])) + def test_ld_deps(self): + """Ensure no (extra) -I args or -Wl, are passed in ld mode.""" + os.environ['SPACK_DEPENDENCIES'] = ':'.join([ + self.dep1, self.dep2, self.dep3, self.dep4]) + + self.check_ld('dump-args', test_command, + 'ld ' + + '-rpath ' + self.prefix + '/lib ' + + '-rpath ' + self.prefix + '/lib64 ' + + + '-L' + self.dep3 + '/lib64 ' + + '-rpath ' + self.dep3 + '/lib64 ' + + + '-L' + self.dep2 + '/lib64 ' + + '-rpath ' + self.dep2 + '/lib64 ' + + + '-L' + self.dep1 + '/lib ' + + '-rpath ' + self.dep1 + '/lib ' + + + ' '.join(test_command)) + From 0da545ecac1c57a04abf814c9fed4f9929726ff5 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 28 Mar 2016 13:16:08 +0200 Subject: [PATCH 758/910] tell amesos that we use mumps 5 --- .../repos/builtin/packages/trilinos/package.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 6608f64ea72..90b7bff1492 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -6,6 +6,9 @@ # https://github.com/koecher/candi/blob/master/deal.II-toolchain/packages/trilinos.package # https://gitlab.com/configurations/cluster-config/blob/master/trilinos.sh # https://github.com/Homebrew/homebrew-science/blob/master/trilinos.rb +# and some relevant documentation/examples: +# https://trilinos.org/docs/dev/packages/amesos2/doc/html/classAmesos2_1_1MUMPS.html +# https://github.com/trilinos/Trilinos/issues/175 class Trilinos(Package): """The Trilinos Project is an effort to develop algorithms and enabling technologies within an object-oriented software framework for the solution of large-scale, complex multi-physics engineering and scientific problems. @@ -46,7 +49,7 @@ class Trilinos(Package): depends_on('mpi') depends_on('netcdf+mpi') depends_on('parmetis',when='+metis') - depends_on('mumps+mpi+shared',when='+mumps') # Amesos link errors with static: "__gfortran_adjustl", referenced from: _dmumps_ in libdmumps.a(dmumps_driver.o) "_mpi_abort_", referenced from: _mumps_abort_ in libmumps_common.a(tools_common.o) + depends_on('mumps@5.0:+mpi+shared',when='+mumps') # Amesos link errors with static: "__gfortran_adjustl", referenced from: _dmumps_ in libdmumps.a(dmumps_driver.o) "_mpi_abort_", referenced from: _mumps_abort_ in libmumps_common.a(tools_common.o) depends_on('scalapack',when='+mumps') depends_on('superlu-dist',when='+superlu-dist') depends_on('hypre',when='+hypre') @@ -67,6 +70,7 @@ def variants_check(self): def install(self, spec, prefix): self.variants_check() + cxx_flags = [] options = [] options.extend(std_cmake_args) @@ -91,7 +95,6 @@ def install(self, spec, prefix): '-DBoost_LIBRARY_DIRS:PATH=%s' % spec['boost'].prefix.lib, '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', '-DTrilinos_ENABLE_CXX11:BOOL=ON', - '-DTrilinos_CXX11_FLAGS=-std=c++11', '-DTPL_ENABLE_Netcdf:BOOL=ON', '-DTPL_ENABLE_HYPRE:BOOL=ON', '-DTPL_ENABLE_HDF5:BOOL=ON' @@ -154,6 +157,10 @@ def install(self, spec, prefix): '-DTPL_ENABLE_SCALAPACK:BOOL=ON', '-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 ]) + # see https://github.com/trilinos/Trilinos/blob/master/packages/amesos/README-MUMPS + cxx_flags.extend([ + '-DMUMPS_5_0' + ]) else: options.extend([ '-DTPL_ENABLE_MUMPS:BOOL=OFF', @@ -194,6 +201,10 @@ def install(self, spec, prefix): '-DTrilinos_ENABLE_PyTrilinos:BOOL=OFF' ]) + # collect CXX flags: + options.extend([ + '-DCMAKE_CXX_FLAGS:STRING=%s' % (' '.join(cxx_flags)), + ]) # disable due to compiler / config errors: options.extend([ From 5eefca43e75d2df1c2f859d199f7bb37e7142993 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 28 Mar 2016 14:35:01 +0200 Subject: [PATCH 759/910] petsc: extend variants by mumps, complex and debug --- .../repos/builtin/packages/petsc/package.py | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 3dd117eed11..dd3d2aa1626 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -17,14 +17,17 @@ class Petsc(Package): version('3.5.1', 'a557e029711ebf425544e117ffa44d8f') version('3.4.4', '7edbc68aa6d8d6a3295dd5f6c2f6979d') - variant('shared', default=True, description='Enables the build of shared libraries') - variant('mpi', default=True, description='Activates MPI support') - variant('double', default=True, description='Switches between single and double precision') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('mpi', default=True, description='Activates MPI support') + variant('double', default=True, description='Switches between single and double precision') + variant('complex', default=False, description='Build with complex numbers') + variant('debug', default=False, description='Compile in debug mode') - variant('metis', default=True, description='Activates support for metis and parmetis') - variant('hdf5', default=True, description='Activates support for HDF5 (only parallel)') - variant('boost', default=True, description='Activates support for Boost') - variant('hypre', default=True, description='Activates support for Hypre') + variant('metis', default=True, description='Activates support for metis and parmetis') + variant('hdf5', default=True, description='Activates support for HDF5 (only parallel)') + variant('boost', default=True, description='Activates support for Boost') + variant('hypre', default=True, description='Activates support for Hypre (only parallel)') + variant('mumps', default=True, description='Activates support for MUMPS (only parallel)') # Virtual dependencies depends_on('blas') @@ -40,7 +43,10 @@ class Petsc(Package): depends_on('hdf5+mpi', when='+hdf5+mpi') depends_on('parmetis', when='+metis+mpi') - depends_on('hypre', when='+hypre+mpi') + depends_on('hypre', when='+hypre+mpi~complex') # Hypre does not support complex numbers + + depends_on('mumps+mpi', when='+mumps+mpi') + depends_on('scalapack', when='+mumps+mpi') def mpi_dependent_options(self): if '~mpi' in self.spec: @@ -55,7 +61,7 @@ def mpi_dependent_options(self): # If mpi is disabled (~mpi), it's an error to have any of these enabled. # This generates a list of any such errors. errors = [error_message_fmt.format(library=x) - for x in ('hdf5', 'hypre', 'parmetis') + for x in ('hdf5', 'hypre', 'parmetis','mumps') if ('+'+x) in self.spec] if errors: errors = ['incompatible variants given'] + errors @@ -77,16 +83,17 @@ def mpi_dependent_options(self): return compiler_opts def install(self, spec, prefix): - options = ['--with-debugging=0', - '--with-ssl=0'] + options = ['--with-ssl=0'] options.extend(self.mpi_dependent_options()) options.extend([ '--with-precision=%s' % ('double' if '+double' in spec else 'single'), + '--with-scalar-type=%s' % ('complex' if '+complex' in spec else 'real'), '--with-shared-libraries=%s' % ('1' if '+shared' in spec else '0'), + '--with-debugging=%s' % ('1' if '+debug' in spec else '0'), '--with-blas-lapack-dir=%s' % spec['lapack'].prefix ]) # Activates library support if needed - for library in ('metis', 'boost', 'hdf5', 'hypre', 'parmetis'): + for library in ('metis', 'boost', 'hdf5', 'hypre', 'parmetis','mumps','scalapack'): options.append( '--with-{library}={value}'.format(library=library, value=('1' if library in spec else '0')) ) From dd60cc326c12bfa8434b68afec72f4b68c18a94c Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 28 Mar 2016 17:52:22 +0200 Subject: [PATCH 760/910] adopt modifed superlu and hypre --- var/spack/repos/builtin/packages/trilinos/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 90b7bff1492..b2dc8df0ea1 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -52,7 +52,7 @@ class Trilinos(Package): depends_on('mumps@5.0:+mpi+shared',when='+mumps') # Amesos link errors with static: "__gfortran_adjustl", referenced from: _dmumps_ in libdmumps.a(dmumps_driver.o) "_mpi_abort_", referenced from: _mumps_abort_ in libmumps_common.a(tools_common.o) depends_on('scalapack',when='+mumps') depends_on('superlu-dist',when='+superlu-dist') - depends_on('hypre',when='+hypre') + depends_on('hypre~internal-superlu',when='+hypre') depends_on('hdf5+mpi',when='+hdf5') depends_on('python',when='+python') # Needs py-numpy activated @@ -179,7 +179,7 @@ def install(self, spec, prefix): options.extend([ '-DTPL_ENABLE_SuperLUDist:BOOL=ON', '-DSuperLUDist_LIBRARY_DIRS=%s' % spec['superlu-dist'].prefix.lib, - '-DSuperLUDist_INCLUDE_DIRS=%s/superlu_dist' % spec['superlu-dist'].prefix.include # superlu_dist and superlu have the same header names :-( In order to avoid conflicts, try to keep "dist" version headers in a subfolder + '-DSuperLUDist_INCLUDE_DIRS=%s' % spec['superlu-dist'].prefix.include ]) if spec.satisfies('^superlu-dist@4.0:'): options.extend([ From 207215980b05e4916e05ded8b05044b3edbe92c0 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 28 Mar 2016 17:54:27 +0200 Subject: [PATCH 761/910] make hypre and hdf5 be ON/OFF --- var/spack/repos/builtin/packages/trilinos/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index b2dc8df0ea1..eb3bd1e0ee4 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -96,8 +96,8 @@ def install(self, spec, prefix): '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', '-DTrilinos_ENABLE_CXX11:BOOL=ON', '-DTPL_ENABLE_Netcdf:BOOL=ON', - '-DTPL_ENABLE_HYPRE:BOOL=ON', - '-DTPL_ENABLE_HDF5:BOOL=ON' + '-DTPL_ENABLE_HYPRE:BOOL=%s' % ('ON' if '+hypre' in spec else 'OFF'),, + '-DTPL_ENABLE_HDF5:BOOL=%s' % ('ON' if '+hdf5' in spec else 'OFF'), ]) # Fortran lib From 4e87cf9d7492133d30675a64703b91ec6992ca00 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 28 Mar 2016 18:00:34 +0200 Subject: [PATCH 762/910] fix extra comma --- var/spack/repos/builtin/packages/trilinos/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index eb3bd1e0ee4..71c0be61c38 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -96,7 +96,7 @@ def install(self, spec, prefix): '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', '-DTrilinos_ENABLE_CXX11:BOOL=ON', '-DTPL_ENABLE_Netcdf:BOOL=ON', - '-DTPL_ENABLE_HYPRE:BOOL=%s' % ('ON' if '+hypre' in spec else 'OFF'),, + '-DTPL_ENABLE_HYPRE:BOOL=%s' % ('ON' if '+hypre' in spec else 'OFF'), '-DTPL_ENABLE_HDF5:BOOL=%s' % ('ON' if '+hdf5' in spec else 'OFF'), ]) From 09e77812b67d92094bea8dae54617e3e02ebaa61 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 28 Mar 2016 17:49:20 +0200 Subject: [PATCH 763/910] fix installation of superlu_dist headers and add it as a dependency to petsc; add a variant to hypre to disable internal superlu --- .../repos/builtin/packages/hypre/package.py | 5 +++++ .../repos/builtin/packages/petsc/package.py | 21 ++++++++++++++++--- .../builtin/packages/superlu-dist/package.py | 11 +++++----- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py index 8d93d48d1f6..0e995532933 100644 --- a/var/spack/repos/builtin/packages/hypre/package.py +++ b/var/spack/repos/builtin/packages/hypre/package.py @@ -14,6 +14,8 @@ class Hypre(Package): # hypre does not know how to build shared libraries on Darwin variant('shared', default=sys.platform!='darwin', description="Build shared library version (disables static library)") + # SuperluDist have conflicting headers with those in Hypre + variant('internal-superlu', default=True, description="Use internal Superlu routines") depends_on("mpi") depends_on("blas") @@ -38,6 +40,9 @@ def install(self, spec, prefix): if '+shared' in self.spec: configure_args.append("--enable-shared") + if '~internal-superlu' in self.spec: + configure_args.append("--without-superlu") + # Hypre's source is staged under ./src so we'll have to manually # cd into it. with working_dir("src"): diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index dd3d2aa1626..e9b7c8a732c 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -28,6 +28,7 @@ class Petsc(Package): variant('boost', default=True, description='Activates support for Boost') variant('hypre', default=True, description='Activates support for Hypre (only parallel)') variant('mumps', default=True, description='Activates support for MUMPS (only parallel)') + variant('superlu-dist', default=True, description='Activates support for SuperluDist (only parallel)') # Virtual dependencies depends_on('blas') @@ -43,8 +44,11 @@ class Petsc(Package): depends_on('hdf5+mpi', when='+hdf5+mpi') depends_on('parmetis', when='+metis+mpi') - depends_on('hypre', when='+hypre+mpi~complex') # Hypre does not support complex numbers - + # Hypre does not support complex numbers. + # Also PETSc prefer to build it without internal superlu, likely due to conflict in headers + # see https://bitbucket.org/petsc/petsc/src/90564b43f6b05485163c147b464b5d6d28cde3ef/config/BuildSystem/config/packages/hypre.py + depends_on('hypre~internal-superlu', when='+hypre+mpi~complex') + depends_on('superlu-dist', when='+superlu-dist+mpi') depends_on('mumps+mpi', when='+mumps+mpi') depends_on('scalapack', when='+mumps+mpi') @@ -61,7 +65,7 @@ def mpi_dependent_options(self): # If mpi is disabled (~mpi), it's an error to have any of these enabled. # This generates a list of any such errors. errors = [error_message_fmt.format(library=x) - for x in ('hdf5', 'hypre', 'parmetis','mumps') + for x in ('hdf5', 'hypre', 'parmetis','mumps','superlu-dist') if ('+'+x) in self.spec] if errors: errors = ['incompatible variants given'] + errors @@ -101,6 +105,17 @@ def install(self, spec, prefix): options.append( '--with-{library}-dir={path}'.format(library=library, path=spec[library].prefix) ) + # PETSc does not pick up SuperluDist from the dir as they look for superlu_dist_4.1.a + if 'superlu-dist' in spec: + options.extend([ + '--with-superlu_dist-include=%s' % spec['superlu-dist'].prefix.include, + '--with-superlu_dist-lib=%s' % join_path(spec['superlu-dist'].prefix.lib, 'libsuperlu_dist.a'), + '--with-superlu_dist=1' + ]) + else: + options.append( + '--with-superlu_dist=0' + ) configure('--prefix=%s' % prefix, *options) diff --git a/var/spack/repos/builtin/packages/superlu-dist/package.py b/var/spack/repos/builtin/packages/superlu-dist/package.py index 9a94de8ba50..ddcb7f92252 100644 --- a/var/spack/repos/builtin/packages/superlu-dist/package.py +++ b/var/spack/repos/builtin/packages/superlu-dist/package.py @@ -1,4 +1,5 @@ from spack import * +import glob class SuperluDist(Package): """A general purpose library for the direct solution of large, sparse, nonsymmetric systems of linear equations on high performance machines.""" @@ -52,13 +53,13 @@ def install(self, spec, prefix): # system "make" # need to install by hand - headers_location = join_path(self.prefix.include,'superlu_dist') + headers_location = self.prefix.include mkdirp(headers_location) mkdirp(prefix.lib) - # FIXME: fetch all headers in the folder automatically - for header in ['Cnames.h','cublas_utils.h','dcomplex.h','html_mainpage.h','machines.h','old_colamd.h','psymbfact.h','superlu_ddefs.h','superlu_defs.h','superlu_enum_consts.h','superlu_zdefs.h','supermatrix.h','util_dist.h']: - superludist_header = join_path(self.stage.source_path, 'SRC/',header) - install(superludist_header, headers_location) + + headers = glob.glob(join_path(self.stage.source_path, 'SRC','*.h')) + for h in headers: + install(h,headers_location) superludist_lib = join_path(self.stage.source_path, 'lib/libsuperlu_dist.a') install(superludist_lib,self.prefix.lib) From 1141f1195572a81cb98e5fa439db6860fd326d43 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 29 Mar 2016 00:28:02 +0200 Subject: [PATCH 764/910] uninstall : added user confirmation --- lib/spack/spack/cmd/uninstall.py | 47 ++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 8da0fe1c4ae..7183817908d 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -24,15 +24,14 @@ ############################################################################## from __future__ import print_function -import argparse import sys +import argparse import llnl.util.tty as tty import spack import spack.cmd import spack.repository from spack.cmd.find import display_specs -from spack.package import PackageStillNeededError description = "Remove an installed package" @@ -41,6 +40,16 @@ b) use spack uninstall -a to uninstall ALL matching specs. """ +def ask_for_confirmation(message): + while True: + tty.msg(message + '[y/n]') + choice = raw_input().lower() + if choice == 'y': + break + elif choice == 'n': + sys.exit(1) + tty.warning('Please reply either "y" or "n"') + def setup_parser(subparser): subparser.add_argument( @@ -53,7 +62,11 @@ def setup_parser(subparser): "libelf are uninstalled. This is both useful and dangerous, like rm -r.") subparser.add_argument( '-r', '--recursive', action='store_true', dest='recursive', - help='Uninstall all the packages that depends on the ones for which we required explicit removal.' + help='Also uninstall any packages that depend on the ones given via command line.' + ) + subparser.add_argument( + '-y', '--yes-to-all', action='store_true', dest='yes_to_all', + help='Assume "yes" is the answer to every confirmation asked to the user.' ) subparser.add_argument('packages', nargs=argparse.REMAINDER, help="specs of packages to uninstall") @@ -96,6 +109,15 @@ def concretize_specs(specs, allow_multiple_matches=False, force=False): def installed_dependents(specs): + """ + Returns a dictionary that maps a spec with a list of its installed dependents + + Args: + specs: list of specs to be checked for dependents + + Returns: + dictionary of installed dependents + """ dependents = {} for item in specs: lst = [x for x in item.package.installed_dependents if x not in specs] @@ -105,7 +127,13 @@ def installed_dependents(specs): def do_uninstall(specs, force): - specs = list(set(specs)) # Make specs unique + """ + Uninstalls all the specs in a list. + + Args: + specs: list of specs to be uninstalled + force: force uninstallation (boolean) + """ packages = [] for item in specs: try: @@ -136,7 +164,7 @@ def uninstall(parser, args): uninstall_list = concretize_specs(specs, args.all, args.force) # takes care of '-a' is given in the cli dependent_list = installed_dependents(uninstall_list) # takes care of '-r' - # There are dependents but recursive uninstall wasn't a requirement + # Process dependent_list and update uninstall_list has_error = False if dependent_list and not args.recursive and not args.force: for spec, lst in dependent_list.items(): @@ -149,8 +177,17 @@ def uninstall(parser, args): elif args.recursive: for key, lst in dependent_list.items(): uninstall_list.extend(lst) + uninstall_list = list(set(uninstall_list)) if has_error: tty.die('You can use spack uninstall -f to force this action') + + if not args.yes_to_all: + tty.msg("The following packages will be uninstalled : ") + print('') + display_specs(uninstall_list, long=True) + print('') + ask_for_confirmation('Do you want to proceed ? ') + # Uninstall everything on the list do_uninstall(uninstall_list, args.force) From a14527ec0619151d6defc909a46aefd2a9378cb7 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 28 Mar 2016 15:34:25 -0700 Subject: [PATCH 765/910] Add command to compiler input log. --- lib/spack/env/cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 4217159a25c..68cd8514f44 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -241,7 +241,7 @@ fi if [[ $SPACK_DEBUG = TRUE ]]; then input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.in.log" output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.out.log" - echo "$input_command" >> $input_log + echo "$command $input_command" >> $input_log echo "$mode ${full_command[@]}" >> $output_log fi From f9bcce6a683b1d948190b9927254e3ee29d3606c Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 29 Mar 2016 09:19:24 +0200 Subject: [PATCH 766/910] turn off python by default --- var/spack/repos/builtin/packages/trilinos/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 71c0be61c38..0ab4addd52a 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -31,7 +31,8 @@ class Trilinos(Package): variant('hypre', default=True, description='Compile with Hypre preconditioner') variant('hdf5', default=True, description='Compile with HDF5') variant('suite-sparse', default=True, description='Compile with SuiteSparse solvers') - variant('python', default=True, description='Build python wrappers') + # not everyone has py-numpy activated, keep it disabled by default to avoid configure errors + variant('python', default=False, description='Build python wrappers') variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds a debug version of the libraries') @@ -55,7 +56,7 @@ class Trilinos(Package): depends_on('hypre~internal-superlu',when='+hypre') depends_on('hdf5+mpi',when='+hdf5') - depends_on('python',when='+python') # Needs py-numpy activated + depends_on('python',when='+python') patch('umfpack_from_suitesparse.patch') From 7753d823cbe1d1983957fb948bf095d624d76b7c Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 29 Mar 2016 09:27:55 +0200 Subject: [PATCH 767/910] add a comment on static mumps --- var/spack/repos/builtin/packages/trilinos/package.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 0ab4addd52a..6223848c689 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -7,7 +7,6 @@ # https://gitlab.com/configurations/cluster-config/blob/master/trilinos.sh # https://github.com/Homebrew/homebrew-science/blob/master/trilinos.rb # and some relevant documentation/examples: -# https://trilinos.org/docs/dev/packages/amesos2/doc/html/classAmesos2_1_1MUMPS.html # https://github.com/trilinos/Trilinos/issues/175 class Trilinos(Package): """The Trilinos Project is an effort to develop algorithms and enabling technologies within an object-oriented @@ -50,7 +49,14 @@ class Trilinos(Package): depends_on('mpi') depends_on('netcdf+mpi') depends_on('parmetis',when='+metis') - depends_on('mumps@5.0:+mpi+shared',when='+mumps') # Amesos link errors with static: "__gfortran_adjustl", referenced from: _dmumps_ in libdmumps.a(dmumps_driver.o) "_mpi_abort_", referenced from: _mumps_abort_ in libmumps_common.a(tools_common.o) + # Trilinos' Tribits config system is limited which makes it + # very tricky to link Amesos with static MUMPS, see + # https://trilinos.org/docs/dev/packages/amesos2/doc/html/classAmesos2_1_1MUMPS.html + # One could work it out by getting linking flags from mpif90 --showme:link (or alike) + # and adding results to -DTrilinos_EXTRA_LINK_FLAGS + # together with Blas and Lapack and ScaLAPACK and Blacs and -lgfortran and + # it may work at the end. But let's avoid all this by simply using shared libs + depends_on('mumps@5.0:+mpi+shared',when='+mumps') depends_on('scalapack',when='+mumps') depends_on('superlu-dist',when='+superlu-dist') depends_on('hypre~internal-superlu',when='+hypre') From 63c3feb79e7e4022e5565deb9f2749fa9732e7d2 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 29 Mar 2016 09:41:57 +0200 Subject: [PATCH 768/910] minor fixes to hypre and muparser on Linux --- var/spack/repos/builtin/packages/hypre/package.py | 3 +++ var/spack/repos/builtin/packages/muparser/package.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py index 0e995532933..4b915daa687 100644 --- a/var/spack/repos/builtin/packages/hypre/package.py +++ b/var/spack/repos/builtin/packages/hypre/package.py @@ -42,6 +42,9 @@ def install(self, spec, prefix): if '~internal-superlu' in self.spec: configure_args.append("--without-superlu") + # MLI and FEI do not build without superlu on Linux + configure_args.append("--without-mli") + configure_args.append("--without-fei") # Hypre's source is staged under ./src so we'll have to manually # cd into it. diff --git a/var/spack/repos/builtin/packages/muparser/package.py b/var/spack/repos/builtin/packages/muparser/package.py index a1a9ff90e53..19ca8ce2874 100644 --- a/var/spack/repos/builtin/packages/muparser/package.py +++ b/var/spack/repos/builtin/packages/muparser/package.py @@ -14,5 +14,5 @@ def install(self, spec, prefix): configure(*options) - make() + make(parallel=False) make("install") From f80e839ff48809a5ba8b7343d2378133a3fbce82 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 29 Mar 2016 02:58:05 -0700 Subject: [PATCH 769/910] Handle Darwin's ld -r option properly - ld -r doesn't work with RPATH on OS X. - for GNU ld, the -r option only means 'relocatable', and doesn't affect RPATH. - This adds special handling to omit RPATHs for ld -r on OS X --- lib/spack/env/cc | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 68cd8514f44..2eb6f46afe5 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -85,6 +85,10 @@ done # ccld compile & link # vcheck version check # +# Depending on the mode, we may or may not add extra rpaths. +# This variable controls whether they are added. +add_rpaths=true + command=$(basename "$0") case "$command" in cc|c89|c99|gcc|clang|icc|pgcc|xlc) @@ -108,6 +112,17 @@ case "$command" in ;; ld) mode=ld + + # Darwin's linker has a -r argument that merges object files + # together. It doesn't work with -rpath. + if [[ $OSTYPE = darwin* ]]; then + for arg in "$@"; do + if [ "$arg" = -r ]; then + add_rpaths=false + break + fi + done + fi ;; *) die "Unkown compiler: $command" @@ -176,27 +191,31 @@ for dep in "${deps[@]}"; do # Prepend lib and RPATH directories if [[ -d $dep/lib ]]; then if [[ $mode = ccld ]]; then - args=("-L$dep/lib" "-Wl,-rpath,$dep/lib" "${args[@]}") + $add_rpaths && args=("-Wl,-rpath,$dep/lib" "${args[@]}") + args=("-L$dep/lib" "${args[@]}") elif [[ $mode = ld ]]; then - args=("-L$dep/lib" "-rpath" "$dep/lib" "${args[@]}") + $add_rpaths && args=("-rpath" "$dep/lib" "${args[@]}") + args=("-L$dep/lib" "${args[@]}") fi fi # Prepend lib64 and RPATH directories if [[ -d $dep/lib64 ]]; then if [[ $mode = ccld ]]; then - args=("-L$dep/lib64" "-Wl,-rpath,$dep/lib64" "${args[@]}") + $add_rpaths && args=("-Wl,-rpath,$dep/lib64" "${args[@]}") + args=("-L$dep/lib64" "${args[@]}") elif [[ $mode = ld ]]; then - args=("-L$dep/lib64" "-rpath" "$dep/lib64" "${args[@]}") + $add_rpaths && args=("-rpath" "$dep/lib64" "${args[@]}") + args=("-L$dep/lib64" "${args[@]}") fi fi done # Include all -L's and prefix/whatever dirs in rpath if [[ $mode = ccld ]]; then - args=("-Wl,-rpath,$SPACK_PREFIX/lib" "-Wl,-rpath,$SPACK_PREFIX/lib64" "${args[@]}") + $add_rpaths && args=("-Wl,-rpath,$SPACK_PREFIX/lib" "-Wl,-rpath,$SPACK_PREFIX/lib64" "${args[@]}") elif [[ $mode = ld ]]; then - args=("-rpath" "$SPACK_PREFIX/lib" "-rpath" "$SPACK_PREFIX/lib64" "${args[@]}") + $add_rpaths && args=("-rpath" "$SPACK_PREFIX/lib" "-rpath" "$SPACK_PREFIX/lib64" "${args[@]}") fi # @@ -241,8 +260,8 @@ fi if [[ $SPACK_DEBUG = TRUE ]]; then input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.in.log" output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.out.log" - echo "$command $input_command" >> $input_log - echo "$mode ${full_command[@]}" >> $output_log + echo "[$mode] $command $input_command" >> $input_log + echo "[$mode] ${full_command[@]}" >> $output_log fi exec "${full_command[@]}" From ccc1b23bea57a9c8a31ab1f4e9b739b54440466b Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Mon, 28 Mar 2016 16:28:29 -0500 Subject: [PATCH 770/910] this allows gcc4.9.3 to build on OSX10 --- .../packages/gcc/darwin/gcc-4.9.patch1 | 42 +++++++++++++++++++ .../packages/gcc/darwin/gcc-4.9.patch2 | 28 +++++++++++++ .../repos/builtin/packages/gcc/package.py | 12 +++++- 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 var/spack/repos/builtin/packages/gcc/darwin/gcc-4.9.patch1 create mode 100644 var/spack/repos/builtin/packages/gcc/darwin/gcc-4.9.patch2 diff --git a/var/spack/repos/builtin/packages/gcc/darwin/gcc-4.9.patch1 b/var/spack/repos/builtin/packages/gcc/darwin/gcc-4.9.patch1 new file mode 100644 index 00000000000..444e292786d --- /dev/null +++ b/var/spack/repos/builtin/packages/gcc/darwin/gcc-4.9.patch1 @@ -0,0 +1,42 @@ +diff --git a/gcc/configure b/gcc/configure +index 9523773..52b0bf7 100755 +--- a/gcc/configure ++++ b/gcc/configure +@@ -24884,7 +24884,7 @@ if test "${gcc_cv_as_ix86_filds+set}" = set; then : + else + gcc_cv_as_ix86_filds=no + if test x$gcc_cv_as != x; then +- $as_echo 'filds mem; fists mem' > conftest.s ++ $as_echo 'filds (%ebp); fists (%ebp)' > conftest.s + if { ac_try='$gcc_cv_as $gcc_cv_as_flags -o conftest.o conftest.s >&5' + { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5 + (eval $ac_try) 2>&5 +@@ -24915,7 +24915,7 @@ if test "${gcc_cv_as_ix86_fildq+set}" = set; then : + else + gcc_cv_as_ix86_fildq=no + if test x$gcc_cv_as != x; then +- $as_echo 'fildq mem; fistpq mem' > conftest.s ++ $as_echo 'fildq (%ebp); fistpq (%ebp)' > conftest.s + if { ac_try='$gcc_cv_as $gcc_cv_as_flags -o conftest.o conftest.s >&5' + { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5 + (eval $ac_try) 2>&5 +diff --git a/gcc/configure.ac b/gcc/configure.ac +index 68b0ee8..bd53978 100644 +--- a/gcc/configure.ac ++++ b/gcc/configure.ac +@@ -3869,13 +3869,13 @@ foo: nop + + gcc_GAS_CHECK_FEATURE([filds and fists mnemonics], + gcc_cv_as_ix86_filds,,, +- [filds mem; fists mem],, ++ [filds (%ebp); fists (%ebp)],, + [AC_DEFINE(HAVE_AS_IX86_FILDS, 1, + [Define if your assembler uses filds and fists mnemonics.])]) + + gcc_GAS_CHECK_FEATURE([fildq and fistpq mnemonics], + gcc_cv_as_ix86_fildq,,, +- [fildq mem; fistpq mem],, ++ [fildq (%ebp); fistpq (%ebp)],, + [AC_DEFINE(HAVE_AS_IX86_FILDQ, 1, + [Define if your assembler uses fildq and fistq mnemonics.])]) + diff --git a/var/spack/repos/builtin/packages/gcc/darwin/gcc-4.9.patch2 b/var/spack/repos/builtin/packages/gcc/darwin/gcc-4.9.patch2 new file mode 100644 index 00000000000..b065997f453 --- /dev/null +++ b/var/spack/repos/builtin/packages/gcc/darwin/gcc-4.9.patch2 @@ -0,0 +1,28 @@ +From 82f81877458ea372176eabb5de36329431dce99b Mon Sep 17 00:00:00 2001 +From: Iain Sandoe +Date: Sat, 21 Dec 2013 00:30:18 +0000 +Subject: [PATCH] don't try to mark local symbols as no-dead-strip + +--- + gcc/config/darwin.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/gcc/config/darwin.c b/gcc/config/darwin.c +index 40804b8..0080299 100644 +--- a/gcc/config/darwin.c ++++ b/gcc/config/darwin.c +@@ -1259,6 +1259,11 @@ darwin_encode_section_info (tree decl, rtx rtl, int first ATTRIBUTE_UNUSED) + void + darwin_mark_decl_preserved (const char *name) + { ++ /* Actually we shouldn't mark any local symbol this way, but for now ++ this only happens with ObjC meta-data. */ ++ if (darwin_label_is_anonymous_local_objc_name (name)) ++ return; ++ + fprintf (asm_out_file, "\t.no_dead_strip "); + assemble_name (asm_out_file, name); + fputc ('\n', asm_out_file); +-- +2.2.1 + diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 232d0020df4..6043b622793 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -27,6 +27,7 @@ from contextlib import closing from glob import glob import sys +import os class Gcc(Package): """The GNU Compiler Collection includes front ends for C, C++, @@ -63,6 +64,9 @@ class Gcc(Package): # TODO: integrate these libraries. #depends_on("ppl") #depends_on("cloog") + if sys.platform == 'darwin': + patch('darwin/gcc-4.9.patch1', when='@4.9.3') + patch('darwin/gcc-4.9.patch2', when='@4.9.3') def install(self, spec, prefix): # libjava/configure needs a minor fix to install into spack paths. @@ -70,6 +74,7 @@ def install(self, spec, prefix): string=True) enabled_languages = set(('c', 'c++', 'fortran', 'java', 'objc')) + if spec.satisfies("@4.7.1:") and sys.platform != 'darwin': enabled_languages.add('go') @@ -101,12 +106,17 @@ def install(self, spec, prefix): isl_options = ["--with-isl=%s" % spec['isl'].prefix] options.extend(isl_options) + if sys.platform == 'darwin' : + darwin_options = [ "--with-build-config=bootstrap-debug" ] + options.extend(darwin_options) + build_dir = join_path(self.stage.path, 'spack-build') configure = Executable( join_path(self.stage.source_path, 'configure') ) with working_dir(build_dir, create=True): # Rest of install is straightforward. configure(*options) - make() + if sys.platform == 'darwin' : make("bootstrap") + else: make() make("install") self.write_rpath_specs() From dfc5cf288c1017ef62390489cb534848aef5a9a6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 29 Mar 2016 04:35:41 -0700 Subject: [PATCH 771/910] Fix bug in restage - Restage previously only removed the source directory from the stage. - Now removes any other directories in stage as well. --- lib/spack/spack/fetch_strategy.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 0d0a7db8a91..4ea87bea7ea 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -289,8 +289,14 @@ def reset(self): if not self.archive_file: raise NoArchiveFileError("Tried to reset URLFetchStrategy before fetching", "Failed on reset() for URL %s" % self.url) - if self.stage.source_path: - shutil.rmtree(self.stage.source_path, ignore_errors=True) + + # Remove everythigng but the archive from the stage + for filename in os.listdir(self.stage.path): + abspath = os.path.join(self.stage.path, filename) + if abspath != self.archive_file: + shutil.rmtree(abspath, ignore_errors=True) + + # Expand the archive again self.expand() def __repr__(self): From 653d7c52574b7a6e3795fb3beeeee553bd76b293 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 29 Mar 2016 17:28:46 +0200 Subject: [PATCH 772/910] uninstall : minor fixes --- lib/spack/spack/cmd/uninstall.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 7183817908d..72ad1321129 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -40,6 +40,7 @@ b) use spack uninstall -a to uninstall ALL matching specs. """ + def ask_for_confirmation(message): while True: tty.msg(message + '[y/n]') @@ -122,6 +123,7 @@ def installed_dependents(specs): for item in specs: lst = [x for x in item.package.installed_dependents if x not in specs] if lst: + lst = list(set(lst)) dependents[item] = lst return dependents From ad402ff85bc4150aa49fb394929b062b40b27604 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 21 Mar 2016 22:36:31 +0100 Subject: [PATCH 773/910] astyle@2.04: add new package --- .../repos/builtin/packages/astyle/package.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 var/spack/repos/builtin/packages/astyle/package.py diff --git a/var/spack/repos/builtin/packages/astyle/package.py b/var/spack/repos/builtin/packages/astyle/package.py new file mode 100644 index 00000000000..7260fd74a1d --- /dev/null +++ b/var/spack/repos/builtin/packages/astyle/package.py @@ -0,0 +1,17 @@ +from spack import * +import os + +class Astyle(Package): + """A Free, Fast, and Small Automatic Formatter for C, C++, C++/CLI, Objective-C, C#, and Java Source Code.""" + homepage = "http://astyle.sourceforge.net/" + url = "http://downloads.sourceforge.net/project/astyle/astyle/astyle%202.04/astyle_2.04_linux.tar.gz" + + version('2.04', '30b1193a758b0909d06e7ee8dd9627f6') + + def install(self, spec, prefix): + + with working_dir('src'): + make('-f', + join_path(self.stage.source_path,'build','clang','Makefile'), + parallel=False) + install(join_path(self.stage.source_path, 'src','bin','astyle'), self.prefix.bin) From d96ea5ba0075c63e8cd042337a1d99225b92b2d5 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Tue, 29 Mar 2016 12:20:11 -0500 Subject: [PATCH 774/910] add modules package and have setup script find it if built --- share/spack/setup-env.sh | 10 ++++++++ .../repos/builtin/packages/modules/package.py | 25 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 var/spack/repos/builtin/packages/modules/package.py diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 764af684001..529517ff834 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -176,3 +176,13 @@ _spack_pathadd PATH "${_sp_prefix%/}/bin" _sp_sys_type=$(spack-python -c 'print(spack.architecture.sys_type())') _spack_pathadd DK_NODE "${_sp_share_dir%/}/dotkit/$_sp_sys_type" _spack_pathadd MODULEPATH "${_sp_share_dir%/}/modules/$_sp_sys_type" + +# +# Use spack built modules package if available +# for system that does not have it +# + +_modules_install_path=$(command spack location -i modules) +if [ -n "$_modules_install_path" ] ; then + . $_modules_install_path/Modules/init/sh +fi diff --git a/var/spack/repos/builtin/packages/modules/package.py b/var/spack/repos/builtin/packages/modules/package.py new file mode 100644 index 00000000000..b014ee460c2 --- /dev/null +++ b/var/spack/repos/builtin/packages/modules/package.py @@ -0,0 +1,25 @@ +from spack import * + +class Modules(Package): + """ The Environment Modules package provides for the dynamic modification of a user's environment via modulefiles. """ + + homepage = "http://modules.sf.net" + url = "http://downloads.sourceforge.net/project/modules/Modules/modules-3.2.10/modules-3.2.10.tar.gz" + + version('3.2.10', '8b097fdcb90c514d7540bb55a3cb90fb') + + depends_on("tcl") + + def install(self, spec, prefix): + + options = ['--prefix=%s' % prefix, + '--disable-debug', + '--disable-dependency-tracking', + '--disable-silent-rules', + '--disable-versioning', + '--datarootdir=%s' % prefix.share, + 'CPPFLAGS=-DUSE_INTERP_ERRORLINE'] + + configure(*options) + make() + make("install") From 4ddba5f7edd97c8c17781368edc5b9a0db74417d Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 29 Mar 2016 14:47:16 -0500 Subject: [PATCH 775/910] Add CUDA and HOOMD-blue packages --- .../repos/builtin/packages/cuda/package.py | 42 ++++++++++++ .../builtin/packages/hoomd-blue/package.py | 67 +++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 var/spack/repos/builtin/packages/cuda/package.py create mode 100644 var/spack/repos/builtin/packages/hoomd-blue/package.py diff --git a/var/spack/repos/builtin/packages/cuda/package.py b/var/spack/repos/builtin/packages/cuda/package.py new file mode 100644 index 00000000000..547b43d6d26 --- /dev/null +++ b/var/spack/repos/builtin/packages/cuda/package.py @@ -0,0 +1,42 @@ +from spack import * +from glob import glob +import os + +class Cuda(Package): + """CUDA is a parallel computing platform and programming model invented by + NVIDIA. It enables dramatic increases in computing performance by harnessing + the power of the graphics processing unit (GPU). + + Note: NVIDIA does not provide a download URL for CUDA so you will need to + download it yourself. Go to https://developer.nvidia.com/cuda-downloads + and select your Operating System, Architecture, Distribution, and Version. + For the Installer Type, select runfile and click Download. Spack will search + your current directory for this file. Alternatively, add this file to a + mirror so that Spack can find it. For instructions on how to set up a mirror, + see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "http://www.nvidia.com/object/cuda_home_new.html" + url = "file://%s/cuda_7.5.18_linux.run" % os.getcwd() + + version('7.5.18', '4b3bcecf0dfc35928a0898793cf3e4c6', expand=False) + + def install(self, spec, prefix): + runfile = glob(os.path.join(self.stage.path, 'cuda*.run'))[0] + chmod = which('chmod') + chmod('+x', runfile) + runfile = which(runfile) + + # Note: NVIDIA does not officially support many newer versions of compilers. + # For example, on CentOS 6, you must use GCC 4.4.7 or older. + # The --override flag disables these checks. See: + # http://docs.nvidia.com/cuda/cuda-installation-guide-linux/#system-requirements + # for details. + + runfile( + '--silent', # disable interactive prompts + '--verbose', # create verbose log file + '--override', # ignore compiler checks + '--toolkit', # install CUDA Toolkit + '--toolkitpath=%s' % prefix + ) + diff --git a/var/spack/repos/builtin/packages/hoomd-blue/package.py b/var/spack/repos/builtin/packages/hoomd-blue/package.py new file mode 100644 index 00000000000..97ce9c0cf0f --- /dev/null +++ b/var/spack/repos/builtin/packages/hoomd-blue/package.py @@ -0,0 +1,67 @@ +from spack import * +import os + +class HoomdBlue(Package): + """HOOMD-blue is a general-purpose particle simulation toolkit. It scales + from a single CPU core to thousands of GPUs. + + You define particle initial conditions and interactions in a high-level + python script. Then tell HOOMD-blue how you want to execute the job and it + takes care of the rest. Python job scripts give you unlimited flexibility + to create custom initialization routines, control simulation parameters, + and perform in situ analysis.""" + + homepage = "https://codeblue.umich.edu/hoomd-blue/index.html" + url = "https://bitbucket.org/glotzer/hoomd-blue/get/v1.3.3.tar.bz2" + + version('1.3.3', '1469ef4531dc14b579c0acddbfe6a273') + + variant('mpi', default=False, description='Compile with MPI enabled') + variant('cuda', default=False, description='Compile with CUDA Toolkit') + variant('doc', default=False, description='Generate documentation') + + extends('python') + depends_on('py-numpy') + depends_on('boost+python') + depends_on('cmake') + depends_on('mpi', when='+mpi') + depends_on('cuda', when='+cuda') + depends_on('doxygen', when='+doc') + + def install(self, spec, prefix): + + cmake_args = [ + '-DPYTHON_EXECUTABLE=%s/python' % spec['python'].prefix.bin, + '-DBOOST_ROOT=%s' % spec['boost' ].prefix + ] + + # MPI support + if '+mpi' in spec: + os.environ['MPI_HOME'] = spec['mpi'].prefix + cmake_args.append('-DENABLE_MPI=ON') + else: + cmake_args.append('-DENABLE_MPI=OFF') + + # CUDA support + if '+cuda' in spec: + cmake_args.append('-DENABLE_CUDA=ON') + else: + cmake_args.append('-DENABLE_CUDA=OFF') + + # CUDA-aware MPI library support + if '+cuda' in spec and '+mpi' in spec: + cmake_args.append('-DENABLE_MPI_CUDA=ON') + else: + cmake_args.append('-DENABLE_MPI_CUDA=OFF') + + # Documentation + if '+doc' in spec: + cmake_args.append('-DENABLE_DOXYGEN=ON') + else: + cmake_args.append('-DENABLE_DOXYGEN=OFF') + + cmake_args.extend(std_cmake_args) + cmake('.', *cmake_args) + + make() + make("install") From abee9c83b440b8341be6df1f07c7c5f1bef9dd89 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Tue, 29 Mar 2016 17:26:34 -0400 Subject: [PATCH 776/910] Removed googletest FIXME... --- .../repos/builtin/packages/googletest/package.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/var/spack/repos/builtin/packages/googletest/package.py b/var/spack/repos/builtin/packages/googletest/package.py index ed27afa10ce..663b758747b 100644 --- a/var/spack/repos/builtin/packages/googletest/package.py +++ b/var/spack/repos/builtin/packages/googletest/package.py @@ -1,19 +1,3 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install googletest -# -# You can always get back here to change things with: -# -# spack edit googletest -# -# See the spack documentation for more information on building -# packages. -# from spack import * class Googletest(Package): From a4eff6de3faa3624a529015ea4cf5ebb1af41484 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Tue, 29 Mar 2016 17:10:49 -0600 Subject: [PATCH 777/910] + New version of cmake (3.5.1). --- var/spack/repos/builtin/packages/cmake/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index 1f93d39769b..91a4e3b415f 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -30,6 +30,7 @@ class Cmake(Package): homepage = 'https://www.cmake.org' url = 'https://cmake.org/files/v3.4/cmake-3.4.3.tar.gz' + version('3.5.1', 'ca051f4a66375c89d1a524e726da0296') version('3.5.0', '33c5d09d4c33d4ffcc63578a6ba8777e') version('3.4.3', '4cb3ff35b2472aae70f542116d616e63') version('3.4.0', 'cd3034e0a44256a0917e254167217fc8') From 98bb151d85fc35389e0b76eaac90448fa688aa6c Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Tue, 29 Mar 2016 20:25:56 -0700 Subject: [PATCH 778/910] updated geos version --- var/spack/repos/builtin/packages/dbus/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/dbus/package.py b/var/spack/repos/builtin/packages/dbus/package.py index 294b0de54ea..422f5a19eba 100644 --- a/var/spack/repos/builtin/packages/dbus/package.py +++ b/var/spack/repos/builtin/packages/dbus/package.py @@ -13,6 +13,7 @@ class Dbus(Package): homepage = "http://dbus.freedesktop.org/" url = "http://dbus.freedesktop.org/releases/dbus/dbus-1.8.8.tar.gz" + version('1.11.2', '957a07f066f3730d2bb3ea0932f0081b') version('1.9.0', 'ec6895a4d5c0637b01f0d0e7689e2b36') version('1.8.8', 'b9f4a18ee3faa1e07c04aa1d83239c43') version('1.8.6', '6a08ba555d340e9dfe2d623b83c0eea8') From 5b6eefbf0417668c19e4c898c31cf56764d5bc2b Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 30 Mar 2016 11:30:35 +0200 Subject: [PATCH 779/910] add variants and extra dependencies --- .../repos/builtin/packages/dealii/package.py | 178 ++++++++++++++---- 1 file changed, 141 insertions(+), 37 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 9ac1572b95a..396601356cf 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -8,27 +8,48 @@ class Dealii(Package): version('8.4.0', 'ac5dbf676096ff61e092ce98c80c2b00') - depends_on ("cmake") - depends_on ("blas") - depends_on ("lapack") - depends_on ("mpi") + variant('mpi', default=True, description='Compile with MPI') + variant('arpack', default=True, description='Compile with Arpack and PArpack (only with MPI)') + variant('doxygen', default=True, description='Compile with Doxygen documentation') + variant('hdf5', default=True, description='Compile with HDF5 (only with MPI)') + variant('metis', default=True, description='Compile with Metis or ParMetis') + variant('netcdf', default=True, description='Compile with Netcdf (only with MPI)') + variant('oce', default=True, description='Compile with OCE') + variant('p4est', default=True, description='Compile with P4est (only with MPI)') + variant('petsc', default=True, description='Compile with Petsc (only with MPI)') + variant('slepc', default=True, description='Compile with Slepc (only with Petsc and MPI)') + variant('trilinos', default=True, description='Compile with Trilinos (only with MPI)') - depends_on ("arpack-ng+mpi") - depends_on ("boost") - depends_on ("doxygen") - depends_on ("hdf5+mpi~cxx") #FIXME NetCDF declares dependency with ~cxx, why? - depends_on ("metis") + # required dependencies, light version + depends_on ("blas") + depends_on ("boost", when='~mpi') + depends_on ("boost+mpi", when='+mpi') + depends_on ("bzip2") + depends_on ("cmake") + depends_on ("lapack") depends_on ("muparser") - depends_on ("netcdf-cxx") - #depends_on ("numdiff") #FIXME - depends_on ("oce") - depends_on ("p4est") - depends_on ("parmetis") - depends_on ("petsc+mpi") - depends_on ("slepc") depends_on ("suite-sparse") depends_on ("tbb") - depends_on ("trilinos") + depends_on ("zlib") + + # optional dependencies + depends_on ("mpi", when="+mpi") + depends_on ("arpack-ng+mpi", when='+arpack+mpi') + depends_on ("doxygen", when='+doxygen') + depends_on ("hdf5+mpi~cxx", when='+hdf5+mpi') #FIXME NetCDF declares dependency with ~cxx, why? + depends_on ("metis", when='+metis') + depends_on ("netcdf+mpi", when="+netcdf+mpi") + depends_on ("netcdf-cxx", when='+netcdf+mpi') + depends_on ("oce", when='+oce') + depends_on ("p4est", when='+p4est+mpi') + depends_on ("parmetis", when='+metis+mpi') + depends_on ("petsc+mpi", when='+petsc+mpi') + depends_on ("slepc", when='+slepc+petsc+mpi') + depends_on ("trilinos", when='+trilinos+mpi') + + # developer dependnecies + #depends_on ("numdiff") #FIXME + #depends_on ("astyle") #FIXME def install(self, spec, prefix): options = [] @@ -42,33 +63,116 @@ def install(self, spec, prefix): dsuf = 'dylib' if sys.platform == 'darwin' else 'so' options.extend([ '-DCMAKE_BUILD_TYPE=DebugRelease', - '-DDEAL_II_WITH_THREADS:BOOL=ON' - '-DDEAL_II_WITH_MPI:BOOL=ON', - '-DCMAKE_C_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), # FIXME: avoid hardcoding mpi wrappers names - '-DCMAKE_CXX_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpic++'), - '-DCMAKE_Fortran_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), - '-DARPACK_DIR=%s' % spec['arpack-ng'].prefix, + '-DDEAL_II_COMPONENT_EXAMPLES=ON', + '-DDEAL_II_WITH_THREADS:BOOL=ON', '-DBOOST_DIR=%s' % spec['boost'].prefix, - '-DHDF5_DIR=%s' % spec['hdf5'].prefix, - '-DMETIS_DIR=%s' % spec['metis'].prefix, + '-DBZIP2_DIR=%s' % spec['bzip2'].prefix, + # CMake's FindBlas/Lapack may pickup system's blas/lapack instead of Spack's. + # Be more specific to avoid this. + # Note that both lapack and blas are provided in -DLAPACK_XYZ variables + '-DLAPACK_FOUND=true', + '-DLAPACK_INCLUDE_DIRS=%s;%s' % + (spec['lapack'].prefix.include, + spec['blas'].prefix.include), + '-DLAPACK_LIBRARIES=%s;%s' % + (join_path(spec['lapack'].prefix.lib,'liblapack.%s' % dsuf), # FIXME don't hardcode names + join_path(spec['blas'].prefix.lib,'libblas.%s' % dsuf)), # FIXME don't hardcode names '-DMUPARSER_DIR=%s ' % spec['muparser'].prefix, - # since Netcdf is spread among two, need to do it by hand: - '-DNETCDF_FOUND=true', - '-DNETCDF_LIBRARIES=%s;%s' % - (join_path(spec['netcdf-cxx'].prefix.lib,'libnetcdf_c++.%s' % dsuf), - join_path(spec['netcdf'].prefix.lib,'libnetcdf.%s' % dsuf)), - '-DNETCDF_INCLUDE_DIRS=%s;%s' % - (spec['netcdf-cxx'].prefix.include, - spec['netcdf'].prefix.include), - '-DOPENCASCADE_DIR=%s' % spec['oce'].prefix, '-DP4EST_DIR=%s' % spec['p4est'].prefix, - '-DPETSC_DIR=%s' % spec['petsc'].prefix, - '-DSLEPC_DIR=%s' % spec['slepc'].prefix, '-DUMFPACK_DIR=%s' % spec['suite-sparse'].prefix, '-DTBB_DIR=%s' % spec['tbb'].prefix, - '-DTRILINOS_DIR=%s' % spec['trilinos'].prefix + '-DZLIB_DIR=%s' % spec['zlib'].prefix ]) + # MPI + if '+mpi' in spec: + options.extend([ + '-DDEAL_II_WITH_MPI:BOOL=ON', + '-DCMAKE_C_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), # FIXME: avoid hardcoding mpi wrappers names + '-DCMAKE_CXX_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpic++'), + '-DCMAKE_Fortran_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + ]) + else: + options.extend([ + '-DDEAL_II_WITH_MPI:BOOL=OFF', + ]) + + # Optional dependencies for which librariy names are the same as CMake variables + for library in ('hdf5', 'p4est','petsc', 'slepc','trilinos','metis','parmetis'): + if library in spec: + options.extend([ + '-D{library}_DIR={value}'.format(library=library.upper(), value=spec[library].prefix), + '-DDEAL_II_WITH_{library}:BOOL=ON'.format(library=library.upper()) + ]) + else: + options.extend([ + '-DDEAL_II_WITH_{library}:BOOL=OFF'.format(library=library.upper()) + ]) + + # doxygen + options.extend([ + '-DDEAL_II_COMPONENT_DOCUMENTATION=%s' % ('ON' if '+doxygen' in spec else 'OFF'), + ]) + + + # arpack + if '+arpack' in spec: + options.extend([ + '-DARPACK_DIR=%s' % spec['arpack-ng'].prefix, + '-DDEAL_II_WITH_ARPACK=ON', + '-DDEAL_II_ARPACK_WITH_PARPACK=ON' + ]) + else: + options.extend([ + '-DDEAL_II_WITH_ARPACK=OFF' + ]) + + # # metis + # if '+metis' in spec: + # + # if 'mpi' in spec: + # options.extend([ + # '-DMETIS_DIR=%s' % spec['parmetis'].prefix, + # ]) + # else: + # options.extend([ + # '-DMETIS_DIR=%s' % spec['metis'].prefix, + # ]) + # options.extend([ + # '-DDEAL_II_WITH_METIS=ON' + # ]) + # else: + # options.extend([ + # '-DDEAL_II_WITH_METIS=OFF' + # ]) + + # since Netcdf is spread among two, need to do it by hand: + if '+netcdf' in spec: + options.extend([ + '-DNETCDF_FOUND=true', + '-DNETCDF_LIBRARIES=%s;%s' % + (join_path(spec['netcdf-cxx'].prefix.lib,'libnetcdf_c++.%s' % dsuf), + join_path(spec['netcdf'].prefix.lib,'libnetcdf.%s' % dsuf)), + '-DNETCDF_INCLUDE_DIRS=%s;%s' % + (spec['netcdf-cxx'].prefix.include, + spec['netcdf'].prefix.include), + ]) + else: + options.extend([ + '-DDEAL_II_WITH_NETCDF=OFF' + ]) + + # Open Cascade + if '+oce' in spec: + options.extend([ + '-DOPENCASCADE_DIR=%s' % spec['oce'].prefix, + '-DDEAL_II_WITH_OPENCASCADE=ON' + ]) + else: + options.extend([ + '-DDEAL_II_WITH_OPENCASCADE=OFF' + ]) + cmake('.', *options) make() From bafc6cebe1ca37116badc1baf60e88564eb01614 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 30 Mar 2016 12:00:21 +0200 Subject: [PATCH 780/910] doxygen off by default --- .../repos/builtin/packages/dealii/package.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 396601356cf..907aa333567 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -8,17 +8,17 @@ class Dealii(Package): version('8.4.0', 'ac5dbf676096ff61e092ce98c80c2b00') - variant('mpi', default=True, description='Compile with MPI') - variant('arpack', default=True, description='Compile with Arpack and PArpack (only with MPI)') - variant('doxygen', default=True, description='Compile with Doxygen documentation') - variant('hdf5', default=True, description='Compile with HDF5 (only with MPI)') - variant('metis', default=True, description='Compile with Metis or ParMetis') - variant('netcdf', default=True, description='Compile with Netcdf (only with MPI)') - variant('oce', default=True, description='Compile with OCE') - variant('p4est', default=True, description='Compile with P4est (only with MPI)') - variant('petsc', default=True, description='Compile with Petsc (only with MPI)') - variant('slepc', default=True, description='Compile with Slepc (only with Petsc and MPI)') - variant('trilinos', default=True, description='Compile with Trilinos (only with MPI)') + variant('mpi', default=True, description='Compile with MPI') + variant('arpack', default=True, description='Compile with Arpack and PArpack (only with MPI)') + variant('doxygen', default=False, description='Compile with Doxygen documentation') + variant('hdf5', default=True, description='Compile with HDF5 (only with MPI)') + variant('metis', default=True, description='Compile with Metis or ParMetis') + variant('netcdf', default=True, description='Compile with Netcdf (only with MPI)') + variant('oce', default=True, description='Compile with OCE') + variant('p4est', default=True, description='Compile with P4est (only with MPI)') + variant('petsc', default=True, description='Compile with Petsc (only with MPI)') + variant('slepc', default=True, description='Compile with Slepc (only with Petsc and MPI)') + variant('trilinos', default=True, description='Compile with Trilinos (only with MPI)') # required dependencies, light version depends_on ("blas") From 9e9f5d01ad43e9b00262a12d1fd1225ac368581d Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 30 Mar 2016 11:59:13 +0200 Subject: [PATCH 781/910] doxygen: add missing dependencies --- var/spack/repos/builtin/packages/doxygen/package.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/var/spack/repos/builtin/packages/doxygen/package.py b/var/spack/repos/builtin/packages/doxygen/package.py index 3d4a4e47a78..3a1deba9e14 100644 --- a/var/spack/repos/builtin/packages/doxygen/package.py +++ b/var/spack/repos/builtin/packages/doxygen/package.py @@ -4,6 +4,7 @@ #------------------------------------------------------------------------------ from spack import * +import sys class Doxygen(Package): """Doxygen is the de facto standard tool for generating documentation @@ -17,6 +18,10 @@ class Doxygen(Package): version('1.8.10', '79767ccd986f12a0f949015efb5f058f') depends_on("cmake@2.8.12:") + # flex does not build on OSX, but it's provided there anyway + depends_on("flex", sys.platform != 'darwin') + depends_on("bison", sys.platform != 'darwin') + def install(self, spec, prefix): cmake('.', *std_cmake_args) From 2abdd71b414b4f551e927edb7304b4dcc053d09e Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 30 Mar 2016 15:19:29 +0200 Subject: [PATCH 782/910] add dev version and conditionals for tests; remove parmetis --- .../repos/builtin/packages/dealii/package.py | 62 +++++++------------ 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 907aa333567..32dfb33c6d0 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -7,12 +7,13 @@ class Dealii(Package): url = "https://github.com/dealii/dealii/releases/download/v8.4.0/dealii-8.4.0.tar.gz" version('8.4.0', 'ac5dbf676096ff61e092ce98c80c2b00') + version('dev', git='https://github.com/dealii/dealii.git') variant('mpi', default=True, description='Compile with MPI') variant('arpack', default=True, description='Compile with Arpack and PArpack (only with MPI)') variant('doxygen', default=False, description='Compile with Doxygen documentation') variant('hdf5', default=True, description='Compile with HDF5 (only with MPI)') - variant('metis', default=True, description='Compile with Metis or ParMetis') + variant('metis', default=True, description='Compile with Metis') variant('netcdf', default=True, description='Compile with Netcdf (only with MPI)') variant('oce', default=True, description='Compile with OCE') variant('p4est', default=True, description='Compile with P4est (only with MPI)') @@ -42,7 +43,6 @@ class Dealii(Package): depends_on ("netcdf-cxx", when='+netcdf+mpi') depends_on ("oce", when='+oce') depends_on ("p4est", when='+p4est+mpi') - depends_on ("parmetis", when='+metis+mpi') depends_on ("petsc+mpi", when='+petsc+mpi') depends_on ("slepc", when='+slepc+petsc+mpi') depends_on ("trilinos", when='+trilinos+mpi') @@ -98,7 +98,7 @@ def install(self, spec, prefix): ]) # Optional dependencies for which librariy names are the same as CMake variables - for library in ('hdf5', 'p4est','petsc', 'slepc','trilinos','metis','parmetis'): + for library in ('hdf5', 'p4est','petsc', 'slepc','trilinos','metis'): if library in spec: options.extend([ '-D{library}_DIR={value}'.format(library=library.upper(), value=spec[library].prefix), @@ -127,25 +127,6 @@ def install(self, spec, prefix): '-DDEAL_II_WITH_ARPACK=OFF' ]) - # # metis - # if '+metis' in spec: - # - # if 'mpi' in spec: - # options.extend([ - # '-DMETIS_DIR=%s' % spec['parmetis'].prefix, - # ]) - # else: - # options.extend([ - # '-DMETIS_DIR=%s' % spec['metis'].prefix, - # ]) - # options.extend([ - # '-DDEAL_II_WITH_METIS=ON' - # ]) - # else: - # options.extend([ - # '-DDEAL_II_WITH_METIS=OFF' - # ]) - # since Netcdf is spread among two, need to do it by hand: if '+netcdf' in spec: options.extend([ @@ -202,16 +183,18 @@ def install(self, spec, prefix): # list the number of cycles to speed up filter_file(r'(const unsigned int n_cycles = 8;)', ('const unsigned int n_cycles = 2;'), 'step-40.cc') cmake('.') - make('release') - make('run',parallel=False) + if '^petsc' in spec: + make('release') + make('run',parallel=False) print('=====================================') print('========= Step-40 Trilinos ==========') print('=====================================') # change Linear Algebra to Trilinos filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') - make('release') - make('run',parallel=False) + if '^trilinos+hypre' in spec: + make('release') + make('run',parallel=False) print('=====================================') print('=== Step-40 Trilinos SuperluDist ====') @@ -223,31 +206,34 @@ def install(self, spec, prefix): filter_file(r'(preconditioner.initialize\(system_matrix, data\);)', (''), 'step-40.cc') filter_file(r'(solver\.solve \(system_matrix, completely_distributed_solution, system_rhs,)', ('solver.solve (system_matrix, completely_distributed_solution, system_rhs);'), 'step-40.cc') filter_file(r'(preconditioner\);)', (''), 'step-40.cc') - - make('release') - make('run',paralle=False) + if '^trilinos+superlu-dist' in spec: + make('release') + make('run',paralle=False) print('=====================================') print('====== Step-40 Trilinos MUMPS =======') print('=====================================') # switch to Mumps filter_file(r'(Amesos_Superludist)', ('Amesos_Mumps'), 'step-40.cc') - make('release') - make('run',parallel=False) + if '^trilinos+mumps' in spec: + make('release') + make('run',parallel=False) print('=====================================') print('============ Step-36 ================') print('=====================================') with working_dir('examples/step-36'): - cmake('.') - make('release') - make('run',parallel=False) + if 'slepc' in spec: + cmake('.') + make('release') + make('run',parallel=False) print('=====================================') print('============ Step-54 ================') print('=====================================') with working_dir('examples/step-54'): - cmake('.') - make('release') - # FIXME - # make('run',parallel=False) + if 'oce' in spec: + cmake('.') + make('release') + if sys.platform != 'darwin': #FIXME + make('run',parallel=False) From dd1c667d8d5d260146f81b0b90ad52c41160208e Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 30 Mar 2016 16:10:20 +0200 Subject: [PATCH 783/910] uninstall : changed error message --- lib/spack/spack/cmd/uninstall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 72ad1321129..7aadd254e85 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -182,7 +182,7 @@ def uninstall(parser, args): uninstall_list = list(set(uninstall_list)) if has_error: - tty.die('You can use spack uninstall -f to force this action') + tty.die('You can use spack uninstall -r to uninstall these dependencies as well') if not args.yes_to_all: tty.msg("The following packages will be uninstalled : ") From 5b1ef64e31a56877f6859820ca7a79dd51f6fe84 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Wed, 30 Mar 2016 09:33:00 -0600 Subject: [PATCH 784/910] + Provide basic instructions for providing subversion w/o additional language bindings. --- .../apr-util/{apr-util.new => }/package.py | 0 .../packages/apr/{apr.new => }/package.py | 13 ---------- .../{subversion.new => }/package.py | 24 ++++++++++++++++--- 3 files changed, 21 insertions(+), 16 deletions(-) rename var/spack/repos/builtin/packages/apr-util/{apr-util.new => }/package.py (100%) rename var/spack/repos/builtin/packages/apr/{apr.new => }/package.py (71%) rename var/spack/repos/builtin/packages/subversion/{subversion.new => }/package.py (78%) diff --git a/var/spack/repos/builtin/packages/apr-util/apr-util.new/package.py b/var/spack/repos/builtin/packages/apr-util/package.py similarity index 100% rename from var/spack/repos/builtin/packages/apr-util/apr-util.new/package.py rename to var/spack/repos/builtin/packages/apr-util/package.py diff --git a/var/spack/repos/builtin/packages/apr/apr.new/package.py b/var/spack/repos/builtin/packages/apr/package.py similarity index 71% rename from var/spack/repos/builtin/packages/apr/apr.new/package.py rename to var/spack/repos/builtin/packages/apr/package.py index 5cbbad350bd..8a440766ece 100644 --- a/var/spack/repos/builtin/packages/apr/apr.new/package.py +++ b/var/spack/repos/builtin/packages/apr/package.py @@ -31,19 +31,6 @@ class Apr(Package): version('1.5.2', '98492e965963f852ab29f9e61b2ad700') - #variant('ncurses', default=True, description='Enables the build of the ncurses gui') - #variant('qt', default=False, description='Enables the build of cmake-gui') - #variant('doc', default=False, description='Enables the generation of html and man page documentation') - - #depends_on('ncurses', when='+ncurses') - #depends_on('qt', when='+qt') - #depends_on('python@2.7.11:', when='+doc') - #depends_on('py-sphinx', when='+doc') - - #def url_for_version(self, version): - # """Handle CMake's version-based custom URLs.""" - # return 'https://cmake.org/files/v%s/cmake-%s.tar.gz' % (version.up_to(2), version) - def install(self, spec, prefix): options = ['--prefix=%s' % prefix] configure(*options) diff --git a/var/spack/repos/builtin/packages/subversion/subversion.new/package.py b/var/spack/repos/builtin/packages/subversion/package.py similarity index 78% rename from var/spack/repos/builtin/packages/subversion/subversion.new/package.py rename to var/spack/repos/builtin/packages/subversion/package.py index b1c3380238a..5db1c3eb92d 100644 --- a/var/spack/repos/builtin/packages/subversion/subversion.new/package.py +++ b/var/spack/repos/builtin/packages/subversion/package.py @@ -37,7 +37,13 @@ class Subversion(Package): depends_on('apr-util') depends_on('zlib') depends_on('sqlite') + + # Optional: We need swig if we want the Perl, Python or Ruby + # bindings. #depends_on('swig') + #depends_on('python') + #depends_on('perl') + #depends_on('ruby') def install(self, spec, prefix): @@ -53,7 +59,19 @@ def install(self, spec, prefix): configure(*options) make() make('install') - #make('swig-py') # python bindings - #make('install-swig-py') - #make('swig-pl') # perl bindings + + # python bindings + #make('swig-py', + # 'swig-pydir=/usr/lib/python2.7/site-packages/libsvn', + # 'swig_pydir_extra=/usr/lib/python2.7/site-packages/svn') + #make('install-swig-py', + # 'swig-pydir=/usr/lib/python2.7/site-packages/libsvn', + # 'swig_pydir_extra=/usr/lib/python2.7/site-packages/svn') + + # perl bindings + #make('swig-pl') #make('install-swig-pl') + + # ruby bindings + #make('swig-rb') + #make('isntall-swig-rb') From a7ceacb991b2a3b0caba8da41a39e045912f9e22 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Wed, 30 Mar 2016 09:47:31 -0600 Subject: [PATCH 785/910] + Changes to swig are not needed with this PR. --- var/spack/repos/builtin/packages/swig/package.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/swig/package.py b/var/spack/repos/builtin/packages/swig/package.py index de6055e9656..8d46c4fe465 100644 --- a/var/spack/repos/builtin/packages/swig/package.py +++ b/var/spack/repos/builtin/packages/swig/package.py @@ -36,14 +36,11 @@ class Swig(Package): homepage = "http://www.swig.org" url = "http://prdownloads.sourceforge.net/swig/swig-3.0.2.tar.gz" - version('3.0.8', 'c96a1d5ecb13d38604d7e92148c73c97') - version('3.0.2', '62f9b0d010cef36a13a010dc530d0d41') - version('2.0.12', 'c3fb0b2d710cc82ed0154b91e43085a4') + version('3.0.2', '62f9b0d010cef36a13a010dc530d0d41') depends_on('pcre') def install(self, spec, prefix): configure("--prefix=%s" % prefix) - make() make("install") From bc309e42d8f4d5c536ea43dc01a8356bf0aba3f2 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 30 Mar 2016 11:00:18 -0500 Subject: [PATCH 786/910] New package, py-bottleneck. --- .../builtin/packages/py-bottleneck/package.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-bottleneck/package.py diff --git a/var/spack/repos/builtin/packages/py-bottleneck/package.py b/var/spack/repos/builtin/packages/py-bottleneck/package.py new file mode 100644 index 00000000000..0aa4208b4de --- /dev/null +++ b/var/spack/repos/builtin/packages/py-bottleneck/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyBottleneck(Package): + """Bottleneck is a collection of fast NumPy array functions written in Cython.""" + homepage = "https://pypi.python.org/pypi/Bottleneck/1.0.0" + url = "https://pypi.python.org/packages/source/B/Bottleneck/Bottleneck-1.0.0.tar.gz" + + version('1.0.0', '380fa6f275bd24f27e7cf0e0d752f5d2') + + extends('python', ignore=r'bin/f2py$') + depends_on('py-numpy') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 5c95977dccef17a3cceb741d39f94aa075a0a9c4 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 30 Mar 2016 11:06:04 -0500 Subject: [PATCH 787/910] Added version 15.0.1 --- var/spack/repos/builtin/packages/py-virtualenv/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-virtualenv/package.py b/var/spack/repos/builtin/packages/py-virtualenv/package.py index 037a6fc59fe..09303d37b43 100644 --- a/var/spack/repos/builtin/packages/py-virtualenv/package.py +++ b/var/spack/repos/builtin/packages/py-virtualenv/package.py @@ -8,6 +8,7 @@ class PyVirtualenv(Package): version('1.11.6', 'f61cdd983d2c4e6aeabb70b1060d6f49') version('13.0.1', '1ffc011bde6667f0e37ecd976f4934db') + version('15.0.1', '28d76a0d9cbd5dc42046dd14e76a6ecc') extends('python') depends_on('py-setuptools') From 3b19cc303fae89eed11581d7cdfe2ab8e9d469ef Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 30 Mar 2016 11:08:28 -0500 Subject: [PATCH 788/910] Added version 1.10.0. --- var/spack/repos/builtin/packages/py-six/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-six/package.py b/var/spack/repos/builtin/packages/py-six/package.py index 05c5bd00a98..df277100f77 100644 --- a/var/spack/repos/builtin/packages/py-six/package.py +++ b/var/spack/repos/builtin/packages/py-six/package.py @@ -6,6 +6,7 @@ class PySix(Package): url = "https://pypi.python.org/packages/source/s/six/six-1.9.0.tar.gz" version('1.9.0', '476881ef4012262dfc8adc645ee786c4') + version('1.10.0', '34eed507548117b2ab523ab14b2f8b55') extends('python') depends_on('py-setuptools') From 9153e03e5ce5bdd5161aa4b493854208040fcf3d Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 30 Mar 2016 11:09:47 -0500 Subject: [PATCH 789/910] Added version 20.5. --- var/spack/repos/builtin/packages/py-setuptools/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-setuptools/package.py b/var/spack/repos/builtin/packages/py-setuptools/package.py index 26c048bfd42..c6d9be1add7 100644 --- a/var/spack/repos/builtin/packages/py-setuptools/package.py +++ b/var/spack/repos/builtin/packages/py-setuptools/package.py @@ -9,6 +9,7 @@ class PySetuptools(Package): version('16.0', '0ace0b96233516fc5f7c857d086aa3ad') version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06') version('19.2', '78353b1f80375ca5e088f4b4627ffe03') + version('20.5', 'fadc1e1123ddbe31006e5e43e927362b') extends('python') From 42729d9cceafaf2382455f9e765cbc261dc41578 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 30 Mar 2016 11:10:32 -0500 Subject: [PATCH 790/910] Added version 2016.3. --- var/spack/repos/builtin/packages/py-pytz/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-pytz/package.py b/var/spack/repos/builtin/packages/py-pytz/package.py index da6311a784a..060cf0cde41 100644 --- a/var/spack/repos/builtin/packages/py-pytz/package.py +++ b/var/spack/repos/builtin/packages/py-pytz/package.py @@ -7,6 +7,7 @@ class PyPytz(Package): version('2014.10', 'eb1cb941a20c5b751352c52486aa1dd7') version('2015.4', '417a47b1c432d90333e42084a605d3d8') + version('2016.3', 'abae92c3301b27bd8a9f56b14f52cb29') extends('python') From 17075736a0ff4f8f677527664e37d55b39cbfe93 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 30 Mar 2016 11:16:11 -0500 Subject: [PATCH 791/910] Add version 2.5.2. --- var/spack/repos/builtin/packages/py-dateutil/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-dateutil/package.py b/var/spack/repos/builtin/packages/py-dateutil/package.py index 0a17f2f2d24..b67e91ace60 100644 --- a/var/spack/repos/builtin/packages/py-dateutil/package.py +++ b/var/spack/repos/builtin/packages/py-dateutil/package.py @@ -7,6 +7,7 @@ class PyDateutil(Package): version('2.4.0', '75714163bb96bedd07685cdb2071b8bc') version('2.4.2', '4ef68e1c485b09e9f034e10473e5add2') + version('2.5.2', 'eafe168e8f404bf384514f5116eedbb6') extends('python') depends_on('py-setuptools') From 8948cb100be21034b7e4bc78b7383fa73005a03a Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 30 Mar 2016 11:16:35 -0500 Subject: [PATCH 792/910] Add version 1.3.7. --- var/spack/repos/builtin/packages/py-nose/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-nose/package.py b/var/spack/repos/builtin/packages/py-nose/package.py index 4fee99098ed..c2c2b52e03e 100644 --- a/var/spack/repos/builtin/packages/py-nose/package.py +++ b/var/spack/repos/builtin/packages/py-nose/package.py @@ -10,6 +10,7 @@ class PyNose(Package): version('1.3.4', '6ed7169887580ddc9a8e16048d38274d') version('1.3.6', '0ca546d81ca8309080fc80cb389e7a16') + version('1.3.7', '4d3ad0ff07b61373d2cefc89c5d0b20b') extends('python', ignore=r'bin/nosetests.*$') depends_on('py-setuptools') From f3b353f5f95c21641d4bee274393879e3a068c3c Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 30 Mar 2016 11:17:24 -0500 Subject: [PATCH 793/910] Add 'bin/f2py' to ignore so activation works. --- var/spack/repos/builtin/packages/py-matplotlib/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/py-matplotlib/package.py b/var/spack/repos/builtin/packages/py-matplotlib/package.py index 2167735fb89..45e77dd6313 100644 --- a/var/spack/repos/builtin/packages/py-matplotlib/package.py +++ b/var/spack/repos/builtin/packages/py-matplotlib/package.py @@ -12,7 +12,7 @@ class PyMatplotlib(Package): variant('gui', default=False, description='Enable GUI') variant('ipython', default=False, description='Enable ipython support') - extends('python', ignore=r'bin/nosetests.*$|bin/pbr$') + extends('python', ignore=r'bin/nosetests.*$|bin/pbr$|bin/f2py$') depends_on('py-pyside', when='+gui') depends_on('py-ipython', when='+ipython') From e7ae983b41943939a20ca4964824b166833c282b Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 30 Mar 2016 11:20:27 -0500 Subject: [PATCH 794/910] Added version 2.5. Added 'bin/f2py' to ignore so that activation works. --- var/spack/repos/builtin/packages/py-numexpr/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/py-numexpr/package.py b/var/spack/repos/builtin/packages/py-numexpr/package.py index 89f8a525b12..081a79dec68 100644 --- a/var/spack/repos/builtin/packages/py-numexpr/package.py +++ b/var/spack/repos/builtin/packages/py-numexpr/package.py @@ -7,8 +7,9 @@ class PyNumexpr(Package): url = "https://pypi.python.org/packages/source/n/numexpr/numexpr-2.4.6.tar.gz" version('2.4.6', '17ac6fafc9ea1ce3eb970b9abccb4fbd') + version('2.5', '84f66cced45ba3e30dcf77a937763aaa') - extends('python') + extends('python', ignore=r'bin/f2py$') depends_on('py-numpy') def install(self, spec, prefix): From 78be85c84f89d861086c04e4fcc4fa70c21d0a5f Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 30 Mar 2016 11:21:17 -0500 Subject: [PATCH 795/910] Added version 1.11.0. --- var/spack/repos/builtin/packages/py-numpy/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-numpy/package.py b/var/spack/repos/builtin/packages/py-numpy/package.py index a08e612df6c..40988fb44a7 100644 --- a/var/spack/repos/builtin/packages/py-numpy/package.py +++ b/var/spack/repos/builtin/packages/py-numpy/package.py @@ -9,6 +9,7 @@ class PyNumpy(Package): homepage = "http://www.numpy.org/" url = "https://pypi.python.org/packages/source/n/numpy/numpy-1.9.1.tar.gz" + version('1.11.0', 'bc56fb9fc2895aa4961802ffbdb31d0b') version('1.10.4', 'aed294de0aa1ac7bd3f9745f4f1968ad') version('1.9.2', 'a1ed53432dbcd256398898d35bc8e645') version('1.9.1', '78842b73560ec378142665e712ae4ad9') From 8ae84cfabaf22e52b01911c909b2d2342b75f3d9 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 30 Mar 2016 11:21:38 -0500 Subject: [PATCH 796/910] This commit updates pandas and fixes dependencies. Added version 0.18.0. Added 'bin/f2py' to ignore so that activation works. Removed unneccessary dependencies on - libdrm - libpciaccess - llvm - mesa Added recommended dependencies from the pandas project site. http://pandas.pydata.org/pandas-docs/stable/install.html - py-numexpr - py-bottleneck The py-bottleneck is a new package that is also in this branch. Removed optional dependencies - py-scipy - py-matplotlib While those are most likely desired to be in the same python stack as pandas they are not required by pandas. --- var/spack/repos/builtin/packages/py-pandas/package.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-pandas/package.py b/var/spack/repos/builtin/packages/py-pandas/package.py index 5b9997faa94..2320b1f92f3 100644 --- a/var/spack/repos/builtin/packages/py-pandas/package.py +++ b/var/spack/repos/builtin/packages/py-pandas/package.py @@ -8,18 +8,15 @@ class PyPandas(Package): version('0.16.0', 'bfe311f05dc0c351f8955fbd1e296e73') version('0.16.1', 'fac4f25748f9610a3e00e765474bdea8') + version('0.18.0', 'f143762cd7a59815e348adf4308d2cf6') - extends('python') + extends('python', ignore=r'bin/f2py$') depends_on('py-dateutil') depends_on('py-numpy') - depends_on('py-matplotlib') - depends_on('py-scipy') depends_on('py-setuptools') depends_on('py-pytz') - depends_on('libdrm') - depends_on('libpciaccess') - depends_on('llvm') - depends_on('mesa') + depends_on('py-numexpr') + depends_on('py-bottleneck') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) From 9519f3d988d4f83cd3edf9bbdd90080559ba9549 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 30 Mar 2016 11:46:58 -0500 Subject: [PATCH 797/910] Fix MPI-CUDA bug --- var/spack/repos/builtin/packages/cuda/package.py | 11 ++++++----- .../repos/builtin/packages/hoomd-blue/package.py | 13 +++++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/cuda/package.py b/var/spack/repos/builtin/packages/cuda/package.py index 547b43d6d26..eca371d05ae 100644 --- a/var/spack/repos/builtin/packages/cuda/package.py +++ b/var/spack/repos/builtin/packages/cuda/package.py @@ -16,9 +16,12 @@ class Cuda(Package): see http://software.llnl.gov/spack/mirrors.html""" homepage = "http://www.nvidia.com/object/cuda_home_new.html" - url = "file://%s/cuda_7.5.18_linux.run" % os.getcwd() - version('7.5.18', '4b3bcecf0dfc35928a0898793cf3e4c6', expand=False) + version('7.5.18', '4b3bcecf0dfc35928a0898793cf3e4c6', expand=False, + url="file://%s/cuda_7.5.18_linux.run" % os.getcwd()) + version('6.5.14', '90b1b8f77313600cc294d9271741f4da', expand=False, + url="file://%s/cuda_6.5.14_linux_64.run" % os.getcwd()) + def install(self, spec, prefix): runfile = glob(os.path.join(self.stage.path, 'cuda*.run'))[0] @@ -27,15 +30,13 @@ def install(self, spec, prefix): runfile = which(runfile) # Note: NVIDIA does not officially support many newer versions of compilers. - # For example, on CentOS 6, you must use GCC 4.4.7 or older. - # The --override flag disables these checks. See: + # For example, on CentOS 6, you must use GCC 4.4.7 or older. See: # http://docs.nvidia.com/cuda/cuda-installation-guide-linux/#system-requirements # for details. runfile( '--silent', # disable interactive prompts '--verbose', # create verbose log file - '--override', # ignore compiler checks '--toolkit', # install CUDA Toolkit '--toolkitpath=%s' % prefix ) diff --git a/var/spack/repos/builtin/packages/hoomd-blue/package.py b/var/spack/repos/builtin/packages/hoomd-blue/package.py index 97ce9c0cf0f..32fdff94267 100644 --- a/var/spack/repos/builtin/packages/hoomd-blue/package.py +++ b/var/spack/repos/builtin/packages/hoomd-blue/package.py @@ -49,10 +49,14 @@ def install(self, spec, prefix): cmake_args.append('-DENABLE_CUDA=OFF') # CUDA-aware MPI library support - if '+cuda' in spec and '+mpi' in spec: - cmake_args.append('-DENABLE_MPI_CUDA=ON') - else: - cmake_args.append('-DENABLE_MPI_CUDA=OFF') + #if '+cuda' in spec and '+mpi' in spec: + # cmake_args.append('-DENABLE_MPI_CUDA=ON') + #else: + # cmake_args.append('-DENABLE_MPI_CUDA=OFF') + + # There may be a bug in the MPI-CUDA code. See: + # https://groups.google.com/forum/#!msg/hoomd-users/2griTESmc5I/E69s_M5fDwAJ + cmake_args.append('-DENABLE_MPI_CUDA=OFF') # Documentation if '+doc' in spec: @@ -64,4 +68,5 @@ def install(self, spec, prefix): cmake('.', *cmake_args) make() + make("test") make("install") From b909da759d7b6d0443a97c13ba1c97118acd3e7a Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 30 Mar 2016 19:26:03 +0200 Subject: [PATCH 798/910] petsc: remove no longer needed -Qunused-arguments hack --- .../repos/builtin/packages/petsc/package.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index e9b7c8a732c..5c1fc6cc924 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -71,19 +71,10 @@ def mpi_dependent_options(self): errors = ['incompatible variants given'] + errors raise RuntimeError('\n'.join(errors)) else: - if self.compiler.name == "clang": - compiler_opts = [ - '--with-mpi=1', - '--with-cc=%s -Qunused-arguments' % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), # Avoid confusing PETSc config by clang: warning: argument unused during compilation - '--with-cxx=%s -Qunused-arguments' % join_path(self.spec['mpi'].prefix.bin, 'mpic++'), - '--with-fc=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), - '--with-f77=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif77'), - ] - else: - compiler_opts = [ - '--with-mpi=1', - '--with-mpi-dir=%s' % self.spec['mpi'].prefix, - ] + compiler_opts = [ + '--with-mpi=1', + '--with-mpi-dir=%s' % self.spec['mpi'].prefix, + ] return compiler_opts def install(self, spec, prefix): From 01657e6991197c999180c2fec367fc75e59fba15 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 30 Mar 2016 11:41:25 -0700 Subject: [PATCH 799/910] Make suite-sparse use spack compilers. --- .../repos/builtin/packages/suite-sparse/package.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/suite-sparse/package.py b/var/spack/repos/builtin/packages/suite-sparse/package.py index b57f9967c30..c2196dcec4d 100644 --- a/var/spack/repos/builtin/packages/suite-sparse/package.py +++ b/var/spack/repos/builtin/packages/suite-sparse/package.py @@ -23,4 +23,14 @@ def install(self, spec, prefix): # FIXME : this actually uses the current workaround # FIXME : (blas / lapack always provide libblas and liblapack as aliases) - make('install', 'INSTALL=%s' % prefix, 'BLAS=-lblas', 'LAPACK=-llapack') + make('install', 'INSTALL=%s' % prefix, + + # inject Spack compiler wrappers + 'AUTOCC=no', + 'CC=cc', + 'CXX=c++', + 'F77=f77', + + # BLAS arguments require path to libraries + 'BLAS=-lblas', + 'LAPACK=-llapack') From 5506e89aa9f45b94a7b16a24e24e3c081fdf1e38 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 30 Mar 2016 21:41:23 +0200 Subject: [PATCH 800/910] rename variant doxygen to doc --- var/spack/repos/builtin/packages/dealii/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 32dfb33c6d0..7aaf33380e6 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -11,7 +11,7 @@ class Dealii(Package): variant('mpi', default=True, description='Compile with MPI') variant('arpack', default=True, description='Compile with Arpack and PArpack (only with MPI)') - variant('doxygen', default=False, description='Compile with Doxygen documentation') + variant('doc', default=False, description='Compile with documentation') variant('hdf5', default=True, description='Compile with HDF5 (only with MPI)') variant('metis', default=True, description='Compile with Metis') variant('netcdf', default=True, description='Compile with Netcdf (only with MPI)') @@ -36,7 +36,7 @@ class Dealii(Package): # optional dependencies depends_on ("mpi", when="+mpi") depends_on ("arpack-ng+mpi", when='+arpack+mpi') - depends_on ("doxygen", when='+doxygen') + depends_on ("doxygen", when='+doc') depends_on ("hdf5+mpi~cxx", when='+hdf5+mpi') #FIXME NetCDF declares dependency with ~cxx, why? depends_on ("metis", when='+metis') depends_on ("netcdf+mpi", when="+netcdf+mpi") @@ -111,7 +111,7 @@ def install(self, spec, prefix): # doxygen options.extend([ - '-DDEAL_II_COMPONENT_DOCUMENTATION=%s' % ('ON' if '+doxygen' in spec else 'OFF'), + '-DDEAL_II_COMPONENT_DOCUMENTATION=%s' % ('ON' if '+doc' in spec else 'OFF'), ]) From dbb25bfeed7113cbd6be090df52b97fcc10588e1 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Wed, 30 Mar 2016 16:17:52 -0400 Subject: [PATCH 801/910] ninja: fix installation on OS X The cp -t flag is a GNU-ism. Instead, add a trailing slash to bindir to ensure that it is not treated as a file. --- var/spack/repos/builtin/packages/ninja/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/ninja/package.py b/var/spack/repos/builtin/packages/ninja/package.py index 9e6bf4e358d..0722dd49a64 100644 --- a/var/spack/repos/builtin/packages/ninja/package.py +++ b/var/spack/repos/builtin/packages/ninja/package.py @@ -16,7 +16,7 @@ def install(self, spec, prefix): cp = which('cp') - bindir = os.path.join(prefix, 'bin') + bindir = os.path.join(prefix, 'bin/') mkdir(bindir) - cp('-a', '-t', bindir, 'ninja') - cp('-ra', 'misc', prefix) + cp('-a', 'ninja', bindir) + cp('-a', 'misc', prefix) From d1a3adfcf09fb489acce9d29b2f9494df5f5d848 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Wed, 30 Mar 2016 14:25:44 -0600 Subject: [PATCH 802/910] + Make --disable-perl the default configure option. --- var/spack/repos/builtin/packages/graphviz/package.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/graphviz/package.py b/var/spack/repos/builtin/packages/graphviz/package.py index 84bd696af2b..a45cd8743ee 100644 --- a/var/spack/repos/builtin/packages/graphviz/package.py +++ b/var/spack/repos/builtin/packages/graphviz/package.py @@ -7,7 +7,11 @@ class Graphviz(Package): version('2.38.0', '5b6a829b2ac94efcd5fa3c223ed6d3ae') - variant('perl', default=True, description='Disable if you have problems with the optional script language bindings') + # By default disable optional Perl language support to prevent build issues + # related to missing Perl packages. If spack begins support for Perl in the + # future, this package can be updated to depend_on('perl') and the + # ncecessary devel packages. + variant('perl', default=False, description='Disable if you have problems with the optional script language bindings') parallel = False @@ -17,7 +21,7 @@ class Graphviz(Package): def install(self, spec, prefix): options = ['--prefix=%s' % prefix] - if '~perl' in spec: + if not '+perl' in spec: options.append('--disable-perl') configure(*options) From 4ab362ae17da96db2a5fb7bacb3035824817c15d Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Wed, 30 Mar 2016 14:27:23 -0600 Subject: [PATCH 803/910] + Fix variant description. --- var/spack/repos/builtin/packages/graphviz/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/graphviz/package.py b/var/spack/repos/builtin/packages/graphviz/package.py index a45cd8743ee..ecf92620d4a 100644 --- a/var/spack/repos/builtin/packages/graphviz/package.py +++ b/var/spack/repos/builtin/packages/graphviz/package.py @@ -11,7 +11,7 @@ class Graphviz(Package): # related to missing Perl packages. If spack begins support for Perl in the # future, this package can be updated to depend_on('perl') and the # ncecessary devel packages. - variant('perl', default=False, description='Disable if you have problems with the optional script language bindings') + variant('perl', default=False, description='Enable if you need the optional Perl language bindings.') parallel = False From 2bc43f7f3724e86c3ff8875ce0c91178cbc31a6f Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Wed, 30 Mar 2016 16:44:08 -0400 Subject: [PATCH 804/910] docs: add a link to the spack repo in the PR section --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8664953c0cc..1977a4fee96 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,8 @@ can join it here: At the moment, contributing to Spack is relatively simple. Just send us a [pull request](https://help.github.com/articles/using-pull-requests/). -When you send your request, make ``develop`` the destination branch. +When you send your request, make ``develop`` the destination branch on the +[Spack repository](https://github.com/LLNL/spack). Spack is using a rough approximation of the [Git Flow](http://nvie.com/posts/a-successful-git-branching-model/) From f1a0619821fc4c4aaa0a13b114ecaf8ca35b1ce9 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Wed, 30 Mar 2016 17:19:29 -0400 Subject: [PATCH 805/910] mpich: disable fortran without a fortran compiler --- var/spack/repos/builtin/packages/mpich/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index b20dc8dd60c..2d7955e08d3 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -78,6 +78,9 @@ def install(self, spec, prefix): if not self.compiler.fc: config_args.append("--disable-fc") + if not self.compiler.fc and not self.compiler.f77: + config_args.append("--disable-fortran") + configure(*config_args) make() make("install") From 2ad5d2ea2a280dd81a254458fcfb465073aef8da Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Wed, 30 Mar 2016 15:40:04 -0600 Subject: [PATCH 806/910] + For darwin, add a dependency on gettext. This change was recommended in the discussion for PR#601. --- var/spack/repos/builtin/packages/numdiff/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/numdiff/package.py b/var/spack/repos/builtin/packages/numdiff/package.py index a51f6206cf1..e72c60fadbe 100644 --- a/var/spack/repos/builtin/packages/numdiff/package.py +++ b/var/spack/repos/builtin/packages/numdiff/package.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * +import sys class Numdiff(Package): """Numdiff is a little program that can be used to compare putatively @@ -34,6 +35,8 @@ class Numdiff(Package): version('5.8.1', 'a295eb391f6cb1578209fc6b4f9d994e') + depends_on('gettext', sys.platform=='darwin') + def install(self, spec, prefix): options = ['--prefix=%s' % prefix] configure(*options) From a364ab0d0f5ac226ba9d5a519f2da8ab7a09a846 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Wed, 30 Mar 2016 17:12:32 -0600 Subject: [PATCH 807/910] libpng depends on zlib. --- var/spack/repos/builtin/packages/libpng/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/libpng/package.py b/var/spack/repos/builtin/packages/libpng/package.py index e02b08663e0..73c8c623414 100644 --- a/var/spack/repos/builtin/packages/libpng/package.py +++ b/var/spack/repos/builtin/packages/libpng/package.py @@ -9,6 +9,8 @@ class Libpng(Package): version('1.6.15', '829a256f3de9307731d4f52dc071916d') version('1.6.14', '2101b3de1d5f348925990f9aa8405660') + depends_on('zlib') + def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() From a88c15e1010ccadd2f81de9edac48b140d70f9f1 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Wed, 30 Mar 2016 21:18:04 -0600 Subject: [PATCH 808/910] + libdrm depends_on libpciaccess. Use spack provided libpciaccess. --- var/spack/repos/builtin/packages/libdrm/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/libdrm/package.py b/var/spack/repos/builtin/packages/libdrm/package.py index 00736b7811a..d5d779f7967 100644 --- a/var/spack/repos/builtin/packages/libdrm/package.py +++ b/var/spack/repos/builtin/packages/libdrm/package.py @@ -2,7 +2,7 @@ class Libdrm(Package): """A userspace library for accessing the DRM, direct - rendering manager, on Linux, BSD and other operating + rendering manager, on Linux, BSD and other operating systems that support the ioctl interface.""" homepage = "http://dri.freedesktop.org/libdrm/" # no real website... @@ -11,6 +11,8 @@ class Libdrm(Package): version('2.4.59', '105ac7af1afcd742d402ca7b4eb168b6') version('2.4.33', '86e4e3debe7087d5404461e0032231c8') + depends_on('libpciaccess') + def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 44b89474453bf80fd603eb5ea5b629b0d9dff123 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 30 Mar 2016 23:33:57 -0700 Subject: [PATCH 809/910] arpack-ng needs an extra libtoolize to build. --- var/spack/repos/builtin/packages/arpack-ng/package.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/var/spack/repos/builtin/packages/arpack-ng/package.py b/var/spack/repos/builtin/packages/arpack-ng/package.py index dd86b17a53a..6b152f78631 100644 --- a/var/spack/repos/builtin/packages/arpack-ng/package.py +++ b/var/spack/repos/builtin/packages/arpack-ng/package.py @@ -41,10 +41,17 @@ class ArpackNg(Package): depends_on('blas') depends_on('lapack') + depends_on('automake') + depends_on('autoconf') + depends_on('libtool@2.4.2:') + depends_on('mpi', when='+mpi') def install(self, spec, prefix): # Apparently autotools are not bootstrapped + # TODO: switch to use the CMake build in the next version + # rather than bootstrapping. + which('libtoolize')() bootstrap = Executable('./bootstrap') options = ['--prefix=%s' % prefix] From 397e0d1368d8f6077cb7ea59ee953b993ef3d244 Mon Sep 17 00:00:00 2001 From: Mayeul d'Avezac Date: Thu, 31 Mar 2016 10:12:20 +0100 Subject: [PATCH 810/910] Fixes #454 in a simpler way than 34a8f0c Rather than mess with the environment, have espresso install into the right directory (prefix/bin). --- var/spack/repos/builtin/packages/espresso/package.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index 59f362ab46c..167bb38e381 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -24,7 +24,7 @@ class Espresso(Package): depends_on('fftw~mpi', when='~mpi') depends_on('fftw+mpi', when='+mpi') depends_on('scalapack', when='+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation - + def check_variants(self, spec): error = 'you cannot ask for \'+{variant}\' when \'+mpi\' is not active' if '+scalapack' in spec and '~mpi' in spec: @@ -32,14 +32,10 @@ def check_variants(self, spec): if '+elpa' in spec and ('~mpi' in spec or '~scalapack' in spec): raise RuntimeError(error.format(variant='elpa')) - def setup_environment(self, spack_env, run_env): - # Espresso copies every executable in prefix without creating sub-folders - run_env.prepend_path('PATH', self.prefix) - def install(self, spec, prefix): self.check_variants(spec) - options = ['-prefix=%s' % prefix] + options = ['-prefix=%s' % prefix.bin] if '+mpi' in spec: options.append('--enable-parallel') From 7430f9142fc3fc590232a21a193321be9de33df7 Mon Sep 17 00:00:00 2001 From: Mayeul d'Avezac Date: Thu, 31 Mar 2016 10:16:40 +0100 Subject: [PATCH 811/910] Can install espresso on OS/X Espresso uses unix utility `find dir -name \*.x` to determine the executable to install. On OS/X, it also finds a bunch of debug symbols associated with the exectuble. These files have the same name, but are in a different directory. So the install process is done by hand in the package.py file, for OS/X. --- var/spack/repos/builtin/packages/espresso/package.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index 167bb38e381..0dad57a9f61 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -33,6 +33,7 @@ def check_variants(self, spec): raise RuntimeError(error.format(variant='elpa')) def install(self, spec, prefix): + from glob import glob self.check_variants(spec) options = ['-prefix=%s' % prefix.bin] @@ -61,5 +62,11 @@ def install(self, spec, prefix): configure(*options) make('all') - make('install') + + if spec.architecture.startswith('darwin'): + mkdirp(prefix.bin) + for filename in glob("bin/*.x"): + install(filename, prefix.bin) + else: + make('install') From 4f6320a7eb14f7d09c1e31a92d224e54e32c3921 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 31 Mar 2016 13:22:48 +0200 Subject: [PATCH 812/910] uninstall : added unit tests --- lib/spack/spack/test/__init__.py | 3 +- lib/spack/spack/test/cmd/__init__.py | 0 lib/spack/spack/test/cmd/uninstall.py | 37 ++++++++++++ lib/spack/spack/test/database.py | 82 +-------------------------- lib/spack/spack/test/mock_database.py | 78 +++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 81 deletions(-) create mode 100644 lib/spack/spack/test/cmd/__init__.py create mode 100644 lib/spack/spack/test/cmd/uninstall.py create mode 100644 lib/spack/spack/test/mock_database.py diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index cd842561e6c..175a49428c4 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -67,7 +67,8 @@ 'namespace_trie', 'yaml', 'sbang', - 'environment'] + 'environment', + 'cmd.uninstall'] def list_tests(): diff --git a/lib/spack/spack/test/cmd/__init__.py b/lib/spack/spack/test/cmd/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/spack/spack/test/cmd/uninstall.py b/lib/spack/spack/test/cmd/uninstall.py new file mode 100644 index 00000000000..06a24e2958b --- /dev/null +++ b/lib/spack/spack/test/cmd/uninstall.py @@ -0,0 +1,37 @@ +import spack.test.mock_database + +from spack.cmd.uninstall import uninstall + + +class MockArgs(object): + def __init__(self, packages, all=False, force=False, recursive=False): + self.packages = packages + self.all = all + self.force = force + self.recursive = recursive + self.yes_to_all = True + + +class TestUninstall(spack.test.mock_database.MockDatabase): + def test_uninstall(self): + parser = None + # Multiple matches + args = MockArgs(['mpileaks']) + self.assertRaises(SystemExit, uninstall, parser, args) + # Installed dependents + args = MockArgs(['libelf']) + self.assertRaises(SystemExit, uninstall, parser, args) + # Recursive uninstall + args = MockArgs(['callpath'], all=True, recursive=True) + uninstall(parser, args) + + all_specs = spack.install_layout.all_specs() + self.assertEqual(len(all_specs), 7) + # query specs with multiple configurations + mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')] + callpath_specs = [s for s in all_specs if s.satisfies('callpath')] + mpi_specs = [s for s in all_specs if s.satisfies('mpi')] + + self.assertEqual(len(mpileaks_specs), 0) + self.assertEqual(len(callpath_specs), 0) + self.assertEqual(len(mpi_specs), 3) diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index ce6e8a0552b..465263d0574 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -28,16 +28,12 @@ """ import os.path import multiprocessing -import shutil -import tempfile import spack from llnl.util.filesystem import join_path from llnl.util.lock import * from llnl.util.tty.colify import colify -from spack.database import Database -from spack.directory_layout import YamlDirectoryLayout -from spack.test.mock_packages_test import * +from spack.test.mock_database import MockDatabase def _print_ref_counts(): @@ -75,80 +71,7 @@ def add_rec(spec): colify(recs, cols=3) -class DatabaseTest(MockPackagesTest): - - def _mock_install(self, spec): - s = Spec(spec) - s.concretize() - pkg = spack.repo.get(s) - pkg.do_install(fake=True) - - - def _mock_remove(self, spec): - specs = spack.installed_db.query(spec) - assert(len(specs) == 1) - spec = specs[0] - spec.package.do_uninstall(spec) - - - def setUp(self): - super(DatabaseTest, self).setUp() - # - # TODO: make the mockup below easier. - # - - # Make a fake install directory - self.install_path = tempfile.mkdtemp() - self.spack_install_path = spack.install_path - spack.install_path = self.install_path - - self.install_layout = YamlDirectoryLayout(self.install_path) - self.spack_install_layout = spack.install_layout - spack.install_layout = self.install_layout - - # Make fake database and fake install directory. - self.installed_db = Database(self.install_path) - self.spack_installed_db = spack.installed_db - spack.installed_db = self.installed_db - - # make a mock database with some packages installed note that - # the ref count for dyninst here will be 3, as it's recycled - # across each install. - # - # Here is what the mock DB looks like: - # - # o mpileaks o mpileaks' o mpileaks'' - # |\ |\ |\ - # | o callpath | o callpath' | o callpath'' - # |/| |/| |/| - # o | mpich o | mpich2 o | zmpi - # | | o | fake - # | | | - # | |______________/ - # | .____________/ - # |/ - # o dyninst - # |\ - # | o libdwarf - # |/ - # o libelf - # - - # Transaction used to avoid repeated writes. - with spack.installed_db.write_transaction(): - self._mock_install('mpileaks ^mpich') - self._mock_install('mpileaks ^mpich2') - self._mock_install('mpileaks ^zmpi') - - - def tearDown(self): - super(DatabaseTest, self).tearDown() - shutil.rmtree(self.install_path) - spack.install_path = self.spack_install_path - spack.install_layout = self.spack_install_layout - spack.installed_db = self.spack_installed_db - - +class DatabaseTest(MockDatabase): def test_005_db_exists(self): """Make sure db cache file exists after creating.""" index_file = join_path(self.install_path, '.spack-db', 'index.yaml') @@ -157,7 +80,6 @@ def test_005_db_exists(self): self.assertTrue(os.path.exists(index_file)) self.assertTrue(os.path.exists(lock_file)) - def test_010_all_install_sanity(self): """Ensure that the install layout reflects what we think it does.""" all_specs = spack.install_layout.all_specs() diff --git a/lib/spack/spack/test/mock_database.py b/lib/spack/spack/test/mock_database.py new file mode 100644 index 00000000000..6fd05439bff --- /dev/null +++ b/lib/spack/spack/test/mock_database.py @@ -0,0 +1,78 @@ +import shutil +import tempfile + +import spack +from spack.spec import Spec +from spack.database import Database +from spack.directory_layout import YamlDirectoryLayout +from spack.test.mock_packages_test import MockPackagesTest + + +class MockDatabase(MockPackagesTest): + def _mock_install(self, spec): + s = Spec(spec) + s.concretize() + pkg = spack.repo.get(s) + pkg.do_install(fake=True) + + def _mock_remove(self, spec): + specs = spack.installed_db.query(spec) + assert(len(specs) == 1) + spec = specs[0] + spec.package.do_uninstall(spec) + + def setUp(self): + super(MockDatabase, self).setUp() + # + # TODO: make the mockup below easier. + # + + # Make a fake install directory + self.install_path = tempfile.mkdtemp() + self.spack_install_path = spack.install_path + spack.install_path = self.install_path + + self.install_layout = YamlDirectoryLayout(self.install_path) + self.spack_install_layout = spack.install_layout + spack.install_layout = self.install_layout + + # Make fake database and fake install directory. + self.installed_db = Database(self.install_path) + self.spack_installed_db = spack.installed_db + spack.installed_db = self.installed_db + + # make a mock database with some packages installed note that + # the ref count for dyninst here will be 3, as it's recycled + # across each install. + # + # Here is what the mock DB looks like: + # + # o mpileaks o mpileaks' o mpileaks'' + # |\ |\ |\ + # | o callpath | o callpath' | o callpath'' + # |/| |/| |/| + # o | mpich o | mpich2 o | zmpi + # | | o | fake + # | | | + # | |______________/ + # | .____________/ + # |/ + # o dyninst + # |\ + # | o libdwarf + # |/ + # o libelf + # + + # Transaction used to avoid repeated writes. + with spack.installed_db.write_transaction(): + self._mock_install('mpileaks ^mpich') + self._mock_install('mpileaks ^mpich2') + self._mock_install('mpileaks ^zmpi') + + def tearDown(self): + super(MockDatabase, self).tearDown() + shutil.rmtree(self.install_path) + spack.install_path = self.spack_install_path + spack.install_layout = self.spack_install_layout + spack.installed_db = self.spack_installed_db From 6d0b4a28abf8216ed141dac80a6779f6a51f03f1 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 31 Mar 2016 09:39:37 +0200 Subject: [PATCH 813/910] oce: fix bugs related to NULL pointers and compiler optimisation --- .../repos/builtin/packages/dealii/package.py | 3 +- .../repos/builtin/packages/oce/null.patch | 482 ++++++++++++++++++ .../repos/builtin/packages/oce/package.py | 22 +- 3 files changed, 502 insertions(+), 5 deletions(-) create mode 100644 var/spack/repos/builtin/packages/oce/null.patch diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 7aaf33380e6..0b76db3827a 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -235,5 +235,4 @@ def install(self, spec, prefix): if 'oce' in spec: cmake('.') make('release') - if sys.platform != 'darwin': #FIXME - make('run',parallel=False) + make('run',parallel=False) diff --git a/var/spack/repos/builtin/packages/oce/null.patch b/var/spack/repos/builtin/packages/oce/null.patch new file mode 100644 index 00000000000..42a3f0e44f3 --- /dev/null +++ b/var/spack/repos/builtin/packages/oce/null.patch @@ -0,0 +1,482 @@ +From 61cb965b9ffeca419005bc15e635e67589c421dd Mon Sep 17 00:00:00 2001 +From: Martin Siggel +Date: Thu, 28 Jan 2016 19:05:00 +0100 +Subject: [PATCH] Workaround clang optimizations for null references + +OCCT/OCE includes some evil code that uses NULL references, +which are normally not possible. Clang removes code in +branches like if(&myNullRef==NULL) as it assumes this can +never be true. This fix was inspired from the mantis issue +http://tracker.dev.opencascade.org/view.php?id=26042. This +code will be fixed in OCCT 7, but we might require the fix +for earlier releases as well. + +Fixes issue #576 +--- + inc/PLib.hxx | 2 +- + src/BSplCLib/BSplCLib.cxx | 16 ++++++------- + src/BSplCLib/BSplCLib_2.cxx | 6 ++--- + src/BSplCLib/BSplCLib_CurveComputation.gxx | 26 ++++++++++----------- + src/BSplSLib/BSplSLib.cxx | 36 +++++++++++++++--------------- + src/BSplSLib/BSplSLib_BzSyntaxes.cxx | 2 +- + src/PLib/PLib.cxx | 10 ++++----- + 7 files changed, 49 insertions(+), 49 deletions(-) + +diff --git a/inc/PLib.hxx b/inc/PLib.hxx +index 7513234..52b1f84 100644 +--- a/inc/PLib.hxx ++++ b/inc/PLib.hxx +@@ -343,6 +343,6 @@ friend class PLib_DoubleJacobiPolynomial; + + + +- ++#define IS_NULL_REF(ref) ((reinterpret_cast(&ref) & 0xFFFFFF) == 0) + + #endif // _PLib_HeaderFile +diff --git a/src/BSplCLib/BSplCLib.cxx b/src/BSplCLib/BSplCLib.cxx +index 683e4ab..2a2d9ea 100644 +--- a/src/BSplCLib/BSplCLib.cxx ++++ b/src/BSplCLib/BSplCLib.cxx +@@ -298,7 +298,7 @@ void BSplCLib::LocateParameter + Standard_Real& NewU) + { + Standard_Integer first,last; +- if (&Mults) { ++ if (!IS_NULL_REF(Mults)) { + if (Periodic) { + first = Knots.Lower(); + last = Knots.Upper(); +@@ -1434,7 +1434,7 @@ void BSplCLib::BuildKnots(const Standard_Integer Degree, + const Standard_Real * pkn = &Knots(KLower); + pkn -= KLower; + Standard_Real *knot = &LK; +- if (&Mults == NULL) { ++ if (IS_NULL_REF(Mults)) { + switch (Degree) { + case 1 : { + Standard_Integer j = Index ; +@@ -1672,7 +1672,7 @@ Standard_Boolean BSplCLib::PrepareInsertKnots + const Standard_Real Tolerance, + const Standard_Boolean Add) + { +- Standard_Boolean addflat = &AddMults == NULL; ++ Standard_Boolean addflat = IS_NULL_REF(AddMults); + + Standard_Integer first,last; + if (Periodic) { +@@ -1856,7 +1856,7 @@ void BSplCLib::InsertKnots + const Standard_Real Tolerance, + const Standard_Boolean Add) + { +- Standard_Boolean addflat = &AddMults == NULL; ++ Standard_Boolean addflat = IS_NULL_REF(AddMults); + + Standard_Integer i,k,mult,firstmult; + Standard_Integer index,kn,curnk,curk; +@@ -3902,7 +3902,7 @@ void BSplCLib::Resolution( Standard_Real& Poles, + num_poles = FlatKnots.Length() - Deg1; + switch (ArrayDimension) { + case 2 : { +- if (&Weights != NULL) { ++ if (!IS_NULL_REF(Weights)) { + const Standard_Real * WG = &Weights(Weights.Lower()); + min_weights = WG[0]; + +@@ -3970,7 +3970,7 @@ void BSplCLib::Resolution( Standard_Real& Poles, + break; + } + case 3 : { +- if (&Weights != NULL) { ++ if (!IS_NULL_REF(Weights)) { + const Standard_Real * WG = &Weights(Weights.Lower()); + min_weights = WG[0]; + +@@ -4047,7 +4047,7 @@ void BSplCLib::Resolution( Standard_Real& Poles, + break; + } + case 4 : { +- if (&Weights != NULL) { ++ if (!IS_NULL_REF(Weights)) { + const Standard_Real * WG = &Weights(Weights.Lower()); + min_weights = WG[0]; + +@@ -4134,7 +4134,7 @@ void BSplCLib::Resolution( Standard_Real& Poles, + } + default : { + Standard_Integer kk; +- if (&Weights != NULL) { ++ if (!IS_NULL_REF(Weights)) { + const Standard_Real * WG = &Weights(Weights.Lower()); + min_weights = WG[0]; + +diff --git a/src/BSplCLib/BSplCLib_2.cxx b/src/BSplCLib/BSplCLib_2.cxx +index 35c4639..653b7cd 100644 +--- a/src/BSplCLib/BSplCLib_2.cxx ++++ b/src/BSplCLib/BSplCLib_2.cxx +@@ -70,7 +70,7 @@ void BSplCLib::BuildEval(const Standard_Integer Degree, + Standard_Integer i; + Standard_Integer ip = PLower + Index - 1; + Standard_Real w, *pole = &LP; +- if (&Weights == NULL) { ++ if (IS_NULL_REF(Weights)) { + + for (i = 0; i <= Degree; i++) { + ip++; +@@ -115,13 +115,13 @@ static void PrepareEval + + // make the knots + BSplCLib::BuildKnots(Degree,index,Periodic,Knots,Mults,*dc.knots); +- if (&Mults == NULL) ++ if (IS_NULL_REF(Mults)) + index -= Knots.Lower() + Degree; + else + index = BSplCLib::PoleIndex(Degree,index,Periodic,Mults); + + // check truly rational +- rational = (&Weights != NULL); ++ rational = (!IS_NULL_REF(Weights)); + if (rational) { + Standard_Integer WLower = Weights.Lower() + index; + rational = BSplCLib::IsRational(Weights, WLower, WLower + Degree); +diff --git a/src/BSplCLib/BSplCLib_CurveComputation.gxx b/src/BSplCLib/BSplCLib_CurveComputation.gxx +index e71b4e0..9d42643 100644 +--- a/src/BSplCLib/BSplCLib_CurveComputation.gxx ++++ b/src/BSplCLib/BSplCLib_CurveComputation.gxx +@@ -92,7 +92,7 @@ Standard_Boolean BSplCLib::RemoveKnot + TColStd_Array1OfInteger& NewMults, + const Standard_Real Tolerance) + { +- Standard_Boolean rational = &Weights != NULL; ++ Standard_Boolean rational = !IS_NULL_REF(Weights); + Standard_Integer dim; + dim = Dimension_gen; + if (rational) dim++; +@@ -133,7 +133,7 @@ void BSplCLib::InsertKnots + const Standard_Real Epsilon, + const Standard_Boolean Add) + { +- Standard_Boolean rational = &Weights != NULL; ++ Standard_Boolean rational = !IS_NULL_REF(Weights); + Standard_Integer dim; + dim = Dimension_gen; + if (rational) dim++; +@@ -222,7 +222,7 @@ void BSplCLib::IncreaseDegree + TColStd_Array1OfReal& NewKnots, + TColStd_Array1OfInteger& NewMults) + { +- Standard_Boolean rational = &Weights != NULL; ++ Standard_Boolean rational = !IS_NULL_REF(Weights); + Standard_Integer dim; + dim = Dimension_gen; + if (rational) dim++; +@@ -256,7 +256,7 @@ void BSplCLib::Unperiodize + Array1OfPoints& NewPoles, + TColStd_Array1OfReal& NewWeights) + { +- Standard_Boolean rational = &Weights != NULL; ++ Standard_Boolean rational = !IS_NULL_REF(Weights); + Standard_Integer dim; + dim = Dimension_gen; + if (rational) dim++; +@@ -292,7 +292,7 @@ void BSplCLib::Trimming(const Standard_Integer Degree, + Array1OfPoints& NewPoles, + TColStd_Array1OfReal& NewWeights) + { +- Standard_Boolean rational = &Weights != NULL; ++ Standard_Boolean rational = !IS_NULL_REF(Weights); + Standard_Integer dim; + dim = Dimension_gen; + if (rational) dim++; +@@ -339,7 +339,7 @@ void BSplCLib::BuildEval(const Standard_Integer Degree, + Standard_Integer PUpper = Poles.Upper(); + Standard_Integer i; + Standard_Integer ip = PLower + Index - 1; +- if (&Weights == NULL) { ++ if (IS_NULL_REF(Weights)) { + for (i = 0; i <= Degree; i++) { + ip++; + if (ip > PUpper) ip = PLower; +@@ -384,13 +384,13 @@ static void PrepareEval + + // make the knots + BSplCLib::BuildKnots(Degree,index,Periodic,Knots,Mults,*dc.knots); +- if (&Mults == NULL) ++ if (IS_NULL_REF(Mults)) + index -= Knots.Lower() + Degree; + else + index = BSplCLib::PoleIndex(Degree,index,Periodic,Mults); + + // check truly rational +- rational = (&Weights != NULL); ++ rational = (!IS_NULL_REF(Weights)); + if (rational) { + Standard_Integer WLower = Weights.Lower() + index; + rational = BSplCLib::IsRational(Weights, WLower, WLower + Degree); +@@ -741,7 +741,7 @@ void BSplCLib::CacheD0(const Standard_Real Parameter, + Degree * Dimension_gen, + PArray[0], + myPoint[0]) ; +- if (&WeightsArray != NULL) { ++ if (!IS_NULL_REF(WeightsArray)) { + Standard_Real * + WArray = (Standard_Real *) &WeightsArray(WeightsArray.Lower()) ; + PLib::NoDerivativeEvalPolynomial(NewParameter, +@@ -798,7 +798,7 @@ void BSplCLib::CacheD1(const Standard_Real Parameter, + + ModifyCoords (LocalPDerivatives + Dimension_gen, /= SpanLenght); + +- if (&WeightsArray != NULL) { ++ if (!IS_NULL_REF(WeightsArray)) { + Standard_Real * + WArray = (Standard_Real *) &WeightsArray(WeightsArray.Lower()) ; + PLib::EvalPolynomial(NewParameter, +@@ -878,7 +878,7 @@ void BSplCLib::CacheD2(const Standard_Real Parameter, + Index += Dimension_gen; + } + +- if (&WeightsArray != NULL) { ++ if (!IS_NULL_REF(WeightsArray)) { + Standard_Real * + WArray = (Standard_Real *) &WeightsArray(WeightsArray.Lower()) ; + +@@ -971,7 +971,7 @@ void BSplCLib::CacheD3(const Standard_Real Parameter, + Index += Dimension_gen; + } + +- if (&WeightsArray != NULL) { ++ if (!IS_NULL_REF(WeightsArray)) { + Standard_Real * + WArray = (Standard_Real *) &WeightsArray(WeightsArray.Lower()) ; + +@@ -1081,7 +1081,7 @@ void BSplCLib::BuildCache + LocalValue *= SpanDomain / (Standard_Real) ii ; + } + +- if (&Weights != NULL) { ++ if (!IS_NULL_REF(Weights)) { + for (ii = 1 ; ii <= Degree + 1 ; ii++) + CacheWeights(ii) = 0.0e0 ; + CacheWeights(1) = 1.0e0 ; +diff --git a/src/BSplSLib/BSplSLib.cxx b/src/BSplSLib/BSplSLib.cxx +index 5ad633c..07040d5 100644 +--- a/src/BSplSLib/BSplSLib.cxx ++++ b/src/BSplSLib/BSplSLib.cxx +@@ -309,12 +309,12 @@ static Standard_Boolean PrepareEval (const Standard_Real U, + BSplCLib::BuildKnots(UDegree,uindex,UPer,UKnots,UMults,*dc.knots1); + BSplCLib::BuildKnots(VDegree,vindex,VPer,VKnots,VMults,*dc.knots2); + +- if (&UMults == NULL) ++ if (IS_NULL_REF(UMults)) + uindex -= UKLower + UDegree; + else + uindex = BSplCLib::PoleIndex(UDegree,uindex,UPer,UMults); + +- if (&VMults == NULL) ++ if (IS_NULL_REF(VMults)) + vindex -= VKLower + VDegree; + else + vindex = BSplCLib::PoleIndex(VDegree,vindex,VPer,VMults); +@@ -460,12 +460,12 @@ static Standard_Boolean PrepareEval (const Standard_Real U, + BSplCLib::BuildKnots(UDegree,uindex,UPer,UKnots,UMults,*dc.knots2); + BSplCLib::BuildKnots(VDegree,vindex,VPer,VKnots,VMults,*dc.knots1); + +- if (&UMults == NULL) ++ if (IS_NULL_REF(UMults)) + uindex -= UKLower + UDegree; + else + uindex = BSplCLib::PoleIndex(UDegree,uindex,UPer,UMults); + +- if (&VMults == NULL) ++ if (IS_NULL_REF(VMults)) + vindex -= VKLower + VDegree; + else + vindex = BSplCLib::PoleIndex(VDegree,vindex,VPer,VMults); +@@ -1299,7 +1299,7 @@ void BSplSLib::Iso(const Standard_Real Param, + { + Standard_Integer index = 0; + Standard_Real u = Param; +- Standard_Boolean rational = &Weights != NULL; ++ Standard_Boolean rational = !IS_NULL_REF(Weights); + Standard_Integer dim = rational ? 4 : 3; + + // compute local knots +@@ -1307,7 +1307,7 @@ void BSplSLib::Iso(const Standard_Real Param, + NCollection_LocalArray locknots1 (2*Degree); + BSplCLib::LocateParameter(Degree,Knots,Mults,u,Periodic,index,u); + BSplCLib::BuildKnots(Degree,index,Periodic,Knots,Mults,*locknots1); +- if (&Mults == NULL) ++ if (IS_NULL_REF(Mults)) + index -= Knots.Lower() + Degree; + else + index = BSplCLib::PoleIndex(Degree,index,Periodic,Mults); +@@ -1381,7 +1381,7 @@ void BSplSLib::Iso(const Standard_Real Param, + } + + // if the input is not rational but weights are wanted +- if (!rational && (&CWeights != NULL)) { ++ if (!rational && (!IS_NULL_REF(CWeights))) { + + for (i = CWeights.Lower(); i <= CWeights.Upper(); i++) + CWeights(i) = 1.; +@@ -1741,7 +1741,7 @@ void BSplSLib::InsertKnots(const Standard_Boolean UDirection, + const Standard_Real Epsilon, + const Standard_Boolean Add ) + { +- Standard_Boolean rational = &Weights != NULL; ++ Standard_Boolean rational = !IS_NULL_REF(Weights); + Standard_Integer dim = 3; + if (rational) dim++; + +@@ -1787,7 +1787,7 @@ Standard_Boolean BSplSLib::RemoveKnot + TColStd_Array1OfInteger& NewMults, + const Standard_Real Tolerance) + { +- Standard_Boolean rational = &Weights != NULL; ++ Standard_Boolean rational = !IS_NULL_REF(Weights); + Standard_Integer dim = 3; + if (rational) dim++; + +@@ -1834,7 +1834,7 @@ void BSplSLib::IncreaseDegree + TColStd_Array1OfReal& NewKnots, + TColStd_Array1OfInteger& NewMults) + { +- Standard_Boolean rational = &Weights != NULL; ++ Standard_Boolean rational = !IS_NULL_REF(Weights); + Standard_Integer dim = 3; + if (rational) dim++; + +@@ -1876,7 +1876,7 @@ void BSplSLib::Unperiodize + TColgp_Array2OfPnt& NewPoles, + TColStd_Array2OfReal& NewWeights) + { +- Standard_Boolean rational = &Weights != NULL; ++ Standard_Boolean rational = !IS_NULL_REF(Weights); + Standard_Integer dim = 3; + if (rational) dim++; + +@@ -1929,7 +1929,7 @@ void BSplSLib::BuildCache + Standard_Boolean rational,rational_u,rational_v,flag_u_or_v; + Standard_Integer kk,d1,d1p1,d2,d2p1,ii,jj,iii,jjj,Index; + Standard_Real u1,min_degree_domain,max_degree_domain,f,factor[2],u2; +- if (&Weights != NULL) ++ if (!IS_NULL_REF(Weights)) + rational_u = rational_v = Standard_True; + else + rational_u = rational_v = Standard_False; +@@ -2025,7 +2025,7 @@ void BSplSLib::BuildCache + } + factor[0] *= max_degree_domain / (Standard_Real) (iii) ; + } +- if (&Weights != NULL) { ++ if (!IS_NULL_REF(Weights)) { + // + // means that PrepareEval did found out that the surface was + // locally polynomial but since the surface is constructed +@@ -2110,7 +2110,7 @@ void BSplSLib::CacheD0(const Standard_Real UParameter, + (min_degree << 1) + min_degree, + locpoles[0], + myPoint[0]) ; +- if (&WeightsArray != NULL) { ++ if (!IS_NULL_REF(WeightsArray)) { + dimension = min_degree + 1 ; + Standard_Real * + WArray = (Standard_Real *) +@@ -2190,7 +2190,7 @@ void BSplSLib::CacheD1(const Standard_Real UParameter, + // the coefficients + // + // +- if (&WeightsArray != NULL) { ++ if (!IS_NULL_REF(WeightsArray)) { + + local_poles_array [0][0][0] = 0.0e0 ; + local_poles_array [0][0][1] = 0.0e0 ; +@@ -2275,7 +2275,7 @@ void BSplSLib::CacheD1(const Standard_Real UParameter, + locpoles[dimension], + local_poles_array[1][0][0]) ; + +- if (&WeightsArray != NULL) { ++ if (!IS_NULL_REF(WeightsArray)) { + dimension = min_degree + 1 ; + Standard_Real * + WArray = (Standard_Real *) +@@ -2435,7 +2435,7 @@ void BSplSLib::CacheD2(const Standard_Real UParameter, + // the coefficients + // + // +- if (&WeightsArray != NULL) { ++ if (!IS_NULL_REF(WeightsArray)) { + + local_poles_and_weights_array[0][0][0] = 0.0e0 ; + local_poles_and_weights_array[0][0][1] = 0.0e0 ; +@@ -2564,7 +2564,7 @@ void BSplSLib::CacheD2(const Standard_Real UParameter, + locpoles[dimension + dimension], + local_poles_array[2][0][0]) ; + +- if (&WeightsArray != NULL) { ++ if (!IS_NULL_REF(WeightsArray)) { + dimension = min_degree + 1 ; + Standard_Real * + WArray = (Standard_Real *) +diff --git a/src/BSplSLib/BSplSLib_BzSyntaxes.cxx b/src/BSplSLib/BSplSLib_BzSyntaxes.cxx +index 0faf6b6..f2c0f74 100644 +--- a/src/BSplSLib/BSplSLib_BzSyntaxes.cxx ++++ b/src/BSplSLib/BSplSLib_BzSyntaxes.cxx +@@ -68,7 +68,7 @@ void BSplSLib::PolesCoefficients (const TColgp_Array2OfPnt& Poles, + biduflatknots,bidvflatknots, + Poles,Weights, + CPoles,CWeights); +- if (&Weights == NULL) { ++ if (IS_NULL_REF(Weights)) { + + for (ii = 1; ii <= uclas; ii++) { + +diff --git a/src/PLib/PLib.cxx b/src/PLib/PLib.cxx +index 23fa302..7ee231f 100644 +--- a/src/PLib/PLib.cxx ++++ b/src/PLib/PLib.cxx +@@ -2427,7 +2427,7 @@ void PLib::CoefficientsPoles (const Standard_Integer dim, + TColStd_Array1OfReal& Poles, + TColStd_Array1OfReal& Weights) + { +- Standard_Boolean rat = &WCoefs != NULL; ++ Standard_Boolean rat = !IS_NULL_REF(WCoefs); + Standard_Integer loc = Coefs.Lower(); + Standard_Integer lop = Poles.Lower(); + Standard_Integer lowc=0; +@@ -2550,7 +2550,7 @@ void PLib::Trimming(const Standard_Real U1, + Standard_Integer indc, indw=0; + Standard_Integer upc = Coefs.Upper() - dim + 1, upw=0; + Standard_Integer len = Coefs.Length()/dim; +- Standard_Boolean rat = &WCoefs != NULL; ++ Standard_Boolean rat = !IS_NULL_REF(WCoefs); + + if (rat) { + if(len != WCoefs.Length()) +@@ -2607,7 +2607,7 @@ void PLib::CoefficientsPoles (const TColgp_Array2OfPnt& Coefs, + TColgp_Array2OfPnt& Poles, + TColStd_Array2OfReal& Weights) + { +- Standard_Boolean rat = (&WCoefs != NULL); ++ Standard_Boolean rat = (!IS_NULL_REF(WCoefs)); + Standard_Integer LowerRow = Poles.LowerRow(); + Standard_Integer UpperRow = Poles.UpperRow(); + Standard_Integer LowerCol = Poles.LowerCol(); +@@ -2701,7 +2701,7 @@ void PLib::UTrimming(const Standard_Real U1, + TColgp_Array2OfPnt& Coeffs, + TColStd_Array2OfReal& WCoeffs) + { +- Standard_Boolean rat = &WCoeffs != NULL; ++ Standard_Boolean rat = !IS_NULL_REF(WCoeffs); + Standard_Integer lr = Coeffs.LowerRow(); + Standard_Integer ur = Coeffs.UpperRow(); + Standard_Integer lc = Coeffs.LowerCol(); +@@ -2735,7 +2735,7 @@ void PLib::VTrimming(const Standard_Real V1, + TColgp_Array2OfPnt& Coeffs, + TColStd_Array2OfReal& WCoeffs) + { +- Standard_Boolean rat = &WCoeffs != NULL; ++ Standard_Boolean rat = !IS_NULL_REF(WCoeffs); + Standard_Integer lr = Coeffs.LowerRow(); + Standard_Integer ur = Coeffs.UpperRow(); + Standard_Integer lc = Coeffs.LowerCol(); \ No newline at end of file diff --git a/var/spack/repos/builtin/packages/oce/package.py b/var/spack/repos/builtin/packages/oce/package.py index 4d5081ac9d4..3fe6638e66c 100644 --- a/var/spack/repos/builtin/packages/oce/package.py +++ b/var/spack/repos/builtin/packages/oce/package.py @@ -13,9 +13,20 @@ class Oce(Package): version('0.17' , 'f1a89395c4b0d199bea3db62b85f818d') version('0.16.1', '4d591b240c9293e879f50d86a0cb2bb3') version('0.16' , '7a4b4df5a104d75a537e25e7dd387eca') - version('0.15' , '7ec541a1c350ca8a684f74980e48801c') + + variant('tbb', default=True, description='Build with Intel Threading Building Blocks') depends_on('cmake@2.8:') + depends_on('tbb', when='+tbb') + + # There is a bug in OCE which appears with Clang (version?) or GCC 6.0 + # and has to do with compiler optimization, see + # https://github.com/tpaviot/oce/issues/576 + # http://tracker.dev.opencascade.org/view.php?id=26042 + # https://github.com/tpaviot/oce/issues/605 + # https://github.com/tpaviot/oce/commit/61cb965b9ffeca419005bc15e635e67589c421dd.patch + patch('null.patch',when='@0.16:0.17.1') + def install(self, spec, prefix): options = [] @@ -23,12 +34,12 @@ def install(self, spec, prefix): options.extend([ '-DOCE_INSTALL_PREFIX=%s' % prefix, '-DOCE_BUILD_SHARED_LIB:BOOL=ON', - '-DOCE_BUILD_TYPE:STRING=Release', + '-DCMAKE_BUILD_TYPE:STRING=Release', '-DOCE_DATAEXCHANGE:BOOL=ON', '-DOCE_DISABLE_X11:BOOL=ON', '-DOCE_DRAW:BOOL=OFF', '-DOCE_MODEL:BOOL=ON', - '-DOCE_MULTITHREAD_LIBRARY:STRING=NONE', # FIXME: add tbb + '-DOCE_MULTITHREAD_LIBRARY:STRING=%s' % ('TBB' if '+tbb' in spec else 'NONE'), '-DOCE_OCAF:BOOL=ON', '-DOCE_USE_TCL_TEST_FRAMEWORK:BOOL=OFF', '-DOCE_VISUALISATION:BOOL=OFF', @@ -46,6 +57,11 @@ def install(self, spec, prefix): make("install/strip") + # OCE tests build is brocken at least on Darwin. + # Unit tests are linked against libTKernel.10.dylib isntead of /full/path/libTKernel.10.dylib + # see https://github.com/tpaviot/oce/issues/612 + # make("test") + # The shared libraries are not installed correctly on Darwin; correct this if (sys.platform == 'darwin'): fix_darwin_install_name(prefix.lib) From 0bcfd095362c5d04b9add7ebc882b74fad68f9dd Mon Sep 17 00:00:00 2001 From: Dhanannjay Deo Date: Fri, 25 Mar 2016 13:25:29 -0400 Subject: [PATCH 814/910] add: VTK lastest versions --- var/spack/repos/builtin/packages/vtk/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/repos/builtin/packages/vtk/package.py b/var/spack/repos/builtin/packages/vtk/package.py index 4a27a8fedba..aeb03fe0a37 100644 --- a/var/spack/repos/builtin/packages/vtk/package.py +++ b/var/spack/repos/builtin/packages/vtk/package.py @@ -7,6 +7,10 @@ class Vtk(Package): homepage = "http://www.vtk.org" url = "http://www.vtk.org/files/release/6.1/VTK-6.1.0.tar.gz" + version("7.0.0", "5fe35312db5fb2341139b8e4955c367d", url="http://www.vtk.org/files/release/7.0/VTK-7.0.0.tar.gz") + + version("6.3.0", '0231ca4840408e9dd60af48b314c5b6d', url="http://www.vtk.org/files/release/6.3/VTK-6.3.0.tar.gz") + version('6.1.0', '25e4dfb3bad778722dcaec80cd5dab7d') depends_on("qt") From 123996aad0c58bab7de6f615159b2fd7b44c7369 Mon Sep 17 00:00:00 2001 From: Dhanannjay Deo Date: Thu, 31 Mar 2016 10:29:47 -0400 Subject: [PATCH 815/910] Fix build for vtk6.1.0 --- var/spack/repos/builtin/packages/vtk/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/repos/builtin/packages/vtk/package.py b/var/spack/repos/builtin/packages/vtk/package.py index aeb03fe0a37..b29d52d0f03 100644 --- a/var/spack/repos/builtin/packages/vtk/package.py +++ b/var/spack/repos/builtin/packages/vtk/package.py @@ -39,6 +39,10 @@ def install(self, spec, prefix): if spec['qt'].satisfies('@5'): cmake_args.append("-DVTK_QT_VERSION:STRING=5") + if spec.satisfies("@6.1.0"): + cmake_args.append("-DCMAKE_C_FLAGS=-DGLX_GLXEXT_LEGACY") + cmake_args.append("-DCMAKE_CXX_FLAGS=-DGLX_GLXEXT_LEGACY") + cmake(*cmake_args) make() make("install") From 3b0311a1e484a4f731d1cc6448dc6804410ecb3c Mon Sep 17 00:00:00 2001 From: Dhanannjay Deo Date: Thu, 31 Mar 2016 10:30:32 -0400 Subject: [PATCH 816/910] Add variant opengl2, which is ON by default Needed for visit which depends on vtk@6.1.0~opengl2 --- var/spack/repos/builtin/packages/vtk/package.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/var/spack/repos/builtin/packages/vtk/package.py b/var/spack/repos/builtin/packages/vtk/package.py index b29d52d0f03..24382af406c 100644 --- a/var/spack/repos/builtin/packages/vtk/package.py +++ b/var/spack/repos/builtin/packages/vtk/package.py @@ -15,7 +15,15 @@ class Vtk(Package): depends_on("qt") + # VTK7 defaults to OpenGL2 rendering backend + variant('opengl2', default=True, description='Build with OpenGL instead of OpenGL2 as rendering backend') + def install(self, spec, prefix): + def feature_to_bool(feature, on='ON', off='OFF'): + if feature in spec: + return on + return off + with working_dir('spack-build', create=True): cmake_args = [ "..", @@ -43,6 +51,8 @@ def install(self, spec, prefix): cmake_args.append("-DCMAKE_C_FLAGS=-DGLX_GLXEXT_LEGACY") cmake_args.append("-DCMAKE_CXX_FLAGS=-DGLX_GLXEXT_LEGACY") + cmake_args.append('-DVTK_RENDERING_BACKEND:STRING=%s' % feature_to_bool('+opengl2', 'OpenGL2', 'OpenGL')) + cmake(*cmake_args) make() make("install") From a0902ad8d8a8c629eb921b8701332385d622bc12 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 31 Mar 2016 11:04:29 -0500 Subject: [PATCH 817/910] Change variant defaults and add comment --- var/spack/repos/builtin/packages/cuda/package.py | 6 +++++- var/spack/repos/builtin/packages/hoomd-blue/package.py | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/cuda/package.py b/var/spack/repos/builtin/packages/cuda/package.py index eca371d05ae..ea083d8651d 100644 --- a/var/spack/repos/builtin/packages/cuda/package.py +++ b/var/spack/repos/builtin/packages/cuda/package.py @@ -13,7 +13,11 @@ class Cuda(Package): For the Installer Type, select runfile and click Download. Spack will search your current directory for this file. Alternatively, add this file to a mirror so that Spack can find it. For instructions on how to set up a mirror, - see http://software.llnl.gov/spack/mirrors.html""" + see http://software.llnl.gov/spack/mirrors.html + + Note: This package does not currently install the drivers necessary to run + CUDA. These will need to be installed manually. See: + http://docs.nvidia.com/cuda/cuda-getting-started-guide-for-linux for details.""" homepage = "http://www.nvidia.com/object/cuda_home_new.html" diff --git a/var/spack/repos/builtin/packages/hoomd-blue/package.py b/var/spack/repos/builtin/packages/hoomd-blue/package.py index 32fdff94267..d310b7687a5 100644 --- a/var/spack/repos/builtin/packages/hoomd-blue/package.py +++ b/var/spack/repos/builtin/packages/hoomd-blue/package.py @@ -16,9 +16,9 @@ class HoomdBlue(Package): version('1.3.3', '1469ef4531dc14b579c0acddbfe6a273') - variant('mpi', default=False, description='Compile with MPI enabled') - variant('cuda', default=False, description='Compile with CUDA Toolkit') - variant('doc', default=False, description='Generate documentation') + variant('mpi', default=True, description='Compile with MPI enabled') + variant('cuda', default=True, description='Compile with CUDA Toolkit') + variant('doc', default=True, description='Generate documentation') extends('python') depends_on('py-numpy') @@ -56,6 +56,7 @@ def install(self, spec, prefix): # There may be a bug in the MPI-CUDA code. See: # https://groups.google.com/forum/#!msg/hoomd-users/2griTESmc5I/E69s_M5fDwAJ + # This prevented "make test" from passing for me. cmake_args.append('-DENABLE_MPI_CUDA=OFF') # Documentation From 63f824b218af9fcea4c13d7bef8b1d8ff31d09b2 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 11:53:27 -0800 Subject: [PATCH 818/910] add a path argument to the stage command Allow users to use spack to stage a, potentially complex, package into a given path. This is nice for packages with multiple resources that must be placed, for example LLVM with all sub-projects. --- lib/spack/spack/cmd/stage.py | 8 +++++++- lib/spack/spack/package.py | 8 ++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index 5786780efb4..749fa908686 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -35,6 +35,9 @@ def setup_parser(subparser): subparser.add_argument( '-n', '--no-checksum', action='store_true', dest='no_checksum', help="Do not check downloaded packages against checksum") + subparser.add_argument( + '-p', '--path', dest='path', + help="Path to stage package, does not add to spack tree") subparser.add_argument( 'specs', nargs=argparse.REMAINDER, help="specs of packages to stage") @@ -50,4 +53,7 @@ def stage(parser, args): specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.do_stage() + if args.path: + package.do_stage(path=args.path) + else: + package.do_stage() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 9af32218374..ce314b7b0a5 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -709,14 +709,18 @@ def do_fetch(self, mirror_only=False): if spack.do_checksum and self.version in self.versions: self.stage.check() - - def do_stage(self, mirror_only=False): + def do_stage(self, mirror_only=False, path=None): """Unpacks the fetched tarball, then changes into the expanded tarball directory.""" + if not self.spec.concrete: raise ValueError("Can only stage concrete packages.") self.do_fetch(mirror_only) + + if path is not None: + self.stage.path = path + self.stage.expand_archive() self.stage.chdir_to_source() From 5d2151ed645f853a083cd445ae8631f9ed987559 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Thu, 31 Mar 2016 10:20:55 -0700 Subject: [PATCH 819/910] reworked to deal with stage.path as property This version actually pulls the path through the package to deliver it to each stage on creation when passed through the command. This is necessary due to the new StageComposite class that makes setting the path directly on the stage impractical, it also takes the logic out of package for the most part, which seems like an improvement. --- lib/spack/spack/cmd/stage.py | 5 ++--- lib/spack/spack/package.py | 15 +++++++-------- lib/spack/spack/stage.py | 7 +++++-- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index 749fa908686..975bb54ef75 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -54,6 +54,5 @@ def stage(parser, args): for spec in specs: package = spack.repo.get(spec) if args.path: - package.do_stage(path=args.path) - else: - package.do_stage() + package.path = args.path + package.do_stage() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index ce314b7b0a5..9dcfbee6616 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -335,6 +335,9 @@ def __init__(self, spec): if '.' in self.name: self.name = self.name[self.name.rindex('.') + 1:] + # Allow custom staging paths for packages + self.path=None + # Sanity check attributes required by Spack directives. spack.directives.ensure_dicts(type(self)) @@ -445,7 +448,8 @@ def _make_resource_stage(self, root_stage, fetcher, resource): resource_stage_folder = self._resource_stage(resource) resource_mirror = join_path(self.name, os.path.basename(fetcher.url)) stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource, - name=resource_stage_folder, mirror_path=resource_mirror) + name=resource_stage_folder, mirror_path=resource_mirror, + path=self.path) return stage def _make_root_stage(self, fetcher): @@ -455,7 +459,7 @@ def _make_root_stage(self, fetcher): s = self.spec stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash()) # Build the composite stage - stage = Stage(fetcher, mirror_path=mp, name=stage_name) + stage = Stage(fetcher, mirror_path=mp, name=stage_name, path=self.path) return stage def _make_stage(self): @@ -709,18 +713,13 @@ def do_fetch(self, mirror_only=False): if spack.do_checksum and self.version in self.versions: self.stage.check() - def do_stage(self, mirror_only=False, path=None): + def do_stage(self, mirror_only=False): """Unpacks the fetched tarball, then changes into the expanded tarball directory.""" - if not self.spec.concrete: raise ValueError("Can only stage concrete packages.") self.do_fetch(mirror_only) - - if path is not None: - self.stage.path = path - self.stage.expand_archive() self.stage.chdir_to_source() diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index f88f82fc2d6..d711752c208 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -89,7 +89,7 @@ class Stage(object): """ def __init__(self, url_or_fetch_strategy, - name=None, mirror_path=None, keep=False): + name=None, mirror_path=None, keep=False, path=None): """Create a stage object. Parameters: url_or_fetch_strategy @@ -135,7 +135,10 @@ def __init__(self, url_or_fetch_strategy, # Try to construct here a temporary name for the stage directory # If this is a named stage, then construct a named path. - self.path = join_path(spack.stage_path, self.name) + if path is not None: + self.path = path + else: + self.path = join_path(spack.stage_path, self.name) # Flag to decide whether to delete the stage folder on exit or not self.keep = keep From a670408617b76777d35185aa554eeaf3892e0188 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 31 Mar 2016 11:34:00 -0700 Subject: [PATCH 820/910] Quick fix for pkg diff. --- lib/spack/spack/cmd/pkg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index cf478d3763e..20a3fc5fc22 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -77,7 +77,8 @@ def get_git(): def list_packages(rev): git = get_git() - relpath = spack.packages_path[len(spack.prefix + os.path.sep):] + os.path.sep + pkgpath = os.path.join(spack.packages_path, 'packages') + relpath = pkgpath[len(spack.prefix + os.path.sep):] + os.path.sep output = git('ls-tree', '--full-tree', '--name-only', rev, relpath, output=str) return sorted(line[len(relpath):] for line in output.split('\n') if line) From 66038ef729b5fb8d2efadc48c87a868370ca23c5 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 31 Mar 2016 13:34:56 -0500 Subject: [PATCH 821/910] Add url_for_version function and gtk variant --- .../repos/builtin/packages/qt/package.py | 70 ++++++++++++------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index d08e8e81e1a..ab094690422 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -1,41 +1,31 @@ -import os from spack import * import os class Qt(Package): """Qt is a comprehensive cross-platform C++ application framework.""" - homepage = "http://qt.io" - list_url = 'http://download.qt-project.org/official_releases/qt/' - list_depth = 2 + homepage = 'http://qt.io' + url = 'http://download.qt.io/archive/qt/5.5/5.5.1/single/qt-everywhere-opensource-src-5.5.1.tar.gz' - version('5.4.2', 'fa1c4d819b401b267eb246a543a63ea5', - url='http://download.qt-project.org/official_releases/qt/5.4/5.4.2/single/qt-everywhere-opensource-src-5.4.2.tar.gz') + version('5.5.1', '59f0216819152b77536cf660b015d784') + version('5.4.2', 'fa1c4d819b401b267eb246a543a63ea5') + version('5.4.0', 'e8654e4b37dd98039ba20da7a53877e6') + version('5.3.2', 'febb001129927a70174467ecb508a682') + version('5.2.1', 'a78408c887c04c34ce615da690e0b4c8') + version('4.8.6', '2edbe4d6c2eff33ef91732602f3518eb') + version('3.3.8b', '9f05b4125cfe477cc52c9742c3c09009') - version('5.4.0', 'e8654e4b37dd98039ba20da7a53877e6', - url='http://download.qt-project.org/official_releases/qt/5.4/5.4.0/single/qt-everywhere-opensource-src-5.4.0.tar.gz') - - version('5.3.2', 'febb001129927a70174467ecb508a682', - url='http://download.qt.io/archive/qt/5.3/5.3.2/single/qt-everywhere-opensource-src-5.3.2.tar.gz') - - version('5.2.1', 'a78408c887c04c34ce615da690e0b4c8', - url='http://download.qt.io/archive/qt/5.2/5.2.1/single/qt-everywhere-opensource-src-5.2.1.tar.gz') - - version('4.8.6', '2edbe4d6c2eff33ef91732602f3518eb', - url="http://download.qt-project.org/official_releases/qt/4.8/4.8.6/qt-everywhere-opensource-src-4.8.6.tar.gz") - - version('3.3.8b', '9f05b4125cfe477cc52c9742c3c09009', - url="http://download.qt.io/archive/qt/3/qt-x11-free-3.3.8b.tar.gz") - - variant('mesa', default=False, description='depend on mesa') # Add patch for compile issues with qt3 found with use in the OpenSpeedShop project - variant('krellpatch', default=False, description="build with openspeedshop based patch.") + variant('krellpatch', default=False, description="Build with openspeedshop based patch.") + variant('mesa', default=False, description="Depend on mesa.") + variant('gtk', default=False, description="Build with gtkplus.") + patch('qt3krell.patch', when='@3.3.8b+krellpatch') # Use system openssl for security. #depends_on("openssl") depends_on("glib") - depends_on("gtkplus") + depends_on("gtkplus", when='+gtk') depends_on("libxml2") depends_on("zlib") depends_on("dbus", when='@4:') @@ -56,6 +46,34 @@ class Qt(Package): depends_on("libxcb") + def url_for_version(self, version): + url = "http://download.qt.io/archive/qt/" + + if version >= Version('5'): + url += "%s/%s/single/qt-everywhere-opensource-src-%s.tar.gz" % \ + (version.up_to(2), version, version) + elif version >= Version('4.8'): + url += "%s/%s/qt-everywhere-opensource-src-%s.tar.gz" % \ + (version.up_to(2), version, version) + elif version >= Version('4.6'): + url += "%s/qt-everywhere-opensource-src-%s.tar.gz" % \ + (version.up_to(2), version) + elif version >= Version('4.0'): + url += "%s/qt-x11-opensource-src-%s.tar.gz" % \ + (version.up_to(2), version) + elif version >= Version('3'): + url += "%s/qt-x11-free-%s.tar.gz" % \ + (version.up_to(1), version) + elif version >= Version('2.1'): + url += "%s/qt-x11-%s.tar.gz" % \ + (version.up_to(1), version) + else: + url += "%s/qt-%s.tar.gz" % \ + (version.up_to(1), version) + + return url + + def setup_environment(self, spack_env, env): env.set('QTDIR', self.prefix) @@ -88,7 +106,7 @@ def common_config_args(self): '-v', '-opensource', '-opengl', - "-release", + '-release', '-shared', '-confirm-license', '-openssl-linked', @@ -97,7 +115,7 @@ def common_config_args(self): '-no-openvg', '-no-pch', # NIS is deprecated in more recent glibc - "-no-nis"] + '-no-nis'] # Don't disable all the database drivers, but should # really get them into spack at some point. From 87b70e2cb45660c5c037dfce963c5dc946b2daf5 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 31 Mar 2016 14:07:03 -0500 Subject: [PATCH 822/910] remove the part I added to setup-env.sh as this does not work unless the modules package is already built. --- share/spack/setup-env.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 529517ff834..764af684001 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -176,13 +176,3 @@ _spack_pathadd PATH "${_sp_prefix%/}/bin" _sp_sys_type=$(spack-python -c 'print(spack.architecture.sys_type())') _spack_pathadd DK_NODE "${_sp_share_dir%/}/dotkit/$_sp_sys_type" _spack_pathadd MODULEPATH "${_sp_share_dir%/}/modules/$_sp_sys_type" - -# -# Use spack built modules package if available -# for system that does not have it -# - -_modules_install_path=$(command spack location -i modules) -if [ -n "$_modules_install_path" ] ; then - . $_modules_install_path/Modules/init/sh -fi From b1b94d2b7f35e898489b0044beb22df249967548 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Thu, 31 Mar 2016 13:59:49 -0700 Subject: [PATCH 823/910] Added the initial version of the 'zoltan' package. --- .../repos/builtin/packages/zoltan/package.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 var/spack/repos/builtin/packages/zoltan/package.py diff --git a/var/spack/repos/builtin/packages/zoltan/package.py b/var/spack/repos/builtin/packages/zoltan/package.py new file mode 100644 index 00000000000..91d3c305f82 --- /dev/null +++ b/var/spack/repos/builtin/packages/zoltan/package.py @@ -0,0 +1,52 @@ +from spack import * + +class Zoltan(Package): + """The Zoltan library is a toolkit of parallel combinatorial algorithms for + parallel, unstructured, and/or adaptive scientific applications. Zoltan's + largest component is a suite of dynamic load-balancing and paritioning + algorithms that increase applications' parallel performance by reducing + idle time. Zoltan also has graph coloring and graph ordering algorithms, + which are useful in task schedulers and parallel preconditioners.""" + + homepage = "http://www.cs.sandia.gov/zoltan" + base_url = "http://www.cs.sandia.gov/~kddevin/Zoltan_Distributions" + + version('3.83', '1ff1bc93f91e12f2c533ddb01f2c095f') + version('3.3', '5eb8f00bda634b25ceefa0122bd18d65') + + variant('fortran', default=True, description='Enable Fortran support') + variant('mpi', default=False, description='Enable MPI support') + + depends_on('mpi', when='+mpi') + + def install(self, spec, prefix): + config_args = [ + '--enable-f90interface' if '+fortan' in spec else '--disable-f90interface', + '--enable-mpi' if '+mpi' in spec else '--disable-mpi', + ] + + if '+mpi' in spec: + config_args.append('--with-mpi=%s' % spec['mpi'].prefix) + config_args.append('--with-mpi-compilers=%s' % spec['mpi'].prefix.bin) + + # NOTE: Early versions of Zoltan come packaged with a few embedded + # library packages (e.g. ParMETIS, Scotch), which messes with Spack's + # ability to descend directly into the package's source directory. + if spec.satisfies('@:3.3'): + cd('Zoltan_v%s' % self.version) + + mkdirp('build') + cd('build') + + config_zoltan = Executable('../configure') + config_zoltan('--prefix=%s' % pwd(), *config_args) + + make() + make('install') + + mkdirp(prefix) + move('include', prefix) + move('lib', prefix) + + def url_for_version(self, version): + return '%s/zoltan_distrib_v%s.tar.gz' % (Zoltan.base_url, version) From 09d657e98e50fd0194e35cfea39105ea31bd5e55 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Thu, 31 Mar 2016 14:36:32 -0700 Subject: [PATCH 824/910] Fixed the MPI variant for the 'zoltan' package. --- var/spack/repos/builtin/packages/zoltan/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/zoltan/package.py b/var/spack/repos/builtin/packages/zoltan/package.py index 91d3c305f82..e20ae81adbe 100644 --- a/var/spack/repos/builtin/packages/zoltan/package.py +++ b/var/spack/repos/builtin/packages/zoltan/package.py @@ -28,6 +28,8 @@ def install(self, spec, prefix): if '+mpi' in spec: config_args.append('--with-mpi=%s' % spec['mpi'].prefix) config_args.append('--with-mpi-compilers=%s' % spec['mpi'].prefix.bin) + config_args.append('CC=%s/mpicc' % spec['mpi'].prefix.bin) + config_args.append('CXX=%s/mpicxx' % spec['mpi'].prefix.bin) # NOTE: Early versions of Zoltan come packaged with a few embedded # library packages (e.g. ParMETIS, Scotch), which messes with Spack's From d4e6b15d981a6bad77cb6f36cdc3a41adf05525b Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 1 Apr 2016 15:15:23 -0400 Subject: [PATCH 825/910] Fixed TCL/Tk --- var/spack/repos/builtin/packages/tcl/package.py | 7 +++++-- var/spack/repos/builtin/packages/tk/package.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/tcl/package.py b/var/spack/repos/builtin/packages/tcl/package.py index 529adf77883..c2cb09ab46a 100644 --- a/var/spack/repos/builtin/packages/tcl/package.py +++ b/var/spack/repos/builtin/packages/tcl/package.py @@ -9,9 +9,12 @@ class Tcl(Package): that is truly cross platform, easily deployed and highly extensible.""" homepage = "http://www.tcl.tk" + urlpat = 'http://prdownloads.sourceforge.net/tcl/tcl%s-src.tar.gz' - version('8.6.3', 'db382feca91754b7f93da16dc4cdad1f', - url="http://prdownloads.sourceforge.net/tcl/tcl8.6.3-src.tar.gz") + version('8.6.5', '0e6426a4ca9401825fbc6ecf3d89a326', url=urlpat%'8.6.5') + version('8.6.4', 'd7cbb91f1ded1919370a30edd1534304', url=urlpat%'8.6.4') + version('8.6.3', 'db382feca91754b7f93da16dc4cdad1f', url=urlpat%'8.6.3') + version('8.5.19', '0e6426a4ca9401825fbc6ecf3d89a326', url=urlpat%'8.6.5') depends_on('zlib') diff --git a/var/spack/repos/builtin/packages/tk/package.py b/var/spack/repos/builtin/packages/tk/package.py index 96736f6f95d..6da0a423a9d 100644 --- a/var/spack/repos/builtin/packages/tk/package.py +++ b/var/spack/repos/builtin/packages/tk/package.py @@ -10,7 +10,7 @@ class Tk(Package): homepage = "http://www.tcl.tk" url = "http://prdownloads.sourceforge.net/tcl/tk8.6.3-src.tar.gz" - version('src', '85ca4dbf4dcc19777fd456f6ee5d0221') + version('8.6.3', '85ca4dbf4dcc19777fd456f6ee5d0221') depends_on("tcl") From cfd5e69b816eaca1a4d6d4467d5f1113fa0af407 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 1 Apr 2016 15:16:28 -0400 Subject: [PATCH 826/910] Added Environment Modules package. --- .../packages/environment-modules/package.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 var/spack/repos/builtin/packages/environment-modules/package.py diff --git a/var/spack/repos/builtin/packages/environment-modules/package.py b/var/spack/repos/builtin/packages/environment-modules/package.py new file mode 100644 index 00000000000..df903fddf8b --- /dev/null +++ b/var/spack/repos/builtin/packages/environment-modules/package.py @@ -0,0 +1,27 @@ +from spack import * + + +class EnvironmentModules(Package): + """Environment Modules for HPC""" + + homepage = "https://sourceforge.net/p/modules/wiki/Home/" + url = "http://prdownloads.sourceforge.net/modules/modules-3.2.10.tar.gz" + + version('3.2.10', '8b097fdcb90c514d7540bb55a3cb90fb') + + # Dependencies: + depends_on('tcl') + + def install(self, spec, prefix): + # See: https://sourceforge.net/p/modules/bugs/62/ + CPPFLAGS = ['-DUSE_INTERP_ERRORLINE'] + config_args = [ + "--prefix=%s" % prefix, + "--with-tcl=%s" % join_path(spec['tcl'].prefix, 'lib'), # It looks for tclConfig.sh + 'CPPFLAGS=%s' % ' '.join(CPPFLAGS) + ] + + + configure(*config_args) + make() + make("install") From 459aab628d73c7f6f1d4eecbf64b34ca334e0ada Mon Sep 17 00:00:00 2001 From: citibeth Date: Fri, 1 Apr 2016 16:00:58 -0400 Subject: [PATCH 827/910] Added documentation for installing Environment Modules with Spack. --- lib/spack/docs/basic_usage.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index accf09cc2a4..0727e12f6e5 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -774,6 +774,34 @@ Environment modules Spack provides some limited integration with environment module systems to make it easier to use the packages it provides. + +Installing Environment Modules +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In order to use Spack's generated environment modules, you must have +installed the *Environment Modules* package. On many Linux +distributions, this can be installed from the vendor's repository. +For example: ```yum install environment-modules`` +(Fedora/RHEL/CentOS). If your Linux distribution does not have +Environment Modules, you can get it with Spack: + +1. Install with:: + + spack install environment-modules + +2. Activate with:: + + MODULES_HOME=`spack location -i environment-modules` + MODULES_VERSION=`ls -1 $MODULES_HOME/Modules | head -1` + ${MODULES_HOME}/Modules/${MODULES_VERSION}/bin/add.modules + +This adds to your ``.bashrc`` (or similar) files, enabling Environment +Modules when you log in. It will ask your permission before changing +any files. + +Spack and Environment Modules +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + You can enable shell support by sourcing some files in the ``/share/spack`` directory. From e4f7b78e9b4ddf1d4fd7dac4782068f07626d9e0 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 1 Apr 2016 16:36:31 -0400 Subject: [PATCH 828/910] Fixed tk, and used uverridden url_for_version(). --- var/spack/repos/builtin/packages/tcl/package.py | 12 +++++++----- var/spack/repos/builtin/packages/tk/package.py | 4 +++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/tcl/package.py b/var/spack/repos/builtin/packages/tcl/package.py index c2cb09ab46a..db8bee88d0d 100644 --- a/var/spack/repos/builtin/packages/tcl/package.py +++ b/var/spack/repos/builtin/packages/tcl/package.py @@ -9,12 +9,14 @@ class Tcl(Package): that is truly cross platform, easily deployed and highly extensible.""" homepage = "http://www.tcl.tk" - urlpat = 'http://prdownloads.sourceforge.net/tcl/tcl%s-src.tar.gz' - version('8.6.5', '0e6426a4ca9401825fbc6ecf3d89a326', url=urlpat%'8.6.5') - version('8.6.4', 'd7cbb91f1ded1919370a30edd1534304', url=urlpat%'8.6.4') - version('8.6.3', 'db382feca91754b7f93da16dc4cdad1f', url=urlpat%'8.6.3') - version('8.5.19', '0e6426a4ca9401825fbc6ecf3d89a326', url=urlpat%'8.6.5') + def url_for_version(self, version): + return 'http://prdownloads.sourceforge.net/tcl/tcl%s-src.tar.gz' % version + + version('8.6.5', '0e6426a4ca9401825fbc6ecf3d89a326') + version('8.6.4', 'd7cbb91f1ded1919370a30edd1534304') + version('8.6.3', 'db382feca91754b7f93da16dc4cdad1f') + version('8.5.19', '0e6426a4ca9401825fbc6ecf3d89a326') depends_on('zlib') diff --git a/var/spack/repos/builtin/packages/tk/package.py b/var/spack/repos/builtin/packages/tk/package.py index 6da0a423a9d..839d217f344 100644 --- a/var/spack/repos/builtin/packages/tk/package.py +++ b/var/spack/repos/builtin/packages/tk/package.py @@ -8,7 +8,9 @@ class Tk(Package): applications that run unchanged across Windows, Mac OS X, Linux and more.""" homepage = "http://www.tcl.tk" - url = "http://prdownloads.sourceforge.net/tcl/tk8.6.3-src.tar.gz" + + def url_for_version(self, version): + return "http://prdownloads.sourceforge.net/tcl/tk%s-src.tar.gz" % version version('8.6.3', '85ca4dbf4dcc19777fd456f6ee5d0221') From 71ffe2f1c40edaa311a8b7d25d87d2c3ca25388f Mon Sep 17 00:00:00 2001 From: Geoffrey Oxberry Date: Thu, 31 Mar 2016 17:11:57 -0700 Subject: [PATCH 829/910] mumps: install mpiseq headers for '~mpi' Prior to this commit, spack installs a library called `libmpiseq` into `spec['mumps'].prefix.lib` when it builds MUMPS without MPI. However, it does not also install the headers corresponding to this library, so it is impossible to compile source files that depend on function calls implemented in this library. This commit fixes this problem by installing these headers, which are needed by packages (e.g., IPOPT) that depend on these headers. --- var/spack/repos/builtin/packages/mumps/package.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index 26440ab7c81..025d86ebdc2 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -1,5 +1,5 @@ from spack import * -import os, sys +import os, sys, glob class Mumps(Package): """MUMPS: a MUltifrontal Massively Parallel sparse direct Solver""" @@ -164,10 +164,13 @@ def install(self, spec, prefix): install_tree('lib', prefix.lib) install_tree('include', prefix.include) - if '~mpi' in spec: + + if '~mpi' in spec: lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' lib_suffix = lib_dsuffix if '+shared' in spec else '.a' install('libseq/libmpiseq%s' % lib_suffix, prefix.lib) + for f in glob.glob(join_path('libseq','*.h')): + install(f, prefix.include) # FIXME: extend the tests to mpirun -np 2 (or alike) when build with MPI # FIXME: use something like numdiff to compare blessed output with the current From edf6e9ceacab9aa2d8795340089182ead07c30b3 Mon Sep 17 00:00:00 2001 From: Geoffrey Oxberry Date: Thu, 31 Mar 2016 15:25:32 -0700 Subject: [PATCH 830/910] Add ipopt v3.12.4 package. Add package for the nonlinear programming solver IPOPT. This commit depends on the changes made in GitHub PR #711, since IPOPT requires the headers that correspond to the fake MPI implementation in sequential MUMPS. --- .../repos/builtin/packages/ipopt/package.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 var/spack/repos/builtin/packages/ipopt/package.py diff --git a/var/spack/repos/builtin/packages/ipopt/package.py b/var/spack/repos/builtin/packages/ipopt/package.py new file mode 100644 index 00000000000..13c37bf79c4 --- /dev/null +++ b/var/spack/repos/builtin/packages/ipopt/package.py @@ -0,0 +1,51 @@ +from spack import * + +class Ipopt(Package): + """Ipopt (Interior Point OPTimizer, pronounced eye-pea-Opt) is a + software package for large-scale nonlinear optimization.""" + homepage = "https://projects.coin-or.org/Ipopt" + url = "http://www.coin-or.org/download/source/Ipopt/Ipopt-3.12.4.tgz" + + version('3.12.4', '12a8ecaff8dd90025ddea6c65b49cb03') + version('3.12.3', 'c560cbfa9cbf62acf8b485823c255a1b') + version('3.12.2', 'ec1e855257d7de09e122c446506fb00d') + version('3.12.1', 'ceaf895ce80c77778f2cab68ba9f17f3') + version('3.12.0', 'f7dfc3aa106a6711a85214de7595e827') + + depends_on("blas") + depends_on("lapack") + depends_on("pkg-config") + depends_on("mumps+double~mpi") + + def install(self, spec, prefix): + # Dependency directories + blas_dir = spec['blas'].prefix + lapack_dir = spec['lapack'].prefix + mumps_dir = spec['mumps'].prefix + + # Add directory with fake MPI headers in sequential MUMPS + # install to header search path + mumps_flags = "-ldmumps -lmumps_common -lpord -lmpiseq" + mumps_libcmd = "-L%s " % mumps_dir.lib + mumps_flags + + # By convention, spack links blas & lapack libs to libblas & liblapack + blas_lib = "-L%s" % blas_dir.lib + " -lblas" + lapack_lib = "-L%s" % lapack_dir.lib + " -llapack" + + configure_args = [ + "--prefix=%s" % prefix, + "--with-mumps-incdir=%s" % mumps_dir.include, + "--with-mumps-lib=%s" % mumps_libcmd, + "--enable-shared", + "--with-blas-incdir=%s" % blas_dir.include, + "--with-blas-lib=%s" % blas_lib, + "--with-lapack-incdir=%s" % lapack_dir.include, + "--with-lapack-lib=%s" % lapack_lib + ] + + configure(*configure_args) + + # IPOPT does not build correctly in parallel on OS X + make(parallel=False) + make("test", parallel=False) + make("install", parallel=False) From 624b576b1e21ea476eaf7fdc3ee38a97194e45f7 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sat, 2 Apr 2016 12:01:46 +0200 Subject: [PATCH 831/910] metis: fix compilation with Clang 7.3.0, add tests --- .../repos/builtin/packages/metis/package.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py index 9301135f9f3..d3bab554fec 100644 --- a/var/spack/repos/builtin/packages/metis/package.py +++ b/var/spack/repos/builtin/packages/metis/package.py @@ -79,10 +79,28 @@ def install(self, spec, prefix): if '+double' in spec: filter_file('REALTYPEWIDTH 32', 'REALTYPEWIDTH 64', metis_header) + # Make clang 7.3 happy. + # Prevents "ld: section __DATA/__thread_bss extends beyond end of file" + # See upstream LLVM issue https://llvm.org/bugs/show_bug.cgi?id=27059 + # Adopted from https://github.com/Homebrew/homebrew-science/blob/master/metis.rb + if spec.satisfies('%clang@7.3.0'): + filter_file('#define MAX_JBUFS 128', '#define MAX_JBUFS 24', join_path(source_directory, 'GKlib', 'error.c')) + with working_dir(build_directory, create=True): cmake(source_directory, *options) make() make("install") + # now run some tests: + for f in ["4elt", "copter2", "mdual"]: + graph = join_path(source_directory,'graphs','%s.graph' % f) + Executable(join_path(prefix.bin,'graphchk'))(graph) + Executable(join_path(prefix.bin,'gpmetis'))(graph,'2') + Executable(join_path(prefix.bin,'ndmetis'))(graph) + + graph = join_path(source_directory,'graphs','test.mgraph') + Executable(join_path(prefix.bin,'gpmetis'))(graph,'2') + graph = join_path(source_directory,'graphs','metis.mesh') + Executable(join_path(prefix.bin,'mpmetis'))(graph,'2') # install GKlib headers, which will be needed for ParMETIS GKlib_dist = join_path(prefix.include,'GKlib') From a88c6da9acc234484d34acf6588c838bef09e8c4 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sat, 2 Apr 2016 12:02:29 +0200 Subject: [PATCH 832/910] suite-sparse: fix a bug where interla metis was used; add TBB variant --- .../builtin/packages/suite-sparse/package.py | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/suite-sparse/package.py b/var/spack/repos/builtin/packages/suite-sparse/package.py index c2196dcec4d..f2e71f74791 100644 --- a/var/spack/repos/builtin/packages/suite-sparse/package.py +++ b/var/spack/repos/builtin/packages/suite-sparse/package.py @@ -10,10 +10,13 @@ class SuiteSparse(Package): version('4.5.1', 'f0ea9aad8d2d1ffec66a5b6bfeff5319') + variant('tbb', default=True, description='Build with Intel TBB') + depends_on('blas') depends_on('lapack') depends_on('metis@5.1.0', when='@4.5.1') + depends_on('tbb', when='+tbb') def install(self, spec, prefix): # The build system of SuiteSparse is quite old-fashioned @@ -21,16 +24,35 @@ def install(self, spec, prefix): # with a lot of convoluted logic in it. # Any kind of customization will need to go through filtering of that file - # FIXME : this actually uses the current workaround - # FIXME : (blas / lapack always provide libblas and liblapack as aliases) - make('install', 'INSTALL=%s' % prefix, + make_args = ['INSTALL=%s' % prefix] - # inject Spack compiler wrappers + # inject Spack compiler wrappers + make_args.extend([ 'AUTOCC=no', 'CC=cc', 'CXX=c++', 'F77=f77', + ]) - # BLAS arguments require path to libraries - 'BLAS=-lblas', - 'LAPACK=-llapack') + # use Spack's metis in CHOLMOD/Partition module, + # otherwise internal Metis will be compiled + make_args.extend([ + 'MY_METIS_LIB=-L%s -lmetis' % spec['metis'].prefix.lib, + 'MY_METIS_INC=%s' % spec['metis'].prefix.include, + ]) + + # Intel TBB in SuiteSparseQR + if '+tbb' in spec: + make_args.extend([ + 'SPQR_CONFIG=-DHAVE_TBB', + 'TBB=-L%s -ltbb' % spec['tbb'].prefix.lib, + ]) + + # BLAS arguments require path to libraries + # FIXME : (blas / lapack always provide libblas and liblapack as aliases) + make_args.extend([ + 'BLAS=-lblas', + 'LAPACK=-llapack' + ]) + + make('install', *make_args) From adccd18015eec2589d69194fa30be45b04cb515e Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sat, 2 Apr 2016 12:02:59 +0200 Subject: [PATCH 833/910] dealii: add extra test for Petsc+Metis --- var/spack/repos/builtin/packages/dealii/package.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 0b76db3827a..d8a0df643de 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -174,6 +174,19 @@ def install(self, spec, prefix): make('release') make('run',parallel=False) + # An example which uses Metis + PETSc + # FIXME: switch step-18 to MPI + with working_dir('examples/step-18'): + print('=====================================') + print('============= Step-18 ===============') + print('=====================================') + # list the number of cycles to speed up + filter_file(r'(end_time = 10;)', ('end_time = 3;'), 'step-18.cc') + if '^petsc' in spec and '^metis' in spec: + cmake('.') + make('release') + make('run',parallel=False) + # take step-40 which can use both PETSc and Trilinos # FIXME: switch step-40 to MPI run with working_dir('examples/step-40'): From 2126683203f4b8bf88aac539bac8bd8643f5bee1 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sat, 2 Apr 2016 12:43:19 -0400 Subject: [PATCH 834/910] Added deprecation warning to netcdf-cxx --- var/spack/repos/builtin/packages/netcdf-cxx/package.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/netcdf-cxx/package.py b/var/spack/repos/builtin/packages/netcdf-cxx/package.py index 5334dfb853a..582a8e70b44 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx/package.py @@ -1,7 +1,11 @@ from spack import * class NetcdfCxx(Package): - """C++ compatibility bindings for NetCDF""" + “””Deprecated C++ compatibility bindings for NetCDF. + These do NOT read or write NetCDF-4 files, and are no longer + maintained by Unidata. Developers should migrate to current + NetCDF C++ bindings, in Spack package netcdf-cxx4.””” + homepage = "http://www.unidata.ucar.edu/software/netcdf" url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx-4.2.tar.gz" From 9f7db44574093aa9490ffe1a4ff959213b096008 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sat, 2 Apr 2016 12:58:37 -0400 Subject: [PATCH 835/910] Removed useless url in qt --- var/spack/repos/builtin/packages/qt/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index ab094690422..8cb88e6c854 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -4,7 +4,6 @@ class Qt(Package): """Qt is a comprehensive cross-platform C++ application framework.""" homepage = 'http://qt.io' - url = 'http://download.qt.io/archive/qt/5.5/5.5.1/single/qt-everywhere-opensource-src-5.5.1.tar.gz' version('5.5.1', '59f0216819152b77536cf660b015d784') version('5.4.2', 'fa1c4d819b401b267eb246a543a63ea5') From 1c483ffe4e9b0dc03d469eba9c064461624a81be Mon Sep 17 00:00:00 2001 From: citibeth Date: Sat, 2 Apr 2016 13:09:56 -0400 Subject: [PATCH 836/910] Merged the best of two Spack packages for Environment Modules. Named "environment-modules" to be consistent with Fedora and Ubuntu repositories. See d96ea5b for original packages/modules/package.py --- .../packages/environment-modules/package.py | 14 ++++++++--- .../repos/builtin/packages/modules/package.py | 25 ------------------- 2 files changed, 10 insertions(+), 29 deletions(-) delete mode 100644 var/spack/repos/builtin/packages/modules/package.py diff --git a/var/spack/repos/builtin/packages/environment-modules/package.py b/var/spack/repos/builtin/packages/environment-modules/package.py index df903fddf8b..cc5d76760ee 100644 --- a/var/spack/repos/builtin/packages/environment-modules/package.py +++ b/var/spack/repos/builtin/packages/environment-modules/package.py @@ -2,7 +2,8 @@ class EnvironmentModules(Package): - """Environment Modules for HPC""" + """The Environment Modules package provides for the dynamic + modification of a user's environment via modulefiles.""" homepage = "https://sourceforge.net/p/modules/wiki/Home/" url = "http://prdownloads.sourceforge.net/modules/modules-3.2.10.tar.gz" @@ -16,12 +17,17 @@ def install(self, spec, prefix): # See: https://sourceforge.net/p/modules/bugs/62/ CPPFLAGS = ['-DUSE_INTERP_ERRORLINE'] config_args = [ - "--prefix=%s" % prefix, - "--with-tcl=%s" % join_path(spec['tcl'].prefix, 'lib'), # It looks for tclConfig.sh + '--prefix=%s' % prefix, + '--disable-debug', + '--disable-dependency-tracking', + '--disable-silent-rules', + '--disable-versioning', + '--datarootdir=%s' % prefix.share, + '--with-tcl=%s' % join_path(spec['tcl'].prefix, 'lib'), # It looks for tclConfig.sh 'CPPFLAGS=%s' % ' '.join(CPPFLAGS) ] configure(*config_args) make() - make("install") + make('install') diff --git a/var/spack/repos/builtin/packages/modules/package.py b/var/spack/repos/builtin/packages/modules/package.py deleted file mode 100644 index b014ee460c2..00000000000 --- a/var/spack/repos/builtin/packages/modules/package.py +++ /dev/null @@ -1,25 +0,0 @@ -from spack import * - -class Modules(Package): - """ The Environment Modules package provides for the dynamic modification of a user's environment via modulefiles. """ - - homepage = "http://modules.sf.net" - url = "http://downloads.sourceforge.net/project/modules/Modules/modules-3.2.10/modules-3.2.10.tar.gz" - - version('3.2.10', '8b097fdcb90c514d7540bb55a3cb90fb') - - depends_on("tcl") - - def install(self, spec, prefix): - - options = ['--prefix=%s' % prefix, - '--disable-debug', - '--disable-dependency-tracking', - '--disable-silent-rules', - '--disable-versioning', - '--datarootdir=%s' % prefix.share, - 'CPPFLAGS=-DUSE_INTERP_ERRORLINE'] - - configure(*options) - make() - make("install") From c952c0ca160c31123104f68c4d1ac732bc473ede Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 2 Apr 2016 15:50:20 -0500 Subject: [PATCH 837/910] Added version 1.0 of sympy. Beginning with this version, sympy requires the mpmath package. The py-mpmath package is added in this PR to accommodate that. --- .../repos/builtin/packages/py-mpmath/package.py | 13 +++++++++++++ .../repos/builtin/packages/py-sympy/package.py | 2 ++ 2 files changed, 15 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-mpmath/package.py diff --git a/var/spack/repos/builtin/packages/py-mpmath/package.py b/var/spack/repos/builtin/packages/py-mpmath/package.py new file mode 100644 index 00000000000..4d3261ae8f9 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-mpmath/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyMpmath(Package): + """A Python library for arbitrary-precision floating-point arithmetic.""" + homepage = "http://mpmath.org" + url = "https://pypi.python.org/packages/source/m/mpmath/mpmath-all-0.19.tar.gz" + + version('0.19', 'd1b7e19dd6830d0d7b5e1bc93d46c02c') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-sympy/package.py b/var/spack/repos/builtin/packages/py-sympy/package.py index c17e35b95f1..bbce8c74e3c 100644 --- a/var/spack/repos/builtin/packages/py-sympy/package.py +++ b/var/spack/repos/builtin/packages/py-sympy/package.py @@ -6,8 +6,10 @@ class PySympy(Package): url = "https://pypi.python.org/packages/source/s/sympy/sympy-0.7.6.tar.gz" version('0.7.6', '3d04753974306d8a13830008e17babca') + version('1.0', '43e797de799f00f9e8fd2307dba9fab1') extends('python') + depends_on('py-mpmath', when='@1.0:') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) From e436f84244278f842822051a4049ef8418507709 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 2 Apr 2016 20:21:56 -0400 Subject: [PATCH 838/910] Update HDF5 to 1.10.0 --- var/spack/repos/builtin/packages/hdf5/package.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 513a38ee8a4..1bf01e7423a 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -37,6 +37,8 @@ class Hdf5(Package): list_url = "http://www.hdfgroup.org/ftp/HDF5/releases" list_depth = 3 + version('1.10.0', 'bdc935337ee8282579cd6bc4270ad199', + url='http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.0/src/hdf5-1.10.0.tar.gz') version('1.8.16', 'b8ed9a36ae142317f88b0c7ef4b9c618') version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') version('1.8.13', 'c03426e9e77d7766944654280b467289') @@ -80,10 +82,16 @@ def install(self, spec, prefix): # sanity check in configure, so this doesn't merit a variant. extra_args.append("--enable-unsupported") - if '+debug' in spec: - extra_args.append('--enable-debug=all') + if spec.satisfies('@1.10:'): + if '+debug' in spec: + extra_args.append('--enable-build-mode=debug') + else: + extra_args.append('--enable-build-mode=production') else: - extra_args.append('--enable-production') + if '+debug' in spec: + extra_args.append('--enable-debug=all') + else: + extra_args.append('--enable-production') if '+shared' in spec: extra_args.append('--enable-shared') From 40c279ef95a92ef3274fda7ae8dbff4c790ff4cd Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Sun, 3 Apr 2016 00:10:22 -0400 Subject: [PATCH 839/910] Fixed evil non-ASCII quotes --- var/spack/repos/builtin/packages/netcdf-cxx/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf-cxx/package.py b/var/spack/repos/builtin/packages/netcdf-cxx/package.py index 582a8e70b44..8aa1d8b236e 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx/package.py @@ -1,10 +1,10 @@ from spack import * class NetcdfCxx(Package): - “””Deprecated C++ compatibility bindings for NetCDF. + """Deprecated C++ compatibility bindings for NetCDF. These do NOT read or write NetCDF-4 files, and are no longer maintained by Unidata. Developers should migrate to current - NetCDF C++ bindings, in Spack package netcdf-cxx4.””” + NetCDF C++ bindings, in Spack package netcdf-cxx4.""" homepage = "http://www.unidata.ucar.edu/software/netcdf" url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx-4.2.tar.gz" From e0ff3f16cc142fb9ca55cc1c6000fb230627f8fe Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sun, 3 Apr 2016 19:41:20 +0200 Subject: [PATCH 840/910] dealii: blacklist boost --- var/spack/repos/builtin/packages/dealii/package.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index d8a0df643de..1f763ad3588 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -23,8 +23,10 @@ class Dealii(Package): # required dependencies, light version depends_on ("blas") - depends_on ("boost", when='~mpi') - depends_on ("boost+mpi", when='+mpi') + # Boost 1.58 is blacklisted, see https://github.com/dealii/dealii/issues/1591 + # require at least 1.59 + depends_on ("boost@1.59.0:", when='~mpi') + depends_on ("boost@1.59.0:+mpi", when='+mpi') depends_on ("bzip2") depends_on ("cmake") depends_on ("lapack") From 79836520d989a19bc49b912698b22fdc8a65ca59 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sun, 3 Apr 2016 21:38:29 -0400 Subject: [PATCH 841/910] Fixed up to work on a machine without any system-install tcl. --- .../builtin/packages/environment-modules/package.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/environment-modules/package.py b/var/spack/repos/builtin/packages/environment-modules/package.py index cc5d76760ee..45181da41bf 100644 --- a/var/spack/repos/builtin/packages/environment-modules/package.py +++ b/var/spack/repos/builtin/packages/environment-modules/package.py @@ -14,16 +14,21 @@ class EnvironmentModules(Package): depends_on('tcl') def install(self, spec, prefix): + tcl_spec = spec['tcl'] + # See: https://sourceforge.net/p/modules/bugs/62/ CPPFLAGS = ['-DUSE_INTERP_ERRORLINE'] config_args = [ - '--prefix=%s' % prefix, + "--without-tclx", + "--with-tclx-ver=0.0", + "--prefix=%s" % prefix, + "--with-tcl=%s" % join_path(tcl_spec.prefix, 'lib'), # It looks for tclConfig.sh + "--with-tcl-ver=%d.%d" % (tcl_spec.version.version[0], tcl_spec.version.version[1]), '--disable-debug', '--disable-dependency-tracking', '--disable-silent-rules', '--disable-versioning', '--datarootdir=%s' % prefix.share, - '--with-tcl=%s' % join_path(spec['tcl'].prefix, 'lib'), # It looks for tclConfig.sh 'CPPFLAGS=%s' % ' '.join(CPPFLAGS) ] From 2f4d8a634d19debbe069f3c44b4d2d6822f693cb Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 3 Apr 2016 15:45:09 -0400 Subject: [PATCH 842/910] Fix conditional extends (BUG #683) --- lib/spack/spack/modules.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index f6a11c92e31..d797af287d3 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -163,9 +163,14 @@ def write(self): # package-specific modifications spack_env = EnvironmentModifications() for item in self.pkg.extendees: - package = self.spec[item].package - package.setup_dependent_package(self.pkg.module, self.spec) - package.setup_dependent_environment(spack_env, env, self.spec) + try: + package = self.spec[item].package + package.setup_dependent_package(self.pkg.module, self.spec) + package.setup_dependent_environment(spack_env, env, self.spec) + except: + # The extends was conditional, so it doesn't count here + # eg: extends('python', when='+python') + pass # Package-specific environment modifications self.spec.package.setup_environment(spack_env, env) From c3f7b035fdf82c6a9d1d078f9c8c4f36a4158d99 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 3 Apr 2016 16:34:45 -0400 Subject: [PATCH 843/910] dbus: Added missing dependency (expat) --- var/spack/repos/builtin/packages/dbus/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/dbus/package.py b/var/spack/repos/builtin/packages/dbus/package.py index 422f5a19eba..74ce8ef502d 100644 --- a/var/spack/repos/builtin/packages/dbus/package.py +++ b/var/spack/repos/builtin/packages/dbus/package.py @@ -20,6 +20,8 @@ class Dbus(Package): version('1.8.4', '4717cb8ab5b80978fcadf2b4f2f72e1b') version('1.8.2', 'd6f709bbec0a022a1847c7caec9d6068') + depends_on('expat') + def install(self, spec, prefix): configure( "--prefix=%s" % prefix, From 2809fe95db84956b93824402cf4e00d1c437f14e Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 3 Apr 2016 12:14:38 -0400 Subject: [PATCH 844/910] git: added comment about newly discovered dependency --- var/spack/repos/builtin/packages/git/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/git/package.py b/var/spack/repos/builtin/packages/git/package.py index 586b6ce3c39..388f84aefdb 100644 --- a/var/spack/repos/builtin/packages/git/package.py +++ b/var/spack/repos/builtin/packages/git/package.py @@ -36,6 +36,8 @@ class Git(Package): depends_on("curl", when="+curl") depends_on("expat", when="+expat") + # Also depends_on gettext: apt-get install gettext (Ubuntu) + # Use system perl for now. # depends_on("perl") # depends_on("pcre") From 0bf61ad57be3d7c3b301639b5af0c196820ec96d Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 3 Apr 2016 16:35:36 -0400 Subject: [PATCH 845/910] libxcb: Added comments on two dependencies Spack does not yet know how to build. --- var/spack/repos/builtin/packages/libxcb/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/libxcb/package.py b/var/spack/repos/builtin/packages/libxcb/package.py index 1dd5954c997..d7d94c4546e 100644 --- a/var/spack/repos/builtin/packages/libxcb/package.py +++ b/var/spack/repos/builtin/packages/libxcb/package.py @@ -14,6 +14,9 @@ class Libxcb(Package): depends_on("python") depends_on("xcb-proto") + # depends_on('pthread') # Ubuntu: apt-get install libpthread-stubs0-dev + # depends_on('xau') # Ubuntu: apt-get install libxau-dev + def patch(self): filter_file('typedef struct xcb_auth_info_t {', 'typedef struct {', 'src/xcb.h') From 14f073d410abb599801fb933e5518fdc43cea7ef Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 3 Apr 2016 15:46:53 -0400 Subject: [PATCH 846/910] py-pillow: Updated for variants, but still having trouble getting it to use Spack-supplied libjpeg. --- .../builtin/packages/py-pillow/package.py | 63 ++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/py-pillow/package.py b/var/spack/repos/builtin/packages/py-pillow/package.py index adc8507bd57..66d9bb43824 100644 --- a/var/spack/repos/builtin/packages/py-pillow/package.py +++ b/var/spack/repos/builtin/packages/py-pillow/package.py @@ -1,4 +1,5 @@ from spack import * +import os class PyPillow(Package): """Pillow is the friendly PIL fork by Alex Clark and Contributors. PIL is the Python Imaging Library by Fredrik Lundh and Contributors. The Python Imaging Library (PIL) adds image processing capabilities to your Python interpreter. This library supports many file formats, and provides powerful image processing and graphics capabilities.""" @@ -7,8 +8,68 @@ class PyPillow(Package): url = "https://pypi.python.org/packages/source/P/Pillow/Pillow-3.0.0.tar.gz" version('3.0.0', 'fc8ac44e93da09678eac7e30c9b7377d') + provides('PIL') + + # These defaults correspond to Pillow defaults + variant('jpeg', default=True, description='Provide JPEG functionality') + variant('zlib', default=True, description='Access to compressed PNGs') + variant('tiff', default=False, description='Access to TIFF files') + variant('freetype', default=False, description='Font related services') + variant('tk', default=False, description='Support for tkinter bitmap and photo images') + variant('lcms', default=False, description='Color management') + + # Spack does not (yet) support these modes of building + # variant('webp', default=False, description='') + # variant('webpmux', default=False, description='') + # variant('jpeg2000', default=False, description='') + extends('python') + depends_on('binutils') depends_on('py-setuptools') + depends_on('jpeg', when='+jpeg') # BUG: It will use the system libjpeg anyway + depends_on('zlib', when='+zlib') + depends_on('tiff', when='+tiff') + depends_on('freetype', when='+freetype') + depends_on('lcms', when='+lcms') + depends_on('tcl', when='+tk') + depends_on('tk', when='+tk') + def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) + libpath=[] + + if '+jpeg' in spec: + libpath.append(join_path(spec['jpeg'].prefix, 'lib')) + if '+zlib' in spec: + libpath.append(join_path(spec['zlib'].prefix, 'lib')) + if '+tiff' in spec: + libpath.append(join_path(spec['tiff'].prefix, 'lib')) + if '+freetype' in spec: + libpath.append(join_path(spec['freetype'].prefix, 'lib')) + if '+lcms' in spec: + libpath.append(join_path(spec['lcms'].prefix, 'lib')) + + # This has not been tested, and likely needs some other treatment. + #if '+tk' in spec: + # libpath.append(join_path(spec['tcl'].prefix, 'lib')) + # libpath.append(join_path(spec['tk'].prefix, 'lib')) + + # -------- Building + cmd = ['build_ext', + '--%s-jpeg' % ('enable' if '+jpeg' in spec else 'disable'), + '--%s-zlib' % ('enable' if '+zlib' in spec else 'disable'), + '--%s-tiff' % ('enable' if '+tiff' in spec else 'disable'), + '--%s-freetype' % ('enable' if '+freetype' in spec else 'disable'), + '--%s-lcms' % ('enable' if '+lcms' in spec else 'disable'), + '-L'+':'.join(libpath) # NOTE: This does not make it find libjpeg + ] + + #if '+tk' in spec: + # cmd.extend(['--enable-tcl', '--enable-tk']) + #else: + # cmd.extend(['--disable-tcl', '--disable-tk']) + + # --------- Installation + cmd.extend(['install', '--prefix=%s' % prefix]) + + python('setup.py', *cmd) From 131d34f318a9955ff05c35eb8de0b399c31d5cc2 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 4 Apr 2016 09:05:27 +0200 Subject: [PATCH 847/910] suite-sparse: temporary disable TBB due to linking errors --- var/spack/repos/builtin/packages/suite-sparse/package.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/suite-sparse/package.py b/var/spack/repos/builtin/packages/suite-sparse/package.py index f2e71f74791..a4b3979a157 100644 --- a/var/spack/repos/builtin/packages/suite-sparse/package.py +++ b/var/spack/repos/builtin/packages/suite-sparse/package.py @@ -10,13 +10,18 @@ class SuiteSparse(Package): version('4.5.1', 'f0ea9aad8d2d1ffec66a5b6bfeff5319') - variant('tbb', default=True, description='Build with Intel TBB') + # FIXME: (see below) + # variant('tbb', default=True, description='Build with Intel TBB') depends_on('blas') depends_on('lapack') depends_on('metis@5.1.0', when='@4.5.1') - depends_on('tbb', when='+tbb') + # FIXME: + # in @4.5.1. TBB support in SPQR seems to be broken as TBB-related linkng flags + # does not seem to be used, which leads to linking errors on Linux. + # Try re-enabling in future versions. + # depends_on('tbb', when='+tbb') def install(self, spec, prefix): # The build system of SuiteSparse is quite old-fashioned From 401dcb363539409e7b94d8ce016bc1a2e70db3a1 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 4 Apr 2016 10:28:47 +0200 Subject: [PATCH 848/910] uninstall : renamed `--recursive` to `--dependents` --- lib/spack/spack/cmd/uninstall.py | 6 +++--- lib/spack/spack/test/cmd/uninstall.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 7aadd254e85..231c6fe6615 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -62,7 +62,7 @@ def setup_parser(subparser): "supplied spec. i.e., if you say uninstall libelf, ALL versions of " + "libelf are uninstalled. This is both useful and dangerous, like rm -r.") subparser.add_argument( - '-r', '--recursive', action='store_true', dest='recursive', + '-d', '--dependents', action='store_true', dest='dependents', help='Also uninstall any packages that depend on the ones given via command line.' ) subparser.add_argument( @@ -168,7 +168,7 @@ def uninstall(parser, args): # Process dependent_list and update uninstall_list has_error = False - if dependent_list and not args.recursive and not args.force: + if dependent_list and not args.dependents and not args.force: for spec, lst in dependent_list.items(): tty.error("Will not uninstall %s" % spec.format("$_$@$%@$#", color=True)) print('') @@ -176,7 +176,7 @@ def uninstall(parser, args): display_specs(lst, long=True) print('') has_error = True - elif args.recursive: + elif args.dependents: for key, lst in dependent_list.items(): uninstall_list.extend(lst) uninstall_list = list(set(uninstall_list)) diff --git a/lib/spack/spack/test/cmd/uninstall.py b/lib/spack/spack/test/cmd/uninstall.py index 06a24e2958b..80efe06d36e 100644 --- a/lib/spack/spack/test/cmd/uninstall.py +++ b/lib/spack/spack/test/cmd/uninstall.py @@ -4,11 +4,11 @@ class MockArgs(object): - def __init__(self, packages, all=False, force=False, recursive=False): + def __init__(self, packages, all=False, force=False, dependents=False): self.packages = packages self.all = all self.force = force - self.recursive = recursive + self.dependents = dependents self.yes_to_all = True @@ -22,7 +22,7 @@ def test_uninstall(self): args = MockArgs(['libelf']) self.assertRaises(SystemExit, uninstall, parser, args) # Recursive uninstall - args = MockArgs(['callpath'], all=True, recursive=True) + args = MockArgs(['callpath'], all=True, dependents=True) uninstall(parser, args) all_specs = spack.install_layout.all_specs() From f40b0f52e0c8b8f076c2ab361edfeba9bc6768fb Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 4 Apr 2016 10:59:01 +0200 Subject: [PATCH 849/910] uninstall : updated documentation and error messages --- lib/spack/docs/basic_usage.rst | 42 +++++++++++++++++++++++--------- lib/spack/spack/cmd/uninstall.py | 9 +++---- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index accf09cc2a4..72a02802fb4 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -149,26 +149,46 @@ customize an installation in :ref:`sec-specs`. ``spack uninstall`` ~~~~~~~~~~~~~~~~~~~~~ -To uninstall a package, type ``spack uninstall ``. This will -completely remove the directory in which the package was installed. +To uninstall a package, type ``spack uninstall ``. This will ask the user for +confirmation, and in case will completely remove the directory in which the package was installed. .. code-block:: sh spack uninstall mpich If there are still installed packages that depend on the package to be -uninstalled, spack will refuse to uninstall it. You can override this -behavior with ``spack uninstall -f ``, but you risk breaking -other installed packages. In general, it is safer to remove dependent -packages *before* removing their dependencies. +uninstalled, spack will refuse to uninstall it. -A line like ``spack uninstall mpich`` may be ambiguous, if multiple -``mpich`` configurations are installed. For example, if both +To uninstall a package and every package that depends on it, you may give the +`--dependents` option. + +.. code-block:: sh + + spack uninstall --dependents mpich + +will display a list of all the packages that depends on `mpich` and, upon confirmation, +will uninstall them in the right order. + +A line like + +.. code-block:: sh + + spack uninstall mpich + +may be ambiguous, if multiple ``mpich`` configurations are installed. For example, if both ``mpich@3.0.2`` and ``mpich@3.1`` are installed, ``mpich`` could refer to either one. Because it cannot determine which one to uninstall, -Spack will ask you to provide a version number to remove the -ambiguity. As an example, ``spack uninstall mpich@3.1`` is -unambiguous in this scenario. +Spack will ask you either to provide a version number to remove the +ambiguity or use the ``--all`` option to uninstall all of the matching packages. + +You may force uninstall a package with the `--force` option + +.. code-block:: sh + + spack uninstall --force mpich + +but you risk breaking other installed packages. In general, it is safer to remove dependent +packages *before* removing their dependencies or use the `--dependents` option. Seeing installed packages diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 231c6fe6615..1ff3d8db5f1 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -24,7 +24,6 @@ ############################################################################## from __future__ import print_function -import sys import argparse import llnl.util.tty as tty @@ -48,8 +47,8 @@ def ask_for_confirmation(message): if choice == 'y': break elif choice == 'n': - sys.exit(1) - tty.warning('Please reply either "y" or "n"') + raise SystemExit('Operation aborted') + tty.warn('Please reply either "y" or "n"') def setup_parser(subparser): @@ -164,7 +163,7 @@ def uninstall(parser, args): specs = spack.cmd.parse_specs(args.packages) # Gets the list of installed specs that match the ones give via cli uninstall_list = concretize_specs(specs, args.all, args.force) # takes care of '-a' is given in the cli - dependent_list = installed_dependents(uninstall_list) # takes care of '-r' + dependent_list = installed_dependents(uninstall_list) # takes care of '-d' # Process dependent_list and update uninstall_list has_error = False @@ -182,7 +181,7 @@ def uninstall(parser, args): uninstall_list = list(set(uninstall_list)) if has_error: - tty.die('You can use spack uninstall -r to uninstall these dependencies as well') + tty.die('You can use spack uninstall --dependents to uninstall these dependencies as well') if not args.yes_to_all: tty.msg("The following packages will be uninstalled : ") From bb968fc5a2bf2ceb585676646f68ec2029a298b1 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 4 Apr 2016 02:52:38 -0700 Subject: [PATCH 850/910] Fix #620, Resolve #664. Fix issues with build environment. - Also added better regression tests for build environment. --- lib/spack/spack/build_environment.py | 41 ++++----- lib/spack/spack/test/install.py | 29 ++++-- .../packages/cmake-client/package.py | 89 +++++++++++++++++++ .../builtin.mock/packages/cmake/package.py | 69 ++++++++++++++ 4 files changed, 197 insertions(+), 31 deletions(-) create mode 100644 var/spack/repos/builtin.mock/packages/cmake-client/package.py create mode 100644 var/spack/repos/builtin.mock/packages/cmake/package.py diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 640db0c1d16..f4f8037ac09 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -213,7 +213,7 @@ def set_module_variables_for_package(pkg, module): # TODO: of build dependencies, as opposed to link dependencies. # TODO: Currently, everything is a link dependency, but tools like # TODO: this shouldn't be. - m.cmake = which("cmake") + m.cmake = Executable('cmake') # standard CMake arguments m.std_cmake_args = ['-DCMAKE_INSTALL_PREFIX=%s' % pkg.prefix, @@ -278,21 +278,6 @@ def parent_class_modules(cls): return result -def setup_module_variables_for_dag(pkg): - """Set module-scope variables for all packages in the DAG.""" - for spec in pkg.spec.traverse(order='post'): - # If a user makes their own package repo, e.g. - # spack.repos.mystuff.libelf.Libelf, and they inherit from - # an existing class like spack.repos.original.libelf.Libelf, - # then set the module variables for both classes so the - # parent class can still use them if it gets called. - spkg = spec.package - modules = parent_class_modules(spkg.__class__) - for mod in modules: - set_module_variables_for_package(spkg, mod) - set_module_variables_for_package(spkg, spkg.module) - - def setup_package(pkg): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() @@ -316,20 +301,26 @@ def setup_package(pkg): set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env) - setup_module_variables_for_dag(pkg) - # Allow dependencies to modify the module + # traverse in postorder so package can use vars from its dependencies spec = pkg.spec - for dependency_spec in spec.traverse(root=False): - dpkg = dependency_spec.package - dpkg.setup_dependent_package(pkg.module, spec) + for dspec in pkg.spec.traverse(order='post'): + # If a user makes their own package repo, e.g. + # spack.repos.mystuff.libelf.Libelf, and they inherit from + # an existing class like spack.repos.original.libelf.Libelf, + # then set the module variables for both classes so the + # parent class can still use them if it gets called. + spkg = dspec.package + modules = parent_class_modules(spkg.__class__) + for mod in modules: + set_module_variables_for_package(spkg, mod) + set_module_variables_for_package(spkg, spkg.module) - # Allow dependencies to set up environment as well - for dependency_spec in spec.traverse(root=False): - dpkg = dependency_spec.package + # Allow dependencies to modify the module + dpkg = dspec.package + dpkg.setup_dependent_package(pkg.module, spec) dpkg.setup_dependent_environment(spack_env, run_env, spec) - # Allow the package to apply some settings. pkg.setup_environment(spack_env, run_env) # Make sure nothing's strange about the Spack environment. diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 8297893f012..fc5b7e67dfe 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -64,7 +64,14 @@ def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) - def test_install_and_uninstall(self): + def fake_fetchify(self, pkg): + """Fake the URL for a package so it downloads from a file.""" + fetcher = FetchStrategyComposite() + fetcher.append(URLFetchStrategy(self.repo.url)) + pkg.fetcher = fetcher + + + def ztest_install_and_uninstall(self): # Get a basic concrete spec for the trivial install package. spec = Spec('trivial_install_test_package') spec.concretize() @@ -73,11 +80,7 @@ def test_install_and_uninstall(self): # Get the package pkg = spack.repo.get(spec) - # Fake the URL for the package so it downloads from a file. - - fetcher = FetchStrategyComposite() - fetcher.append(URLFetchStrategy(self.repo.url)) - pkg.fetcher = fetcher + self.fake_fetchify(pkg) try: pkg.do_install() @@ -85,3 +88,17 @@ def test_install_and_uninstall(self): except Exception, e: pkg.remove_prefix() raise + + + def test_install_environment(self): + spec = Spec('cmake-client').concretized() + + for s in spec.traverse(): + self.fake_fetchify(s.package) + + pkg = spec.package + try: + pkg.do_install() + except Exception, e: + pkg.remove_prefix() + raise diff --git a/var/spack/repos/builtin.mock/packages/cmake-client/package.py b/var/spack/repos/builtin.mock/packages/cmake-client/package.py new file mode 100644 index 00000000000..a5d3ef156a6 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/cmake-client/package.py @@ -0,0 +1,89 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +import os + +def check(condition, msg): + """Raise an install error if condition is False.""" + if not condition: + raise InstallError(msg) + + +class CmakeClient(Package): + """A dumy package that uses cmake.""" + homepage = 'https://www.example.com' + url = 'https://www.example.com/cmake-client-1.0.tar.gz' + + version('1.0', '4cb3ff35b2472aae70f542116d616e63') + + depends_on('cmake') + + + def setup_environment(self, spack_env, run_env): + spack_cc # Ensure spack module-scope variable is avaiabl + check(from_cmake == "from_cmake", + "setup_environment couldn't read global set by cmake.") + + check(self.spec['cmake'].link_arg == "test link arg", + "link arg on dependency spec not readable from setup_environment.") + + + def setup_dependent_environment(self, spack_env, run_env, dspec): + spack_cc # Ensure spack module-scope variable is avaiable + check(from_cmake == "from_cmake", + "setup_dependent_environment couldn't read global set by cmake.") + + check(self.spec['cmake'].link_arg == "test link arg", + "link arg on dependency spec not readable from setup_dependent_environment.") + + + def setup_dependent_package(self, module, dspec): + spack_cc # Ensure spack module-scope variable is avaiable + check(from_cmake == "from_cmake", + "setup_dependent_package couldn't read global set by cmake.") + + check(self.spec['cmake'].link_arg == "test link arg", + "link arg on dependency spec not readable from setup_dependent_package.") + + + + def install(self, spec, prefix): + # check that cmake is in the global scope. + global cmake + check(cmake is not None, "No cmake was in environment!") + + # check that which('cmake') returns the right one. + cmake = which('cmake') + check(cmake.exe[0].startswith(spec['cmake'].prefix.bin), + "Wrong cmake was in environment: %s" % cmake) + + check(from_cmake == "from_cmake", + "Couldn't read global set by cmake.") + + check(os.environ['from_cmake'] == 'from_cmake', + "Couldn't read env var set in envieonmnt by dependency") + + mkdirp(prefix.bin) + touch(join_path(prefix.bin, 'dummy')) diff --git a/var/spack/repos/builtin.mock/packages/cmake/package.py b/var/spack/repos/builtin.mock/packages/cmake/package.py new file mode 100644 index 00000000000..deb44c2bf76 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/cmake/package.py @@ -0,0 +1,69 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +import os + +def check(condition, msg): + """Raise an install error if condition is False.""" + if not condition: + raise InstallError(msg) + + +class Cmake(Package): + """A dumy package for the cmake build system.""" + homepage = 'https://www.cmake.org' + url = 'https://cmake.org/files/v3.4/cmake-3.4.3.tar.gz' + + version('3.4.3', '4cb3ff35b2472aae70f542116d616e63', + url='https://cmake.org/files/v3.4/cmake-3.4.3.tar.gz') + + + def setup_environment(self, spack_env, run_env): + spack_cc # Ensure spack module-scope variable is avaiable + spack_env.set('for_install', 'for_install') + + def setup_dependent_environment(self, spack_env, run_env, dspec): + spack_cc # Ensure spack module-scope variable is avaiable + spack_env.set('from_cmake', 'from_cmake') + + + def setup_dependent_package(self, module, dspec): + spack_cc # Ensure spack module-scope variable is avaiable + + self.spec.from_cmake = "from_cmake" + module.from_cmake = "from_cmake" + + self.spec.link_arg = "test link arg" + + + def install(self, spec, prefix): + mkdirp(prefix.bin) + + check(os.environ['for_install'] == 'for_install', + "Couldn't read env var set in compile envieonmnt") + + cmake_exe = join_path(prefix.bin, 'cmake') + touch(cmake_exe) + set_executable(cmake_exe) From 7197f1578335b38eb2037e8d82f15a27d786d5c1 Mon Sep 17 00:00:00 2001 From: Bruno Turcksin Date: Fri, 1 Apr 2016 12:28:46 -0400 Subject: [PATCH 851/910] Add version 2.6.7 of py-setuptools --- var/spack/repos/builtin/packages/py-setuptools/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-setuptools/package.py b/var/spack/repos/builtin/packages/py-setuptools/package.py index c6d9be1add7..9338a5e1578 100644 --- a/var/spack/repos/builtin/packages/py-setuptools/package.py +++ b/var/spack/repos/builtin/packages/py-setuptools/package.py @@ -10,6 +10,7 @@ class PySetuptools(Package): version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06') version('19.2', '78353b1f80375ca5e088f4b4627ffe03') version('20.5', 'fadc1e1123ddbe31006e5e43e927362b') + version('20.6.7', '45d6110f3ec14924e44c33411db64fe6') extends('python') From e0b9f79b9bc20c4e5178ee6dba6b327057cb3c4e Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Mon, 4 Apr 2016 10:33:03 -0700 Subject: [PATCH 852/910] Add changes that remove unsupported libraries from the boost build libraries list based on the boost version numbers. Libraries that are removed include: log - for versions of boost 1.53.0 and older - :atomic - for versions of boost 1.52.0 and older - :locale for versions of boost 1.49.0 and older - :chrono - for versions of boost 1.48.0 and older - and :random - for versions of boost 1.44.0 and older. --- var/spack/repos/builtin/packages/boost/package.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index f889da21f25..12bc9508c31 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -199,6 +199,18 @@ def install(self, spec, prefix): install_tree(src, dst) return + # Remove libraries that the release version does not support + if not spec.satisfies('@1.54.0:'): + withLibs.remove('log') + if not spec.satisfies('@1.53.0:'): + withLibs.remove('atomic') + if not spec.satisfies('@1.48.0:'): + withLibs.remove('locale') + if not spec.satisfies('@1.47.0:'): + withLibs.remove('chrono') + if not spec.satisfies('@1.43.0:'): + withLibs.remove('random') + # to make Boost find the user-config.jam env['BOOST_BUILD_PATH'] = './' From 6ebed7a2a6488a857fc6878c2d39d26ce9bc72f5 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Mon, 4 Apr 2016 11:23:51 -0700 Subject: [PATCH 853/910] Add release 9.1.0 recognition to the Dyninst API package file. --- var/spack/repos/builtin/packages/dyninst/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/dyninst/package.py b/var/spack/repos/builtin/packages/dyninst/package.py index 0111dcbe08d..b28e897a0f5 100644 --- a/var/spack/repos/builtin/packages/dyninst/package.py +++ b/var/spack/repos/builtin/packages/dyninst/package.py @@ -31,6 +31,8 @@ class Dyninst(Package): url = "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1.2/DyninstAPI-8.1.2.tgz" list_url = "http://www.dyninst.org/downloads/dyninst-8.x" + version('9.1.0', '5c64b77521457199db44bec82e4988ac', + url="http://www.paradyn.org/release9.1.0/DyninstAPI-9.1.0.tgz") version('8.2.1', 'abf60b7faabe7a2e4b54395757be39c7', url="http://www.paradyn.org/release8.2/DyninstAPI-8.2.1.tgz") version('8.1.2', 'bf03b33375afa66fe0efa46ce3f4b17a', From 7bc28cc334ea8b64ad4722dda2997871c27dc01a Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 4 Apr 2016 14:33:48 -0400 Subject: [PATCH 854/910] Clean up cc script --- lib/spack/env/cc | 65 ++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 2eb6f46afe5..6445edf8390 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -84,13 +84,12 @@ done # ld link # ccld compile & link # vcheck version check -# -# Depending on the mode, we may or may not add extra rpaths. -# This variable controls whether they are added. -add_rpaths=true command=$(basename "$0") case "$command" in + cpp) + mode=cpp + ;; cc|c89|c99|gcc|clang|icc|pgcc|xlc) command="$SPACK_CC" language="C" @@ -107,22 +106,8 @@ case "$command" in command="$SPACK_F77" language="Fortran 77" ;; - cpp) - mode=cpp - ;; ld) mode=ld - - # Darwin's linker has a -r argument that merges object files - # together. It doesn't work with -rpath. - if [[ $OSTYPE = darwin* ]]; then - for arg in "$@"; do - if [ "$arg" = -r ]; then - add_rpaths=false - break - fi - done - fi ;; *) die "Unkown compiler: $command" @@ -130,11 +115,11 @@ case "$command" in esac # If any of the arguments below is present then the mode is vcheck. In -# vcheck mode nothing is added in terms of extra search paths or -# libraries -if [ -z "$mode" ]; then +# vcheck mode, nothing is added in terms of extra search paths or +# libraries. +if [[ -z $mode ]]; then for arg in "$@"; do - if [ "$arg" = -v -o "$arg" = -V -o "$arg" = --version -o "$arg" = -dumpversion ]; then + if [[ $arg = -v || $arg = -V || $arg = --version || $arg = -dumpversion ]]; then mode=vcheck break fi @@ -142,16 +127,16 @@ if [ -z "$mode" ]; then fi # Finish setting up the mode. -if [ -z "$mode" ]; then +if [[ -z $mode ]]; then mode=ccld for arg in "$@"; do - if [ "$arg" = -E ]; then + if [[ $arg = -E ]]; then mode=cpp break - elif [ "$arg" = -S ]; then + elif [[ $arg = -S ]]; then mode=as break - elif [ "$arg" = -c ]; then + elif [[ $arg = -c ]]; then mode=cc break fi @@ -159,7 +144,7 @@ if [ -z "$mode" ]; then fi # Dump the version and exit if we're in testing mode. -if [ "$SPACK_TEST_COMMAND" = "dump-mode" ]; then +if [[ $SPACK_TEST_COMMAND = dump-mode ]]; then echo "$mode" exit fi @@ -170,10 +155,23 @@ if [[ -z $command ]]; then die "ERROR: Compiler '$SPACK_COMPILER_SPEC' does not support compiling $language programs." fi -if [ "$mode" == vcheck ] ; then +if [[ $mode == vcheck ]]; then exec ${command} "$@" fi +# Darwin's linker has a -r argument that merges object files together. +# It doesn't work with -rpath. +# This variable controls whether they are added. +add_rpaths=true +if [[ mode = ld && $OSTYPE = darwin* ]]; then + for arg in "$@"; do + if [[ $arg = -r ]]; then + add_rpaths=false + break + fi + done +fi + # Save original command for debug logging input_command="$@" args=("$@") @@ -234,11 +232,14 @@ IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" spack_env_dirs+=("" ".") PATH="" for dir in "${env_path[@]}"; do - remove="" - for rm_dir in "${spack_env_dirs[@]}"; do - if [[ $dir = $rm_dir ]]; then remove=True; fi + addpath=true + for env_dir in "${spack_env_dirs[@]}"; do + if [[ $dir = $env_dir ]]; then + addpath=false + break + fi done - if [[ -z $remove ]]; then + if $addpath; then PATH="${PATH:+$PATH:}$dir" fi done From badf5b047b5cca851c3cad14b4cb7167103ed229 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 4 Apr 2016 14:42:36 -0400 Subject: [PATCH 855/910] Calculate version string in a function --- var/spack/repos/builtin/packages/hdf5/package.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 1bf01e7423a..f26e225b83e 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -37,8 +37,7 @@ class Hdf5(Package): list_url = "http://www.hdfgroup.org/ftp/HDF5/releases" list_depth = 3 - version('1.10.0', 'bdc935337ee8282579cd6bc4270ad199', - url='http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.0/src/hdf5-1.10.0.tar.gz') + version('1.10.0', 'bdc935337ee8282579cd6bc4270ad199') version('1.8.16', 'b8ed9a36ae142317f88b0c7ef4b9c618') version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') version('1.8.13', 'c03426e9e77d7766944654280b467289') @@ -147,5 +146,7 @@ def url_for_version(self, version): return "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-" + v + ".tar.gz" elif version < Version("1.7"): return "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-" + version.up_to(2) + "/hdf5-" + v + ".tar.gz" - else: + elif version < Version("1.10"): return "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-" + v + "/src/hdf5-" + v + ".tar.gz" + else: + return "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-" + version.up_to(2) + "/hdf5-" + v + "/src/hdf5-" + v + ".tar.gz" From d0ca3952a34a74f0167b76bbedfa3cf8875a399c Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 2 Apr 2016 20:17:01 -0500 Subject: [PATCH 856/910] Add version 0.17.1 of scikit-learn. --- var/spack/repos/builtin/packages/py-scikit-learn/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-scikit-learn/package.py b/var/spack/repos/builtin/packages/py-scikit-learn/package.py index 5b078ce9011..7f8e25e69d1 100644 --- a/var/spack/repos/builtin/packages/py-scikit-learn/package.py +++ b/var/spack/repos/builtin/packages/py-scikit-learn/package.py @@ -7,6 +7,7 @@ class PyScikitLearn(Package): version('0.15.2', 'd9822ad0238e17b382a3c756ea94fe0d') version('0.16.1', '363ddda501e3b6b61726aa40b8dbdb7e') + version('0.17.1', 'a2f8b877e6d99b1ed737144f5a478dfc') extends('python') From 3a4210e9d7cbdb1b9a441d41a88e4170d08d2879 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 4 Apr 2016 12:07:37 -0500 Subject: [PATCH 857/910] New package - py-dask (http://dask.pydata.org). --- var/spack/repos/builtin/packages/py-dask/package.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-dask/package.py diff --git a/var/spack/repos/builtin/packages/py-dask/package.py b/var/spack/repos/builtin/packages/py-dask/package.py new file mode 100644 index 00000000000..725d47b97c0 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-dask/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyDask(Package): + """Minimal task scheduling abstraction""" + homepage = "https://github.com/dask/dask/" + url = "https://pypi.python.org/packages/source/d/dask/dask-0.8.1.tar.gz" + + version('0.8.1', '5dd8e3a3823b3bc62c9a6d192e2cb5b4') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From a10ab78e28ca14f923e93029027f763c16fab7fc Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 4 Apr 2016 12:09:11 -0500 Subject: [PATCH 858/910] New package - py-decorator (https://github.com/micheles/decorator). --- .../repos/builtin/packages/py-decorator/package.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-decorator/package.py diff --git a/var/spack/repos/builtin/packages/py-decorator/package.py b/var/spack/repos/builtin/packages/py-decorator/package.py new file mode 100644 index 00000000000..99e3dbc49cc --- /dev/null +++ b/var/spack/repos/builtin/packages/py-decorator/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyDecorator(Package): + """The aim of the decorator module it to simplify the usage of decorators for the average programmer, and to popularize decorators by showing various non-trivial examples.""" + homepage = "https://github.com/micheles/decorator" + url = "https://pypi.python.org/packages/source/d/decorator/decorator-4.0.9.tar.gz" + + version('4.0.9', 'f12c5651ccd707e12a0abaa4f76cd69a') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 8404f084c8d768a197b830ff1809eb35f505ca8d Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 4 Apr 2016 12:12:42 -0500 Subject: [PATCH 859/910] New package - py-networkx (https://networkx.github.io/). --- .../repos/builtin/packages/py-networkx/package.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-networkx/package.py diff --git a/var/spack/repos/builtin/packages/py-networkx/package.py b/var/spack/repos/builtin/packages/py-networkx/package.py new file mode 100644 index 00000000000..893146ec3ea --- /dev/null +++ b/var/spack/repos/builtin/packages/py-networkx/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyNetworkx(Package): + """NetworkX is a Python package for the creation, manipulation, and study of the structure, dynamics, and functions of complex networks.""" + homepage = "http://networkx.github.io/" + url = "https://pypi.python.org/packages/source/n/networkx/networkx-1.11.tar.gz" + + version('1.11', '6ef584a879e9163013e9a762e1cf7cd1') + + extends('python') + + depends_on('py-decorator') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 9c60f122a6ffa877d265d523e52e8a4018c2bdec Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 4 Apr 2016 12:13:31 -0500 Subject: [PATCH 860/910] New package - py-scikit-image (http://scikit-image.org/). --- .../packages/py-scikit-image/package.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-scikit-image/package.py diff --git a/var/spack/repos/builtin/packages/py-scikit-image/package.py b/var/spack/repos/builtin/packages/py-scikit-image/package.py new file mode 100644 index 00000000000..22ce1f83740 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-scikit-image/package.py @@ -0,0 +1,20 @@ +from spack import * + +class PyScikitImage(Package): + """Image processing algorithms for SciPy, including IO, morphology, filtering, warping, color manipulation, object detection, etc.""" + homepage = "http://scikit-image.org/" + url = "https://pypi.python.org/packages/source/s/scikit-image/scikit-image-0.12.3.tar.gz" + + version('0.12.3', '04ea833383e0b6ad5f65da21292c25e1') + + extends('python', ignore=r'bin/.*\.py$|bin/f2py$') + + depends_on('py-dask') + depends_on('py-pillow') + depends_on('py-networkx') + depends_on('py-six') + depends_on('py-scipy') + depends_on('py-matplotlib') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 83d6e04d39ce81376c96cf6b4aa3261ff9e6fbc9 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 4 Apr 2016 15:38:21 -0400 Subject: [PATCH 861/910] Convert `=` to `==` in tests; untabify --- lib/spack/env/cc | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 6445edf8390..b845de8a2b7 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -119,7 +119,7 @@ esac # libraries. if [[ -z $mode ]]; then for arg in "$@"; do - if [[ $arg = -v || $arg = -V || $arg = --version || $arg = -dumpversion ]]; then + if [[ $arg == -v || $arg == -V || $arg == --version || $arg == -dumpversion ]]; then mode=vcheck break fi @@ -130,13 +130,13 @@ fi if [[ -z $mode ]]; then mode=ccld for arg in "$@"; do - if [[ $arg = -E ]]; then + if [[ $arg == -E ]]; then mode=cpp break - elif [[ $arg = -S ]]; then + elif [[ $arg == -S ]]; then mode=as break - elif [[ $arg = -c ]]; then + elif [[ $arg == -c ]]; then mode=cc break fi @@ -144,7 +144,7 @@ if [[ -z $mode ]]; then fi # Dump the version and exit if we're in testing mode. -if [[ $SPACK_TEST_COMMAND = dump-mode ]]; then +if [[ $SPACK_TEST_COMMAND == dump-mode ]]; then echo "$mode" exit fi @@ -163,12 +163,12 @@ fi # It doesn't work with -rpath. # This variable controls whether they are added. add_rpaths=true -if [[ mode = ld && $OSTYPE = darwin* ]]; then +if [[ mode == ld && $OSTYPE == darwin* ]]; then for arg in "$@"; do - if [[ $arg = -r ]]; then + if [[ $arg == -r ]]; then add_rpaths=false break - fi + fi done fi @@ -181,17 +181,17 @@ IFS=':' read -ra deps <<< "$SPACK_DEPENDENCIES" for dep in "${deps[@]}"; do # Prepend include directories if [[ -d $dep/include ]]; then - if [[ $mode = cpp || $mode = cc || $mode = as || $mode = ccld ]]; then + if [[ $mode == cpp || $mode == cc || $mode == as || $mode == ccld ]]; then args=("-I$dep/include" "${args[@]}") fi fi # Prepend lib and RPATH directories if [[ -d $dep/lib ]]; then - if [[ $mode = ccld ]]; then + if [[ $mode == ccld ]]; then $add_rpaths && args=("-Wl,-rpath,$dep/lib" "${args[@]}") args=("-L$dep/lib" "${args[@]}") - elif [[ $mode = ld ]]; then + elif [[ $mode == ld ]]; then $add_rpaths && args=("-rpath" "$dep/lib" "${args[@]}") args=("-L$dep/lib" "${args[@]}") fi @@ -199,10 +199,10 @@ for dep in "${deps[@]}"; do # Prepend lib64 and RPATH directories if [[ -d $dep/lib64 ]]; then - if [[ $mode = ccld ]]; then + if [[ $mode == ccld ]]; then $add_rpaths && args=("-Wl,-rpath,$dep/lib64" "${args[@]}") args=("-L$dep/lib64" "${args[@]}") - elif [[ $mode = ld ]]; then + elif [[ $mode == ld ]]; then $add_rpaths && args=("-rpath" "$dep/lib64" "${args[@]}") args=("-L$dep/lib64" "${args[@]}") fi @@ -210,9 +210,9 @@ for dep in "${deps[@]}"; do done # Include all -L's and prefix/whatever dirs in rpath -if [[ $mode = ccld ]]; then +if [[ $mode == ccld ]]; then $add_rpaths && args=("-Wl,-rpath,$SPACK_PREFIX/lib" "-Wl,-rpath,$SPACK_PREFIX/lib64" "${args[@]}") -elif [[ $mode = ld ]]; then +elif [[ $mode == ld ]]; then $add_rpaths && args=("-rpath" "$SPACK_PREFIX/lib" "-rpath" "$SPACK_PREFIX/lib64" "${args[@]}") fi @@ -234,7 +234,7 @@ PATH="" for dir in "${env_path[@]}"; do addpath=true for env_dir in "${spack_env_dirs[@]}"; do - if [[ $dir = $env_dir ]]; then + if [[ $dir == $env_dir ]]; then addpath=false break fi @@ -248,7 +248,7 @@ export PATH full_command=("$command" "${args[@]}") # In test command mode, write out full command for Spack tests. -if [[ $SPACK_TEST_COMMAND = dump-args ]]; then +if [[ $SPACK_TEST_COMMAND == dump-args ]]; then echo "${full_command[@]}" exit elif [[ -n $SPACK_TEST_COMMAND ]]; then @@ -258,7 +258,7 @@ fi # # Write the input and output commands to debug logs if it's asked for. # -if [[ $SPACK_DEBUG = TRUE ]]; then +if [[ $SPACK_DEBUG == TRUE ]]; then input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.in.log" output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.out.log" echo "[$mode] $command $input_command" >> $input_log From 9bb99de0a75a9f9c2cc1ad65596a7219fa700a76 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Mon, 4 Apr 2016 13:47:48 -0600 Subject: [PATCH 862/910] + When using CMake, use Release instead of RelWithDebInfo flags. --- lib/spack/spack/build_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 640db0c1d16..1346af22858 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -217,7 +217,7 @@ def set_module_variables_for_package(pkg, module): # standard CMake arguments m.std_cmake_args = ['-DCMAKE_INSTALL_PREFIX=%s' % pkg.prefix, - '-DCMAKE_BUILD_TYPE=RelWithDebInfo'] + '-DCMAKE_BUILD_TYPE=Release'] if platform.mac_ver()[0]: m.std_cmake_args.append('-DCMAKE_FIND_FRAMEWORK=LAST') From 0ff059e388b856d7eed86d27a665a48b65bfa65f Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 4 Apr 2016 16:00:09 -0400 Subject: [PATCH 863/910] Clean up comments and output messages --- lib/spack/env/cc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index b845de8a2b7..aa1e0dbe290 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -39,7 +39,7 @@ # # This is the list of environment variables that need to be set before -# the script runs. They are set by routines in spack.build_environment +# the script runs. They are set by routines in spack.build_environment # as part of spack.package.Package.do_install(). parameters=" SPACK_PREFIX @@ -50,7 +50,7 @@ SPACK_SHORT_SPEC" # The compiler input variables are checked for sanity later: # SPACK_CC, SPACK_CXX, SPACK_F77, SPACK_FC -# Debug flag is optional; set to true for debug logging: +# Debug flag is optional; set to "TRUE" for debug logging: # SPACK_DEBUG # Test command is used to unit test the compiler script. # SPACK_TEST_COMMAND @@ -66,11 +66,10 @@ function die { for param in $parameters; do if [[ -z ${!param} ]]; then - die "Spack compiler must be run from spack! Input $param was missing!" + die "Spack compiler must be run from Apack! Input '$param' is missing." fi done -# # Figure out the type of compiler, the language, and the mode so that # the compiler script knows what to do. # @@ -78,12 +77,12 @@ done # 'command' is set based on the input command to $SPACK_[CC|CXX|F77|F90] # # 'mode' is set to one of: +# vcheck version check # cpp preprocess # cc compile # as assemble # ld link # ccld compile & link -# vcheck version check command=$(basename "$0") case "$command" in @@ -114,8 +113,8 @@ case "$command" in ;; esac -# If any of the arguments below is present then the mode is vcheck. In -# vcheck mode, nothing is added in terms of extra search paths or +# If any of the arguments below are present, then the mode is vcheck. +# In vcheck mode, nothing is added in terms of extra search paths or # libraries. if [[ -z $mode ]]; then for arg in "$@"; do From cd3086f78d878a86c78672110bbc3fba4894a401 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Mon, 4 Apr 2016 14:35:09 -0600 Subject: [PATCH 864/910] + Since CBLAS is only provided in 3.6.0 or later, modify package.py to on refer to CBLAS when the latest release is requested. --- .../repos/builtin/packages/netlib-lapack/package.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py index 05436332ac5..d1252efb9b6 100644 --- a/var/spack/repos/builtin/packages/netlib-lapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -34,15 +34,18 @@ class NetlibLapack(Package): def patch(self): # Fix cblas CMakeLists.txt -- has wrong case for subdirectory name. - filter_file('${CMAKE_CURRENT_SOURCE_DIR}/CMAKE/', - '${CMAKE_CURRENT_SOURCE_DIR}/cmake/', 'CBLAS/CMakeLists.txt', string=True) + if self.spec.satisfies('@3.6.0:'): + filter_file('${CMAKE_CURRENT_SOURCE_DIR}/CMAKE/', + '${CMAKE_CURRENT_SOURCE_DIR}/cmake/', 'CBLAS/CMakeLists.txt', string=True) def install_one(self, spec, prefix, shared): cmake_args = ['-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if shared else 'OFF'), - '-DCBLAS=ON', # always build CBLAS '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), '-DLAPACKE:BOOL=%s' % ('ON' if '+lapacke' in spec else 'OFF')] + if spec.satisfies('@3.6.0:'): + cmake_args.extend(['-DCBLAS=ON']) # always build CBLAS + if '+external-blas' in spec: # TODO : the mechanism to specify the library should be more general, # TODO : but this allows to have an hook to an external blas @@ -80,6 +83,3 @@ def setup_dependent_package(self, module, dspec): if '+shared' in self.spec: self.spec.blas_shared_lib = join_path(libdir, 'libblas.%s' % dso_suffix) self.spec.lapack_shared_lib = join_path(libdir, 'liblapack.%s' % dso_suffix) - - - From db23d27eb4c7b6be36a0b9df19160cbd3af511e4 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Mon, 4 Apr 2016 14:41:41 -0600 Subject: [PATCH 865/910] + Revert accidental commit that set CMAKE_BUILD_TYPE=Release. --- lib/spack/spack/build_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 2ef874803cb..f4f8037ac09 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -217,7 +217,7 @@ def set_module_variables_for_package(pkg, module): # standard CMake arguments m.std_cmake_args = ['-DCMAKE_INSTALL_PREFIX=%s' % pkg.prefix, - '-DCMAKE_BUILD_TYPE=Release'] + '-DCMAKE_BUILD_TYPE=RelWithDebInfo'] if platform.mac_ver()[0]: m.std_cmake_args.append('-DCMAKE_FIND_FRAMEWORK=LAST') From 0296f96c7b5279b2dd9cf585a493f21e3344fe2b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 4 Apr 2016 13:56:15 -0700 Subject: [PATCH 866/910] Resolves #739. Don't call setup_dependent_* for package itself. --- lib/spack/spack/build_environment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index f4f8037ac09..eb72f2a6b4a 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -304,7 +304,7 @@ def setup_package(pkg): # traverse in postorder so package can use vars from its dependencies spec = pkg.spec - for dspec in pkg.spec.traverse(order='post'): + for dspec in pkg.spec.traverse(order='post', root=False): # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from # an existing class like spack.repos.original.libelf.Libelf, @@ -321,6 +321,7 @@ def setup_package(pkg): dpkg.setup_dependent_package(pkg.module, spec) dpkg.setup_dependent_environment(spack_env, run_env, spec) + set_module_variables_for_package(pkg, pkg.module) pkg.setup_environment(spack_env, run_env) # Make sure nothing's strange about the Spack environment. From c0eb5844e536c25165b913f4dc203fe09f8ecc3d Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Mon, 4 Apr 2016 17:05:30 -0600 Subject: [PATCH 867/910] + Provide dia, a program for drawing structured diagrams. --- .../repos/builtin/packages/dia/package.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 var/spack/repos/builtin/packages/dia/package.py diff --git a/var/spack/repos/builtin/packages/dia/package.py b/var/spack/repos/builtin/packages/dia/package.py new file mode 100644 index 00000000000..0c8bbfc4255 --- /dev/null +++ b/var/spack/repos/builtin/packages/dia/package.py @@ -0,0 +1,67 @@ +from spack import * + +class Dia(Package): + """Dia is a program for drawing structured diagrams.""" + homepage = 'https://wiki.gnome.org/Apps/Dia' + url = 'https://ftp.gnome.org/pub/gnome/sources/dia/0.97/dia-0.97.3.tar.xz' + + version('0.97.3', '0e744a0f6a6c4cb6a089e4d955392c3c') + + #variant('ncurses', default=True, description='Enables the build of the ncurses gui') + #variant('openssl', default=True, description="Enables CMake's OpenSSL features") + #variant('qt', default=False, description='Enables the build of cmake-gui') + #variant('doc', default=False, description='Enables the generation of html and man page documentation') + + depends_on('gtkplus@2.6.0:') + # depends_on('openssl', when='+openssl') + #depends_on('qt', when='+qt') + #depends_on('python@2.7.11:', when='+doc') + depends_on('cairo') + #depends_on('libart') # optional dependency, not yet supported by spack. + depends_on('libpng') + depends_on('libxslt') + depends_on('python') + depends_on('swig') + # depends_on('py-gtk') # optional dependency, not yet supported by spack. + + def url_for_version(self, version): + """Handle Dia's version-based custom URLs.""" + return 'https://ftp.gnome.org/pub/gnome/source/dia/%s/dia-%s.tar.xz' % (version.up_to(2), version) + + # def validate(self, spec): + # """ + # Checks if incompatible versions of qt were specified + + # :param spec: spec of the package + # :raises RuntimeError: in case of inconsistencies + # """ + + # if '+qt' in spec and spec.satisfies('^qt@5.4.0'): + # msg = 'qt-5.4.0 has broken CMake modules.' + # raise RuntimeError(msg) + + def install(self, spec, prefix): + # Consistency check + # self.validate(spec) + + # configure, build, install: + options = ['--prefix=%s' % prefix, + '--with-cairo', + '--with-xslt-prefix=%s' % spec['libxslt'].prefix, + '--with-python', + '--with-swig'] + + # if '+qt' in spec: + # options.append('--qt-gui') + + # if '+doc' in spec: + # options.append('--sphinx-html') + # options.append('--sphinx-man') + + # if '+openssl' in spec: + # options.append('--') + # options.append('-DCMAKE_USE_OPENSSL=ON') + + configure(*options) + make() + make('install') From 77b688f4fa197c2b7d1e792e038e51ed52e5be2a Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Mon, 4 Apr 2016 17:08:19 -0600 Subject: [PATCH 868/910] + Don't include files submitted with PR#741. --- .../repos/builtin/packages/netlib-lapack/package.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py index d1252efb9b6..05436332ac5 100644 --- a/var/spack/repos/builtin/packages/netlib-lapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -34,18 +34,15 @@ class NetlibLapack(Package): def patch(self): # Fix cblas CMakeLists.txt -- has wrong case for subdirectory name. - if self.spec.satisfies('@3.6.0:'): - filter_file('${CMAKE_CURRENT_SOURCE_DIR}/CMAKE/', - '${CMAKE_CURRENT_SOURCE_DIR}/cmake/', 'CBLAS/CMakeLists.txt', string=True) + filter_file('${CMAKE_CURRENT_SOURCE_DIR}/CMAKE/', + '${CMAKE_CURRENT_SOURCE_DIR}/cmake/', 'CBLAS/CMakeLists.txt', string=True) def install_one(self, spec, prefix, shared): cmake_args = ['-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if shared else 'OFF'), + '-DCBLAS=ON', # always build CBLAS '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), '-DLAPACKE:BOOL=%s' % ('ON' if '+lapacke' in spec else 'OFF')] - if spec.satisfies('@3.6.0:'): - cmake_args.extend(['-DCBLAS=ON']) # always build CBLAS - if '+external-blas' in spec: # TODO : the mechanism to specify the library should be more general, # TODO : but this allows to have an hook to an external blas @@ -83,3 +80,6 @@ def setup_dependent_package(self, module, dspec): if '+shared' in self.spec: self.spec.blas_shared_lib = join_path(libdir, 'libblas.%s' % dso_suffix) self.spec.lapack_shared_lib = join_path(libdir, 'liblapack.%s' % dso_suffix) + + + From 3a3f9789ce001a3dc61274eac67ddfc53d91df4c Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Mon, 4 Apr 2016 17:10:28 -0600 Subject: [PATCH 869/910] + General cleanup of package.py. --- .../repos/builtin/packages/dia/package.py | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/var/spack/repos/builtin/packages/dia/package.py b/var/spack/repos/builtin/packages/dia/package.py index 0c8bbfc4255..1cb5910e46d 100644 --- a/var/spack/repos/builtin/packages/dia/package.py +++ b/var/spack/repos/builtin/packages/dia/package.py @@ -7,15 +7,7 @@ class Dia(Package): version('0.97.3', '0e744a0f6a6c4cb6a089e4d955392c3c') - #variant('ncurses', default=True, description='Enables the build of the ncurses gui') - #variant('openssl', default=True, description="Enables CMake's OpenSSL features") - #variant('qt', default=False, description='Enables the build of cmake-gui') - #variant('doc', default=False, description='Enables the generation of html and man page documentation') - depends_on('gtkplus@2.6.0:') - # depends_on('openssl', when='+openssl') - #depends_on('qt', when='+qt') - #depends_on('python@2.7.11:', when='+doc') depends_on('cairo') #depends_on('libart') # optional dependency, not yet supported by spack. depends_on('libpng') @@ -28,21 +20,7 @@ def url_for_version(self, version): """Handle Dia's version-based custom URLs.""" return 'https://ftp.gnome.org/pub/gnome/source/dia/%s/dia-%s.tar.xz' % (version.up_to(2), version) - # def validate(self, spec): - # """ - # Checks if incompatible versions of qt were specified - - # :param spec: spec of the package - # :raises RuntimeError: in case of inconsistencies - # """ - - # if '+qt' in spec and spec.satisfies('^qt@5.4.0'): - # msg = 'qt-5.4.0 has broken CMake modules.' - # raise RuntimeError(msg) - def install(self, spec, prefix): - # Consistency check - # self.validate(spec) # configure, build, install: options = ['--prefix=%s' % prefix, @@ -51,17 +29,6 @@ def install(self, spec, prefix): '--with-python', '--with-swig'] - # if '+qt' in spec: - # options.append('--qt-gui') - - # if '+doc' in spec: - # options.append('--sphinx-html') - # options.append('--sphinx-man') - - # if '+openssl' in spec: - # options.append('--') - # options.append('-DCMAKE_USE_OPENSSL=ON') - configure(*options) make() make('install') From 48d70d960cb76783ff2d5f431b88ea1c0bf4c8a8 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Mon, 4 Apr 2016 17:05:30 -0600 Subject: [PATCH 870/910] + Provide dia, a program for drawing structured diagrams. --- .../repos/builtin/packages/dia/package.py | 34 +++++++++++++++++++ .../builtin/packages/netlib-lapack/package.py | 12 +++---- 2 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 var/spack/repos/builtin/packages/dia/package.py diff --git a/var/spack/repos/builtin/packages/dia/package.py b/var/spack/repos/builtin/packages/dia/package.py new file mode 100644 index 00000000000..1cb5910e46d --- /dev/null +++ b/var/spack/repos/builtin/packages/dia/package.py @@ -0,0 +1,34 @@ +from spack import * + +class Dia(Package): + """Dia is a program for drawing structured diagrams.""" + homepage = 'https://wiki.gnome.org/Apps/Dia' + url = 'https://ftp.gnome.org/pub/gnome/sources/dia/0.97/dia-0.97.3.tar.xz' + + version('0.97.3', '0e744a0f6a6c4cb6a089e4d955392c3c') + + depends_on('gtkplus@2.6.0:') + depends_on('cairo') + #depends_on('libart') # optional dependency, not yet supported by spack. + depends_on('libpng') + depends_on('libxslt') + depends_on('python') + depends_on('swig') + # depends_on('py-gtk') # optional dependency, not yet supported by spack. + + def url_for_version(self, version): + """Handle Dia's version-based custom URLs.""" + return 'https://ftp.gnome.org/pub/gnome/source/dia/%s/dia-%s.tar.xz' % (version.up_to(2), version) + + def install(self, spec, prefix): + + # configure, build, install: + options = ['--prefix=%s' % prefix, + '--with-cairo', + '--with-xslt-prefix=%s' % spec['libxslt'].prefix, + '--with-python', + '--with-swig'] + + configure(*options) + make() + make('install') diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py index d1252efb9b6..05436332ac5 100644 --- a/var/spack/repos/builtin/packages/netlib-lapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -34,18 +34,15 @@ class NetlibLapack(Package): def patch(self): # Fix cblas CMakeLists.txt -- has wrong case for subdirectory name. - if self.spec.satisfies('@3.6.0:'): - filter_file('${CMAKE_CURRENT_SOURCE_DIR}/CMAKE/', - '${CMAKE_CURRENT_SOURCE_DIR}/cmake/', 'CBLAS/CMakeLists.txt', string=True) + filter_file('${CMAKE_CURRENT_SOURCE_DIR}/CMAKE/', + '${CMAKE_CURRENT_SOURCE_DIR}/cmake/', 'CBLAS/CMakeLists.txt', string=True) def install_one(self, spec, prefix, shared): cmake_args = ['-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if shared else 'OFF'), + '-DCBLAS=ON', # always build CBLAS '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), '-DLAPACKE:BOOL=%s' % ('ON' if '+lapacke' in spec else 'OFF')] - if spec.satisfies('@3.6.0:'): - cmake_args.extend(['-DCBLAS=ON']) # always build CBLAS - if '+external-blas' in spec: # TODO : the mechanism to specify the library should be more general, # TODO : but this allows to have an hook to an external blas @@ -83,3 +80,6 @@ def setup_dependent_package(self, module, dspec): if '+shared' in self.spec: self.spec.blas_shared_lib = join_path(libdir, 'libblas.%s' % dso_suffix) self.spec.lapack_shared_lib = join_path(libdir, 'liblapack.%s' % dso_suffix) + + + From aaf2830b8be96ca3584bc0cca87c86bc5b4355a0 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 4 Apr 2016 19:40:05 -0400 Subject: [PATCH 871/910] Require cmake --- var/spack/repos/builtin/packages/netlib-scalapack/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index d59f8e41fec..276876d197b 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -18,6 +18,7 @@ class NetlibScalapack(Package): provides('scalapack') + depends_on('cmake') depends_on('mpi') depends_on('lapack') From 2b3a8a4a5afc3f263a66b02b2f3cce9a5d1787fc Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 4 Apr 2016 19:01:44 -0500 Subject: [PATCH 872/910] Add missing dependencies. --- var/spack/repos/builtin/packages/py-scikit-learn/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/repos/builtin/packages/py-scikit-learn/package.py b/var/spack/repos/builtin/packages/py-scikit-learn/package.py index 7f8e25e69d1..2d7985b98c3 100644 --- a/var/spack/repos/builtin/packages/py-scikit-learn/package.py +++ b/var/spack/repos/builtin/packages/py-scikit-learn/package.py @@ -11,5 +11,9 @@ class PyScikitLearn(Package): extends('python') + depends_on('py-setuptools') + depends_on('py-numpy') + depends_on('py-scipy') + def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) From c546b75443eea9bc8fe7b5427cdd7eba5afb5add Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Mon, 4 Apr 2016 19:23:48 -0500 Subject: [PATCH 873/910] Add missing dependency on py-setuptools for - py-dask - py-decorator --- var/spack/repos/builtin/packages/py-dask/package.py | 2 ++ var/spack/repos/builtin/packages/py-decorator/package.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/var/spack/repos/builtin/packages/py-dask/package.py b/var/spack/repos/builtin/packages/py-dask/package.py index 725d47b97c0..cf0a16f21e3 100644 --- a/var/spack/repos/builtin/packages/py-dask/package.py +++ b/var/spack/repos/builtin/packages/py-dask/package.py @@ -9,5 +9,7 @@ class PyDask(Package): extends('python') + depends_on('py-setuptools') + def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-decorator/package.py b/var/spack/repos/builtin/packages/py-decorator/package.py index 99e3dbc49cc..abbd9f43d14 100644 --- a/var/spack/repos/builtin/packages/py-decorator/package.py +++ b/var/spack/repos/builtin/packages/py-decorator/package.py @@ -9,5 +9,7 @@ class PyDecorator(Package): extends('python') + depends_on('py-setuptools') + def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) From 4ce03b75bc628bd80a9798300043f3ec0a2ed7fa Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 5 Apr 2016 07:42:23 -0400 Subject: [PATCH 874/910] Correct typo --- lib/spack/env/cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index aa1e0dbe290..18fd8f7bdb1 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -66,7 +66,7 @@ function die { for param in $parameters; do if [[ -z ${!param} ]]; then - die "Spack compiler must be run from Apack! Input '$param' is missing." + die "Spack compiler must be run from Spack! Input '$param' is missing." fi done From a87ae5173f5e30aa9d8b3360e67dbc17568342f5 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 4 Apr 2016 18:16:25 -0400 Subject: [PATCH 875/910] Update Julia - allow checking out the master branche - add dependencies requires by important Julia package - add OpenBLAS patch --- .../builtin/packages/julia/openblas.patch | 68 +++++++++++++++++++ .../repos/builtin/packages/julia/package.py | 68 ++++++++++++------- 2 files changed, 113 insertions(+), 23 deletions(-) create mode 100644 var/spack/repos/builtin/packages/julia/openblas.patch diff --git a/var/spack/repos/builtin/packages/julia/openblas.patch b/var/spack/repos/builtin/packages/julia/openblas.patch new file mode 100644 index 00000000000..f75d7dd04f8 --- /dev/null +++ b/var/spack/repos/builtin/packages/julia/openblas.patch @@ -0,0 +1,68 @@ +diff --git a/deps/Makefile b/deps/Makefile +index 6cb73be..bcd8520 100644 +--- a/deps/Makefile ++++ b/deps/Makefile +@@ -1049,7 +1049,7 @@ OPENBLAS_BUILD_OPTS += NO_AFFINITY=1 + + # Build for all architectures - required for distribution + ifeq ($(OPENBLAS_DYNAMIC_ARCH), 1) +-OPENBLAS_BUILD_OPTS += DYNAMIC_ARCH=1 ++OPENBLAS_BUILD_OPTS += DYNAMIC_ARCH=1 MAKE_NO_J=1 + endif + + # 64-bit BLAS interface +@@ -1085,6 +1085,7 @@ OPENBLAS_BUILD_OPTS += NO_AVX2=1 + endif + + $(OPENBLAS_SRC_DIR)/config.status: $(OPENBLAS_SRC_DIR)/Makefile ++ cd $(dir $@) && patch -p1 < ../openblas-make.patch + ifeq ($(OS),WINNT) + cd $(dir $@) && patch -p1 < ../openblas-win64.patch + endif +diff --git a/deps/openblas.version b/deps/openblas.version +index 7c97e1b..58b9467 100644 +--- a/deps/openblas.version ++++ b/deps/openblas.version +@@ -1,2 +1,2 @@ +-OPENBLAS_BRANCH=v0.2.15 +-OPENBLAS_SHA1=53e849f4fcae4363a64576de00e982722c7304f9 ++OPENBLAS_BRANCH=v0.2.17 ++OPENBLAS_SHA1=a71e8c82f6a9f73093b631e5deab1e8da716b61f +--- a/deps/openblas-make.patch ++++ b/deps/openblas-make.patch +@@ -0,0 +1,35 @@ ++diff --git a/Makefile.system b/Makefile.system ++index b89f60e..2dbdad0 100644 ++--- a/Makefile.system +++++ b/Makefile.system ++@@ -139,6 +139,10 @@ NO_PARALLEL_MAKE=0 ++ endif ++ GETARCH_FLAGS += -DNO_PARALLEL_MAKE=$(NO_PARALLEL_MAKE) ++ +++ifdef MAKE_NO_J +++GETARCH_FLAGS += -DMAKE_NO_J=$(MAKE_NO_J) +++endif +++ ++ ifdef MAKE_NB_JOBS ++ GETARCH_FLAGS += -DMAKE_NB_JOBS=$(MAKE_NB_JOBS) ++ endif ++diff --git a/getarch.c b/getarch.c ++index f9c49e6..dffad70 100644 ++--- a/getarch.c +++++ b/getarch.c ++@@ -1012,6 +1012,7 @@ int main(int argc, char *argv[]){ ++ #endif ++ #endif ++ +++#ifndef MAKE_NO_J ++ #ifdef MAKE_NB_JOBS ++ printf("MAKE += -j %d\n", MAKE_NB_JOBS); ++ #elif NO_PARALLEL_MAKE==1 ++@@ -1021,6 +1022,7 @@ int main(int argc, char *argv[]){ ++ printf("MAKE += -j %d\n", get_num_cores()); ++ #endif ++ #endif +++#endif ++ ++ break; ++ diff --git a/var/spack/repos/builtin/packages/julia/package.py b/var/spack/repos/builtin/packages/julia/package.py index 6900af38e4f..b3a523bc450 100644 --- a/var/spack/repos/builtin/packages/julia/package.py +++ b/var/spack/repos/builtin/packages/julia/package.py @@ -4,43 +4,55 @@ class Julia(Package): """The Julia Language: A fresh approach to technical computing""" homepage = "http://julialang.org" - url = "http://github.com/JuliaLang/julia/releases/download/v0.4.2/julia-0.4.2.tar.gz" + url = "https://github.com/JuliaLang/julia/releases/download/v0.4.3/julia-0.4.3-full.tar.gz" - version('0.4.3', '7b9f096798fca4bef262a64674bc2b52') - version('0.4.2', 'ccfeb4f4090c8b31083f5e1ccb03eb06') + version('master', + git='https://github.com/JuliaLang/julia.git', branch='master') + version('0.4.5', '69141ff5aa6cee7c0ec8c85a34aa49a6') + version('0.4.3', '8a4a59fd335b05090dd1ebefbbe5aaac') - patch('gc.patch') + patch('gc.patch', when='@:0.5') + patch('openblas.patch') - # Build-time dependencies - depends_on("cmake @2.8:") + # Build-time dependencies: # depends_on("awk") # depends_on("m4") # depends_on("pkg-config") - depends_on("python @2.6:2.9") - # I think that Julia requires the dependencies above, but it builds find (on - # my system) without these. We should enable them as necessary. + # Combined build-time and run-time dependencies: + depends_on("cmake @2.8:") + depends_on("git") + depends_on("openssl") + depends_on("python @2.7:2.999") - # Run-time dependencies + # I think that Julia requires the dependencies above, but it + # builds fine (on my system) without these. We should enable them + # as necessary. + + # Run-time dependencies: # depends_on("arpack") # depends_on("fftw +float") # depends_on("gmp") + # depends_on("libgit") # depends_on("mpfr") + # depends_on("openblas") # depends_on("pcre2") - # ARPACK: Requires BLAS and LAPACK; needs to use the same version as Julia. + # ARPACK: Requires BLAS and LAPACK; needs to use the same version + # as Julia. - # BLAS and LAPACK: Julia prefers 64-bit versions on 64-bit systems. OpenBLAS - # has an option for this; make it available as variant. + # BLAS and LAPACK: Julia prefers 64-bit versions on 64-bit + # systems. OpenBLAS has an option for this; make it available as + # variant. - # FFTW: Something doesn't work when using a pre-installed FFTW library; need - # to investigate. + # FFTW: Something doesn't work when using a pre-installed FFTW + # library; need to investigate. - # GMP, MPFR: Something doesn't work when using a pre-installed FFTW library; - # need to investigate. + # GMP, MPFR: Something doesn't work when using a pre-installed + # FFTW library; need to investigate. - # LLVM: Julia works only with specific versions, and might require patches. - # Thus we let Julia install its own LLVM. + # LLVM: Julia works only with specific versions, and might require + # patches. Thus we let Julia install its own LLVM. # Other possible dependencies: # USE_SYSTEM_OPENLIBM=0 @@ -50,11 +62,21 @@ class Julia(Package): # USE_SYSTEM_UTF8PROC=0 # USE_SYSTEM_LIBGIT2=0 + # Run-time dependencies for Julia packages: + depends_on("hdf5") + depends_on("mpi") + def install(self, spec, prefix): - # Explicitly setting CC, CXX, or FC breaks building libuv, one of - # Julia's dependencies. This might be a Darwin-specific problem. Given - # how Spack sets up compilers, Julia should still use Spack's compilers, - # even if we don't specify them explicitly. + if '@master' in spec: + # Julia needs to know the offset from a specific commit + git = which('git') + git('fetch', '--unshallow') + + # Explicitly setting CC, CXX, or FC breaks building libuv, one + # of Julia's dependencies. This might be a Darwin-specific + # problem. Given how Spack sets up compilers, Julia should + # still use Spack's compilers, even if we don't specify them + # explicitly. options = [#"CC=cc", #"CXX=c++", #"FC=fc", From 0ebb192b2f26cfd2bbadc9be72c36469c7493d89 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Tue, 5 Apr 2016 10:36:20 -0700 Subject: [PATCH 876/910] Update the MRNet package with the latest source and patch related to the krell tools needs. Also, reorder the list of version to match with the spack standard: newest to oldest. --- .../builtin/packages/mrnet/krell-5.0.1.patch | 154 ++++++++++++++++++ .../repos/builtin/packages/mrnet/package.py | 17 +- 2 files changed, 167 insertions(+), 4 deletions(-) create mode 100644 var/spack/repos/builtin/packages/mrnet/krell-5.0.1.patch diff --git a/var/spack/repos/builtin/packages/mrnet/krell-5.0.1.patch b/var/spack/repos/builtin/packages/mrnet/krell-5.0.1.patch new file mode 100644 index 00000000000..53294fbbc63 --- /dev/null +++ b/var/spack/repos/builtin/packages/mrnet/krell-5.0.1.patch @@ -0,0 +1,154 @@ +--- mrnet-3093918/include/mrnet/Types.h 2015-12-10 09:32:24.000000000 -0800 ++++ mrnet_top_of_tree/include/mrnet/Types.h 2016-03-16 12:29:33.986132302 -0700 +@@ -23,7 +23,7 @@ + #ifndef MRNET_VERSION_MAJOR + # define MRNET_VERSION_MAJOR 5 + # define MRNET_VERSION_MINOR 0 +-# define MRNET_VERSION_REV 0 ++# define MRNET_VERSION_REV 1 + #endif + + namespace MRN +--- mrnet-3093918/include/mrnet_lightweight/Types.h 2015-12-10 09:32:24.000000000 -0800 ++++ mrnet_top_of_tree/include/mrnet_lightweight/Types.h 2016-03-16 12:29:33.987132302 -0700 +@@ -30,7 +30,7 @@ + #ifndef MRNET_VERSION_MAJOR + #define MRNET_VERSION_MAJOR 5 + #define MRNET_VERSION_MINOR 0 +-#define MRNET_VERSION_REV 0 ++#define MRNET_VERSION_REV 1 + #endif + void get_Version(int* major, + int* minor, +--- mrnet-3093918/src/lightweight/SerialGraph.c 2015-12-10 09:32:24.000000000 -0800 ++++ mrnet_top_of_tree/src/lightweight/SerialGraph.c 2016-03-16 12:29:33.995132302 -0700 +@@ -59,7 +59,7 @@ + + mrn_dbg_func_begin(); + +- sprintf(hoststr, "[%s:%hu:%u:", ihostname, iport, irank); ++ sprintf(hoststr, "[%s:%05hu:%u:", ihostname, iport, irank); + mrn_dbg(5, mrn_printf(FLF, stderr, "looking for SubTreeRoot: '%s'\n", hoststr)); + + byte_array = sg->byte_array; +@@ -110,7 +110,7 @@ + + mrn_dbg_func_begin(); + +- len = (size_t) sprintf(hoststr, "[%s:%hu:%u:0]", ihostname, iport, irank); ++ len = (size_t) sprintf(hoststr, "[%s:%05hu:%u:0]", ihostname, iport, irank); + mrn_dbg(5, mrn_printf(FLF, stderr, "adding sub tree leaf: %s\n", hoststr)); + + len += strlen(sg->byte_array) + 1; +@@ -139,7 +139,7 @@ + + mrn_dbg_func_begin(); + +- len = (size_t) sprintf(hoststr, "[%s:%hu:%u:1", ihostname, iport, irank); ++ len = (size_t) sprintf(hoststr, "[%s:%05hu:%u:1", ihostname, iport, irank); + mrn_dbg(5, mrn_printf(FLF, stderr, "adding sub tree root: %s\n", hoststr)); + + len += strlen(sg->byte_array) + 1; +@@ -360,8 +360,8 @@ + char old_hoststr[256]; + char new_hoststr[256]; + +- sprintf(old_hoststr, "[%s:%hu:%u:", hostname, UnknownPort, irank); +- sprintf(new_hoststr, "[%s:%hu:%u:", hostname, port, irank); ++ sprintf(old_hoststr, "[%s:%05hu:%u:", hostname, UnknownPort, irank); ++ sprintf(new_hoststr, "[%s:%05hu:%u:", hostname, port, irank); + + old_byte_array = sg->byte_array; + new_byte_array = (char*) malloc( strlen(old_byte_array) + 10 ); +--- mrnet-3093918/xplat/src/lightweight/SocketUtils.c 2015-12-10 09:32:24.000000000 -0800 ++++ mrnet_top_of_tree/xplat/src/lightweight/SocketUtils.c 2016-03-16 12:29:34.006132303 -0700 +@@ -15,7 +15,7 @@ + #else + const XPlat_Socket InvalidSocket = INVALID_SOCKET; + #endif +-const XPlat_Port InvalidPort = (XPlat_Port)-1; ++const XPlat_Port InvalidPort = (XPlat_Port)0; + + static bool_t SetTcpNoDelay( XPlat_Socket sock ) + { +--- mrnet-3093918/conf/configure.in 2015-12-10 09:32:24.000000000 -0800 ++++ mrnet_top_of_tree/conf/configure.in 2016-03-16 12:45:54.573196781 -0700 +@@ -107,6 +107,18 @@ + AC_SUBST(PURIFY) + + ++AC_ARG_WITH(expat, ++ [AS_HELP_STRING([--with-expat=PATH], ++ [Absolute path to installation of EXPAT libraries (note: specify the path to the directory containing "include" and "lib" sub-directories)])], ++ [EXPAT_DIR="${withval}"], ++ [EXPAT_DIR=""]) ++ ++if test "x$EXPAT_DIR" = "x" ; then ++ EXPAT_LIB="" ++else ++ EXPAT_LIB="-L$EXPAT_DIR/lib" ++fi ++ + dnl === Checks for header files. + AC_CHECK_HEADERS([assert.h errno.h fcntl.h limits.h netdb.h signal.h stddef.h stdlib.h stdio.h string.h unistd.h arpa/inet.h netinet/in.h sys/ioctl.h sys/socket.h sys/sockio.h sys/time.h]) + AC_HEADER_STDBOOL +@@ -432,7 +444,7 @@ + CRAYXT_ATH_LIBS_SO="$CRAYXT_ATH_LIBS -lalps" + CRAYXT_ATH_LIBS="$CRAYXT_ATH_LIBS -Wl,-Bstatic -lalps -lxmlrpc -Wl,-Bdynamic" + CRAYXE_ATH_LIBS_SO="$CRAYXE_ATH_LIBS -lalps" +- CRAYXE_ATH_LIBS="$CRAYXE_ATH_LIBS -Wl,-Bstatic -lalps -lxmlrpc-epi -lexpat -Wl,-Bdynamic" ++ CRAYXE_ATH_LIBS="$CRAYXE_ATH_LIBS -Wl,-Bstatic -lalps -lxmlrpc-epi $EXPAT_LIB -lexpat -Wl,-Bdynamic" + + AC_CHECK_LIB( [alps], [alps_launch_tool_helper], + [HAVE_ATH_LIBS="yes"; EXTRA_LIBS="$CRAYXT_ATH_LIBS $EXTRA_LIBS"; EXTRA_LIBS_SO="$CRAYXT_ATH_LIBS_SO $EXTRA_LIBS_SO"], +--- mrnet-3093918/configure 2015-12-10 09:32:24.000000000 -0800 ++++ mrnet_top_of_tree/configure 2016-03-16 13:47:20.386439143 -0700 +@@ -742,6 +742,7 @@ + enable_debug + enable_ltwt_threadsafe + with_purify ++with_expat + ' + ac_precious_vars='build_alias + host_alias +@@ -1399,6 +1400,9 @@ + containing "include" and "lib" sub-directories) + --with-launchmon=PATH Absolute path to installation of LaunchMON + --with-purify Use purify for memory debugging ++ --with-expat=PATH Absolute path to installation of EXPAT libraries ++ (note: specify the path to the directory containing ++ "include" and "lib" sub-directories) + + Some influential environment variables: + CC C compiler command +@@ -3541,6 +3545,21 @@ + + + ++# Check whether --with-expat was given. ++if test "${with_expat+set}" = set; then : ++ withval=$with_expat; EXPAT_DIR="${withval}" ++else ++ EXPAT_DIR="" ++fi ++ ++ ++if test "x$EXPAT_DIR" = "x" ; then ++ EXPAT_LIB="" ++else ++ EXPAT_LIB="-L$EXPAT_DIR/lib" ++fi ++ ++ + ac_ext=cpp + ac_cpp='$CXXCPP $CPPFLAGS' + ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +@@ -5473,7 +5492,7 @@ + CRAYXT_ATH_LIBS_SO="$CRAYXT_ATH_LIBS -lalps" + CRAYXT_ATH_LIBS="$CRAYXT_ATH_LIBS -Wl,-Bstatic -lalps -lxmlrpc -Wl,-Bdynamic" + CRAYXE_ATH_LIBS_SO="$CRAYXE_ATH_LIBS -lalps" +- CRAYXE_ATH_LIBS="$CRAYXE_ATH_LIBS -Wl,-Bstatic -lalps -lxmlrpc-epi -lexpat -Wl,-Bdynamic" ++ CRAYXE_ATH_LIBS="$CRAYXE_ATH_LIBS -Wl,-Bstatic -lalps -lxmlrpc-epi $EXPAT_LIB -lexpat -Wl,-Bdynamic" + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for alps_launch_tool_helper in -lalps" >&5 + $as_echo_n "checking for alps_launch_tool_helper in -lalps... " >&6; } diff --git a/var/spack/repos/builtin/packages/mrnet/package.py b/var/spack/repos/builtin/packages/mrnet/package.py index fed944e45f3..b8cbb7e8f9b 100644 --- a/var/spack/repos/builtin/packages/mrnet/package.py +++ b/var/spack/repos/builtin/packages/mrnet/package.py @@ -3,16 +3,25 @@ class Mrnet(Package): """The MRNet Multi-Cast Reduction Network.""" homepage = "http://paradyn.org/mrnet" - url = "ftp://ftp.cs.wisc.edu/paradyn/mrnet/mrnet_4.0.0.tar.gz" + url = "ftp://ftp.cs.wisc.edu/paradyn/mrnet/mrnet_5.0.1.tar.gz" + list_url = "http://ftp.cs.wisc.edu/paradyn/mrnet" - version('4.0.0', 'd00301c078cba57ef68613be32ceea2f') - version('4.1.0', '5a248298b395b329e2371bf25366115c') version('5.0.1', '17f65738cf1b9f9b95647ff85f69ecdd') + version('4.1.0', '5a248298b395b329e2371bf25366115c') + version('4.0.0', 'd00301c078cba57ef68613be32ceea2f') + + # Add a patch that brings mrnet-5.0.1 up to date with the current development tree + # The development tree contains fixes needed for the krell based tools + variant('krellpatch', default=False, description="Build MRNet with krell openspeedshop based patch.") + patch('krell-5.0.1.patch', when='@5.0.1+krellpatch') + + variant('lwthreads', default=False, description="Also build the MRNet LW threadsafe libraries") parallel = False - depends_on("boost") + #depends_on("boost") + depends_on("boost@1.53.0") def install(self, spec, prefix): # Build the MRNet LW thread safe libraries when the krelloptions variant is present From 21ad5162c41b75aa11a5622a58a022a3107e6990 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Tue, 5 Apr 2016 11:53:37 -0600 Subject: [PATCH 877/910] + Global requires ncurses. --- var/spack/repos/builtin/packages/global/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/global/package.py b/var/spack/repos/builtin/packages/global/package.py index e8f06516d9a..aac1cede30f 100644 --- a/var/spack/repos/builtin/packages/global/package.py +++ b/var/spack/repos/builtin/packages/global/package.py @@ -11,6 +11,7 @@ class Global(Package): version('6.5', 'dfec818b4f53d91721e247cf7b218078') depends_on('exuberant-ctags') + depends_on('ncurses') def install(self, spec, prefix): config_args = ['--prefix={0}'.format(prefix)] From 9b3c7b8afa40bd30e8bd54be12f5377ccd448c47 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 6 Apr 2016 10:55:59 +0200 Subject: [PATCH 878/910] openblas: provide basename of compilers to avoid issues with detection of Fortran --- var/spack/repos/builtin/packages/openblas/package.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 4ec829a85b0..9f13d0690be 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -19,8 +19,11 @@ class Openblas(Package): def install(self, spec, prefix): - make_defs = ['CC=%s' % spack_cc, - 'FC=%s' % spack_fc] + # Openblas is picky about compilers. Configure fails with + # FC=/abs/path/to/f77, whereas FC=f77 works fine. + # To circumvent this, provide basename only: + make_defs = ['CC=%s' % os.path.basename(spack_cc), + 'FC=%s' % os.path.basename(spack_f77)] make_targets = ['libs', 'netlib'] @@ -67,4 +70,3 @@ def setup_dependent_package(self, module, dspec): if '+shared' in self.spec: self.spec.blas_shared_lib = join_path(libdir, 'libopenblas.%s' % dso_suffix) self.spec.lapack_shared_lib = self.spec.blas_shared_lib - From 71e9f1ab8d652e96ed0178a9aaf813a7f89aed35 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Wed, 6 Apr 2016 09:52:09 -0600 Subject: [PATCH 879/910] + I am resubmitting this proposed change because it was accidentally overwritten. These modifications were accepted as PR#741 but somehow I clobbered them when PR#742 was merged. There was some unintentional overlap in my local repository that I didn't straighten out before pushing. --- .../repos/builtin/packages/netlib-lapack/package.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py index 05436332ac5..f70e634347c 100644 --- a/var/spack/repos/builtin/packages/netlib-lapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -34,15 +34,17 @@ class NetlibLapack(Package): def patch(self): # Fix cblas CMakeLists.txt -- has wrong case for subdirectory name. - filter_file('${CMAKE_CURRENT_SOURCE_DIR}/CMAKE/', - '${CMAKE_CURRENT_SOURCE_DIR}/cmake/', 'CBLAS/CMakeLists.txt', string=True) - + if self.spec.satisfies('@3.6.0:'): + filter_file('${CMAKE_CURRENT_SOURCE_DIR}/CMAKE/', + '${CMAKE_CURRENT_SOURCE_DIR}/cmake/', 'CBLAS/CMakeLists.txt', string=True) def install_one(self, spec, prefix, shared): cmake_args = ['-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if shared else 'OFF'), - '-DCBLAS=ON', # always build CBLAS '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), '-DLAPACKE:BOOL=%s' % ('ON' if '+lapacke' in spec else 'OFF')] + if spec.satisfies('@3.6.0:'): + cmake_args.extend(['-DCBLAS=ON']) # always build CBLAS + if '+external-blas' in spec: # TODO : the mechanism to specify the library should be more general, # TODO : but this allows to have an hook to an external blas @@ -80,6 +82,3 @@ def setup_dependent_package(self, module, dspec): if '+shared' in self.spec: self.spec.blas_shared_lib = join_path(libdir, 'libblas.%s' % dso_suffix) self.spec.lapack_shared_lib = join_path(libdir, 'liblapack.%s' % dso_suffix) - - - From 73b6214a13339f11dbc33b0e068c13499f139b6f Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 22 Mar 2016 15:23:46 +0100 Subject: [PATCH 880/910] module files : proper cleanup on uninstall fixes #216 Conflicts: lib/spack/spack/test/database.py --- lib/spack/spack/cmd/module.py | 10 ++-------- lib/spack/spack/modules.py | 6 +++++- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 315d9fc9266..a67f5c0c137 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -22,21 +22,16 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys import os import shutil -import argparse +import sys import llnl.util.tty as tty -from llnl.util.lang import partition_list -from llnl.util.filesystem import mkdirp - import spack.cmd +from llnl.util.filesystem import mkdirp from spack.modules import module_types from spack.util.string import * -from spack.spec import Spec - description ="Manipulate modules and dotkits." @@ -98,7 +93,6 @@ def module_refresh(): cls(spec).write() - def module(parser, args): if args.module_command == 'refresh': module_refresh() diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index d797af287d3..61624fbd703 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -211,7 +211,11 @@ def use_name(self): def remove(self): mod_file = self.file_name if os.path.exists(mod_file): - shutil.rmtree(mod_file, ignore_errors=True) + try: + os.remove(mod_file) # Remove the module file + os.removedirs(os.path.dirname(mod_file)) # Remove all the empty directories from the leaf up + except OSError: + pass # removedirs throws OSError on first non-empty directory found class Dotkit(EnvModule): From 400166398239be0880c025672dd3e5eb6c8bac7f Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 6 Apr 2016 22:19:07 +0200 Subject: [PATCH 881/910] leftover from cherry-pick --- lib/spack/spack/test/mock_database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/mock_database.py b/lib/spack/spack/test/mock_database.py index 6fd05439bff..82ba59fc483 100644 --- a/lib/spack/spack/test/mock_database.py +++ b/lib/spack/spack/test/mock_database.py @@ -17,7 +17,7 @@ def _mock_install(self, spec): def _mock_remove(self, spec): specs = spack.installed_db.query(spec) - assert(len(specs) == 1) + assert len(specs) == 1 spec = specs[0] spec.package.do_uninstall(spec) @@ -71,6 +71,8 @@ def setUp(self): self._mock_install('mpileaks ^zmpi') def tearDown(self): + for spec in spack.installed_db.query(): + spec.package.do_uninstall(spec) super(MockDatabase, self).tearDown() shutil.rmtree(self.install_path) spack.install_path = self.spack_install_path From 380a2b23e66fe0acfd16f9e72499c7bd42597eaf Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 6 Apr 2016 15:57:03 -0500 Subject: [PATCH 882/910] Add bash package --- .../repos/builtin/packages/bash/package.py | 20 +++++++++++++++++++ .../repos/builtin/packages/ncurses/package.py | 7 +++---- .../builtin/packages/readline/package.py | 4 ++-- 3 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 var/spack/repos/builtin/packages/bash/package.py diff --git a/var/spack/repos/builtin/packages/bash/package.py b/var/spack/repos/builtin/packages/bash/package.py new file mode 100644 index 00000000000..9c9fbeedcf8 --- /dev/null +++ b/var/spack/repos/builtin/packages/bash/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Bash(Package): + """The GNU Project's Bourne Again SHell.""" + + homepage = "https://www.gnu.org/software/bash/" + url = "ftp://ftp.gnu.org/gnu/bash/bash-4.3.tar.gz" + + version('4.3', '81348932d5da294953e15d4814c74dd1') + + depends_on('readline') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, + '--with-curses', + '--with-installed-readline=%s' % spec['readline'].prefix) + + make() + make("tests") + make("install") diff --git a/var/spack/repos/builtin/packages/ncurses/package.py b/var/spack/repos/builtin/packages/ncurses/package.py index 8dc808caacc..219fbce226e 100644 --- a/var/spack/repos/builtin/packages/ncurses/package.py +++ b/var/spack/repos/builtin/packages/ncurses/package.py @@ -8,11 +8,10 @@ class Ncurses(Package): """ homepage = "http://invisible-island.net/ncurses/ncurses.html" + url = "http://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.0.tar.gz" - version('5.9', '8cb9c412e5f2d96bc6f459aa8c6282a1', - url='http://ftp.gnu.org/pub/gnu/ncurses/ncurses-5.9.tar.gz') - version('6.0', 'ee13d052e1ead260d7c28071f46eefb1', - url='http://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.0.tar.gz') + version('6.0', 'ee13d052e1ead260d7c28071f46eefb1') + version('5.9', '8cb9c412e5f2d96bc6f459aa8c6282a1') patch('patch_gcc_5.txt', when='%gcc@5.0:') diff --git a/var/spack/repos/builtin/packages/readline/package.py b/var/spack/repos/builtin/packages/readline/package.py index 1b870e0e7ff..0c429ea7568 100644 --- a/var/spack/repos/builtin/packages/readline/package.py +++ b/var/spack/repos/builtin/packages/readline/package.py @@ -2,12 +2,12 @@ class Readline(Package): """The GNU Readline library provides a set of functions for use by - applications that allow users to edit command li nes as they + applications that allow users to edit command lines as they are typed in. Both Emacs and vi editing modes are available. The Readline library includes additional functions to maintain a list of previously-entered command lines, to recall and perhaps reedit those lines, and perform csh-like - history expansion on previous commands. """ + history expansion on previous commands.""" homepage = "http://cnswww.cns.cwru.edu/php/chet/readline/rltop.html" url = "ftp://ftp.cwru.edu/pub/bash/readline-6.3.tar.gz" From c113d390a7a6fbd171aae54aa1f081f8b0790eab Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 6 Apr 2016 23:51:47 +0200 Subject: [PATCH 883/910] mvapich2 : MPI compiler wrapper will use spack compilers during the installation phase mpich : fixed wrong function signature --- var/spack/repos/builtin/packages/mpich/package.py | 12 ++++++------ var/spack/repos/builtin/packages/mvapich2/package.py | 7 +++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 2d7955e08d3..b317ec6651d 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -47,12 +47,12 @@ class Mpich(Package): provides('mpi@:3.0', when='@3:') provides('mpi@:1.3', when='@1:') - def setup_dependent_environment(self, env, dependent_spec): - env.set('MPICH_CC', spack_cc) - env.set('MPICH_CXX', spack_cxx) - env.set('MPICH_F77', spack_f77) - env.set('MPICH_F90', spack_f90) - env.set('MPICH_FC', spack_fc) + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + spack_env.set('MPICH_CC', spack_cc) + spack_env.set('MPICH_CXX', spack_cxx) + spack_env.set('MPICH_F77', spack_f77) + spack_env.set('MPICH_F90', spack_fc) + spack_env.set('MPICH_FC', spack_fc) def setup_dependent_package(self, module, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index e4e95f92af7..3e60b517dbe 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -140,6 +140,13 @@ def set_network_type(self, spec, configure_args): configure_args.extend(network_options) + def setup_dependent_environment(self, spack_env, run_env, extension_spec): + spack_env.set('MPICH_CC', spack_cc) + spack_env.set('MPICH_CXX', spack_cxx) + spack_env.set('MPICH_F77', spack_f77) + spack_env.set('MPICH_F90', spack_fc) + spack_env.set('MPICH_FC', spack_fc) + def install(self, spec, prefix): # we'll set different configure flags depending on our environment configure_args = [ From ddaba07d8297544f2587f0c31fcbd2dafa824eb3 Mon Sep 17 00:00:00 2001 From: Geoffrey Oxberry Date: Tue, 5 Apr 2016 19:00:28 -0700 Subject: [PATCH 884/910] metis 4.0.3 Add version 4.0.3 to metis package. Attempted to implement reasonable versions of all variants declared for metis@5.1.0; some of these do not have analogues in metis@4.0.3, and errors are raised accordingly. Also updated dependencies of packages with depends_on('metis') to depends_on('metis@5:') to ensure that these packages still build. --- .../repos/builtin/packages/dealii/package.py | 2 +- .../repos/builtin/packages/eigen/package.py | 2 +- .../repos/builtin/packages/metis/package.py | 83 ++++++++++++++++++- .../repos/builtin/packages/mumps/package.py | 2 +- .../builtin/packages/parmetis/package.py | 2 +- .../repos/builtin/packages/petsc/package.py | 2 +- .../builtin/packages/superlu-dist/package.py | 2 +- .../builtin/packages/trilinos/package.py | 2 +- 8 files changed, 86 insertions(+), 11 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 1f763ad3588..b251d50ca1d 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -40,7 +40,7 @@ class Dealii(Package): depends_on ("arpack-ng+mpi", when='+arpack+mpi') depends_on ("doxygen", when='+doc') depends_on ("hdf5+mpi~cxx", when='+hdf5+mpi') #FIXME NetCDF declares dependency with ~cxx, why? - depends_on ("metis", when='+metis') + depends_on ("metis@5:", when='+metis') depends_on ("netcdf+mpi", when="+netcdf+mpi") depends_on ("netcdf-cxx", when='+netcdf+mpi') depends_on ("oce", when='+oce') diff --git a/var/spack/repos/builtin/packages/eigen/package.py b/var/spack/repos/builtin/packages/eigen/package.py index 8d6e672f86c..1501989812d 100644 --- a/var/spack/repos/builtin/packages/eigen/package.py +++ b/var/spack/repos/builtin/packages/eigen/package.py @@ -45,7 +45,7 @@ class Eigen(Package): # TODO : dependency on googlehash, superlu, adolc missing - depends_on('metis', when='+metis') + depends_on('metis@5:', when='+metis') depends_on('scotch', when='+scotch') depends_on('fftw', when='+fftw') depends_on('suite-sparse', when='+suitesparse') diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py index d3bab554fec..e348eca4ba1 100644 --- a/var/spack/repos/builtin/packages/metis/package.py +++ b/var/spack/repos/builtin/packages/metis/package.py @@ -24,7 +24,7 @@ ############################################################################## from spack import * -import glob,sys +import glob, sys, os class Metis(Package): """ @@ -37,6 +37,8 @@ class Metis(Package): url = "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz" version('5.1.0', '5465e67079419a69e0116de24fce58fe') + version('4.0.3', '5efa35de80703c1b2c4d0de080fafbcf4e0d363a21149a1ad2f96e0144841a55', + url='http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz') variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds the library in debug mode') @@ -45,12 +47,85 @@ class Metis(Package): variant('idx64', default=False, description='Use int64_t as default index type') variant('double', default=False, description='Use double precision floating point types') - depends_on('cmake @2.8:') # build-time dependency - + depends_on('cmake @2.8:', when='@5:') # build-time dependency depends_on('gdb', when='+gdb') - patch('install_gklib_defs_rename.patch') + patch('install_gklib_defs_rename.patch', when='@5:') + + @when('@4:4.0.3') + def install(self, spec, prefix): + + if '+gdb' in spec: + raise InstallError('gdb support not implemented in METIS 4!') + if '+idx64' in spec: + raise InstallError('idx64 option not implemented in METIS 4!') + if '+double' in spec: + raise InstallError('double option not implemented for METIS 4!') + + options = ['COPTIONS=-fPIC'] + if '+debug' in spec: + options.append('OPTFLAGS=-g -O0') + make(*options) + + mkdir(prefix.bin) + for x in ('pmetis', 'kmetis', 'oemetis', 'onmetis', 'partnmesh', + 'partdmesh', 'mesh2nodal', 'mesh2dual', 'graphchk'): + install(x, prefix.bin) + + mkdir(prefix.lib) + install('libmetis.a', prefix.lib) + + mkdir(prefix.include) + for h in glob.glob(join_path('Lib', '*.h')): + install(h, prefix.include) + + mkdir(prefix.share) + for f in (join_path(*p) + for p in (('Programs', 'io.c'), + ('Test','mtest.c'), + ('Graphs','4elt.graph'), + ('Graphs', 'metis.mesh'), + ('Graphs', 'test.mgraph'))): + install(f, prefix.share) + + if '+shared' in spec: + if sys.platform == 'darwin': + lib_dsuffix = 'dylib' + load_flag = '-Wl,-all_load' + no_load_flag = '' + else: + lib_dsuffix = 'so' + load_flag = '-Wl,-whole-archive' + no_load_flag = '-Wl,-no-whole-archive' + + os.system(spack_cc + ' -fPIC -shared ' + load_flag + + ' libmetis.a ' + no_load_flag + ' -o libmetis.' + + lib_dsuffix) + install('libmetis.' + lib_dsuffix, prefix.lib) + + # Set up and run tests on installation + symlink(join_path(prefix.share, 'io.c'), 'io.c') + symlink(join_path(prefix.share, 'mtest.c'), 'mtest.c') + os.system(spack_cc + ' -I%s' % prefix.include + ' -c io.c') + os.system(spack_cc + ' -I%s' % prefix.include + + ' -L%s' % prefix.lib + ' -lmetis mtest.c io.o -o mtest') + _4eltgraph = join_path(prefix.share, '4elt.graph') + test_mgraph = join_path(prefix.share, 'test.mgraph') + metis_mesh = join_path(prefix.share, 'metis.mesh') + kmetis = join_path(prefix.bin, 'kmetis') + os.system('./mtest ' + _4eltgraph) + os.system(kmetis + ' ' + _4eltgraph + ' 40') + os.system(join_path(prefix.bin, 'onmetis') + ' ' + _4eltgraph) + os.system(join_path(prefix.bin, 'pmetis') + ' ' + test_mgraph + ' 2') + os.system(kmetis + ' ' + test_mgraph + ' 2') + os.system(kmetis + ' ' + test_mgraph + ' 5') + os.system(join_path(prefix.bin, 'partnmesh') + metis_mesh + ' 10') + os.system(join_path(prefix.bin, 'partdmesh') + metis_mesh + ' 10') + os.system(join_path(prefix.bin, 'mesh2dual') + metis_mesh) + + + @when('@5:') def install(self, spec, prefix): options = [] diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index 025d86ebdc2..58f790ec32b 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -23,7 +23,7 @@ class Mumps(Package): depends_on('scotch + esmumps', when='~ptscotch+scotch') depends_on('scotch + esmumps + mpi', when='+ptscotch') - depends_on('metis', when='+metis') + depends_on('metis@5:', when='+metis') depends_on('parmetis', when="+parmetis") depends_on('blas') depends_on('lapack') diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py index ff4370aa4b4..b49f8dae00d 100644 --- a/var/spack/repos/builtin/packages/parmetis/package.py +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -44,7 +44,7 @@ class Parmetis(Package): depends_on('mpi') patch('enable_external_metis.patch') - depends_on('metis') + depends_on('metis@5:') # bug fixes from PETSc developers # https://bitbucket.org/petsc/pkg-parmetis/commits/1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b/raw/ diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 5c1fc6cc924..1161dd7d67a 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -40,7 +40,7 @@ class Petsc(Package): # Other dependencies depends_on('boost', when='+boost') - depends_on('metis', when='+metis') + depends_on('metis@5:', when='+metis') depends_on('hdf5+mpi', when='+hdf5+mpi') depends_on('parmetis', when='+metis+mpi') diff --git a/var/spack/repos/builtin/packages/superlu-dist/package.py b/var/spack/repos/builtin/packages/superlu-dist/package.py index ddcb7f92252..5cf5e129b41 100644 --- a/var/spack/repos/builtin/packages/superlu-dist/package.py +++ b/var/spack/repos/builtin/packages/superlu-dist/package.py @@ -15,7 +15,7 @@ class SuperluDist(Package): depends_on ('blas') depends_on ('lapack') depends_on ('parmetis') - depends_on ('metis') + depends_on ('metis@5:') def install(self, spec, prefix): makefile_inc = [] diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 6223848c689..0f72055fa72 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -42,7 +42,7 @@ class Trilinos(Package): depends_on('matio') depends_on('glm') depends_on('swig') - depends_on('metis',when='+metis') + depends_on('metis@5:',when='+metis') depends_on('suite-sparse',when='+suite-sparse') # MPI related dependencies From bbd328d307b6b6a1350114a68a1ce294347a0176 Mon Sep 17 00:00:00 2001 From: Geoffrey Oxberry Date: Wed, 6 Apr 2016 16:58:48 -0700 Subject: [PATCH 885/910] metis 5: fix download url Spack picks up the wrong download for metis 5.1.0 from parsing the url field for metis 4.0.3. Add an explicit url field to fix this bug. --- var/spack/repos/builtin/packages/metis/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py index e348eca4ba1..41e3ebb429f 100644 --- a/var/spack/repos/builtin/packages/metis/package.py +++ b/var/spack/repos/builtin/packages/metis/package.py @@ -36,7 +36,8 @@ class Metis(Package): homepage = 'http://glaros.dtc.umn.edu/gkhome/metis/metis/overview' url = "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz" - version('5.1.0', '5465e67079419a69e0116de24fce58fe') + version('5.1.0', '5465e67079419a69e0116de24fce58fe', + url='http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz') version('4.0.3', '5efa35de80703c1b2c4d0de080fafbcf4e0d363a21149a1ad2f96e0144841a55', url='http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz') From 7d19154e1852ad43af889505e19a36873d0e9c18 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 8 Apr 2016 12:01:28 -0400 Subject: [PATCH 886/910] pkg-config: use the internal glib glib requires pkg-config itself, so on machines without pkg-config, there's a bootstrapping problem. --- var/spack/repos/builtin/packages/pkg-config/package.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/pkg-config/package.py b/var/spack/repos/builtin/packages/pkg-config/package.py index 9964c6ce346..a803bc3f9b8 100644 --- a/var/spack/repos/builtin/packages/pkg-config/package.py +++ b/var/spack/repos/builtin/packages/pkg-config/package.py @@ -10,7 +10,12 @@ class PkgConfig(Package): parallel = False def install(self, spec, prefix): - configure("--prefix=%s" %prefix, "--enable-shared") + configure("--prefix=%s" %prefix, + "--enable-shared", + "--with-internal-glib") # There's a bootstrapping problem here; + # glib uses pkg-config as well, so + # break the cycle by using the internal + # glib. make() make("install") From 6c409d6b9238199370512ff8a8d30c6b2ff67769 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 8 Apr 2016 12:03:05 -0400 Subject: [PATCH 887/910] py-matplotlib: depend on pkg-config On OS X, freetype isn't found by default, but pkg-config can. --- var/spack/repos/builtin/packages/py-matplotlib/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-matplotlib/package.py b/var/spack/repos/builtin/packages/py-matplotlib/package.py index 45e77dd6313..19194c942e7 100644 --- a/var/spack/repos/builtin/packages/py-matplotlib/package.py +++ b/var/spack/repos/builtin/packages/py-matplotlib/package.py @@ -26,6 +26,7 @@ class PyMatplotlib(Package): depends_on('py-pbr') depends_on('py-funcsigs') + depends_on('pkg-config') depends_on('freetype') depends_on('qt', when='+gui') depends_on('bzip2') From dcd6b1934808e3700a3d1fc71e261087d9e71ae7 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 8 Apr 2016 12:03:36 -0400 Subject: [PATCH 888/910] py-setuptools: sort versions --- var/spack/repos/builtin/packages/py-setuptools/package.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-setuptools/package.py b/var/spack/repos/builtin/packages/py-setuptools/package.py index c6d9be1add7..38158f4d27c 100644 --- a/var/spack/repos/builtin/packages/py-setuptools/package.py +++ b/var/spack/repos/builtin/packages/py-setuptools/package.py @@ -5,11 +5,11 @@ class PySetuptools(Package): homepage = "https://pypi.python.org/pypi/setuptools" url = "https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz" - version('11.3.1', '01f69212e019a2420c1693fb43593930') - version('16.0', '0ace0b96233516fc5f7c857d086aa3ad') - version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06') - version('19.2', '78353b1f80375ca5e088f4b4627ffe03') version('20.5', 'fadc1e1123ddbe31006e5e43e927362b') + version('19.2', '78353b1f80375ca5e088f4b4627ffe03') + version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06') + version('16.0', '0ace0b96233516fc5f7c857d086aa3ad') + version('11.3.1', '01f69212e019a2420c1693fb43593930') extends('python') From 9eaf735bcdba9f3fd8a51219d0eebb61f8b2f166 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 8 Apr 2016 12:03:45 -0400 Subject: [PATCH 889/910] py-setuptools: add 20.6.7 --- var/spack/repos/builtin/packages/py-setuptools/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-setuptools/package.py b/var/spack/repos/builtin/packages/py-setuptools/package.py index 38158f4d27c..bc3420a25e8 100644 --- a/var/spack/repos/builtin/packages/py-setuptools/package.py +++ b/var/spack/repos/builtin/packages/py-setuptools/package.py @@ -5,6 +5,7 @@ class PySetuptools(Package): homepage = "https://pypi.python.org/pypi/setuptools" url = "https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz" + version('20.6.7', '45d6110f3ec14924e44c33411db64fe6') version('20.5', 'fadc1e1123ddbe31006e5e43e927362b') version('19.2', '78353b1f80375ca5e088f4b4627ffe03') version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06') From 1af88be3712d1b6090a2dde5d7a68405f7372ca7 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Fri, 8 Apr 2016 11:06:14 -0700 Subject: [PATCH 890/910] Spack was no longer using $TMPDIR for its stage area at LLNL. Spack's directory search was unnecessarily putting candidates that contained your username at the end of its search list. --- lib/spack/spack/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index aee11f061f3..9108e1d0e33 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -136,9 +136,7 @@ # don't add a second username if it's already unique by user. if not _tmp_user in path: tmp_dirs.append(join_path(path, '%u', 'spack-stage')) - -for path in _tmp_candidates: - if not path in tmp_dirs: + else: tmp_dirs.append(join_path(path, 'spack-stage')) # Whether spack should allow installation of unsafe versions of From f461b9972265df09e8ff9ac5c99a3f3d96783883 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 8 Apr 2016 14:31:40 -0400 Subject: [PATCH 891/910] Building OpenBLAS requires recent binutils --- var/spack/repos/builtin/packages/julia/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/julia/package.py b/var/spack/repos/builtin/packages/julia/package.py index b3a523bc450..75d13fda8f9 100644 --- a/var/spack/repos/builtin/packages/julia/package.py +++ b/var/spack/repos/builtin/packages/julia/package.py @@ -20,6 +20,7 @@ class Julia(Package): # depends_on("pkg-config") # Combined build-time and run-time dependencies: + depends_on("binutils") depends_on("cmake @2.8:") depends_on("git") depends_on("openssl") From 3a4aac0213f46ff233b6252be5db7c645501ab34 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 8 Apr 2016 15:08:09 -0400 Subject: [PATCH 892/910] paraview: use the right cmake variables --- .../repos/builtin/packages/paraview/package.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index c16054816c1..888add30332 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -75,13 +75,13 @@ def nfeature_to_bool(feature): cmake('..', '-DCMAKE_INSTALL_PREFIX:PATH=%s' % prefix, '-DBUILD_TESTING:BOOL=OFF', - '-DVTK_USER_SYSTEM_FREETYPE:BOOL=ON', - '-DVTK_USER_SYSTEM_HDF5:BOOL=ON', - '-DVTK_USER_SYSTEM_JPEG:BOOL=ON', - '-DVTK_USER_SYSTEM_LIBXML2:BOOL=ON', - '-DVTK_USER_SYSTEM_NETCDF:BOOL=ON', - '-DVTK_USER_SYSTEM_TIFF:BOOL=ON', - '-DVTK_USER_SYSTEM_ZLIB:BOOL=ON', + '-DVTK_USE_SYSTEM_FREETYPE:BOOL=ON', + '-DVTK_USE_SYSTEM_HDF5:BOOL=ON', + '-DVTK_USE_SYSTEM_JPEG:BOOL=ON', + '-DVTK_USE_SYSTEM_LIBXML2:BOOL=ON', + '-DVTK_USE_SYSTEM_NETCDF:BOOL=ON', + '-DVTK_USE_SYSTEM_TIFF:BOOL=ON', + '-DVTK_USE_SYSTEM_ZLIB:BOOL=ON', *feature_args) make() make('install') From 6e56ba9f24df62ad205006a6b7bc13ae067a6623 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 8 Apr 2016 15:08:29 -0400 Subject: [PATCH 893/910] paraview: use internal netcdf VTK needs to learn to cope with netcdf being split like this. --- var/spack/repos/builtin/packages/paraview/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index 888add30332..13cd076286f 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -33,7 +33,8 @@ class Paraview(Package): depends_on('libpng') depends_on('libtiff') depends_on('libxml2') - depends_on('netcdf') + #depends_on('netcdf') + #depends_on('netcdf-cxx') #depends_on('protobuf') # version mismatches? #depends_on('sqlite') # external version not supported depends_on('zlib') @@ -79,7 +80,7 @@ def nfeature_to_bool(feature): '-DVTK_USE_SYSTEM_HDF5:BOOL=ON', '-DVTK_USE_SYSTEM_JPEG:BOOL=ON', '-DVTK_USE_SYSTEM_LIBXML2:BOOL=ON', - '-DVTK_USE_SYSTEM_NETCDF:BOOL=ON', + '-DVTK_USE_SYSTEM_NETCDF:BOOL=OFF', '-DVTK_USE_SYSTEM_TIFF:BOOL=ON', '-DVTK_USE_SYSTEM_ZLIB:BOOL=ON', *feature_args) From 20b9f34b5c650eefb23f4118bb52985c468a9098 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 8 Apr 2016 15:09:02 -0400 Subject: [PATCH 894/910] paraview: use internal hdf5 Spack's HDF5 is too new. Rather than forcing everything in a ParaView chain to use older HDF5, use the internal one until ParaView is patched properly. --- var/spack/repos/builtin/packages/paraview/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index 13cd076286f..60f8d3c243d 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -27,8 +27,8 @@ class Paraview(Package): depends_on('bzip2') depends_on('freetype') - depends_on('hdf5+mpi', when='+mpi') - depends_on('hdf5~mpi', when='~mpi') + #depends_on('hdf5+mpi', when='+mpi') + #depends_on('hdf5~mpi', when='~mpi') depends_on('jpeg') depends_on('libpng') depends_on('libtiff') @@ -77,7 +77,7 @@ def nfeature_to_bool(feature): '-DCMAKE_INSTALL_PREFIX:PATH=%s' % prefix, '-DBUILD_TESTING:BOOL=OFF', '-DVTK_USE_SYSTEM_FREETYPE:BOOL=ON', - '-DVTK_USE_SYSTEM_HDF5:BOOL=ON', + '-DVTK_USE_SYSTEM_HDF5:BOOL=OFF', '-DVTK_USE_SYSTEM_JPEG:BOOL=ON', '-DVTK_USE_SYSTEM_LIBXML2:BOOL=ON', '-DVTK_USE_SYSTEM_NETCDF:BOOL=OFF', From df7e3f8635530574c5190d9501f2fdf1b0aa0aff Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 9 Apr 2016 09:58:48 -0400 Subject: [PATCH 895/910] Correct version dependency for OpenBLAS patch --- var/spack/repos/builtin/packages/julia/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/julia/package.py b/var/spack/repos/builtin/packages/julia/package.py index 75d13fda8f9..25d782266b3 100644 --- a/var/spack/repos/builtin/packages/julia/package.py +++ b/var/spack/repos/builtin/packages/julia/package.py @@ -11,8 +11,8 @@ class Julia(Package): version('0.4.5', '69141ff5aa6cee7c0ec8c85a34aa49a6') version('0.4.3', '8a4a59fd335b05090dd1ebefbbe5aaac') - patch('gc.patch', when='@:0.5') - patch('openblas.patch') + patch('gc.patch') + patch('openblas.patch', when='@0.4:0.4.5') # Build-time dependencies: # depends_on("awk") From cb6c6fb3747aafbfb0fe81803eacf2807dde301f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 11 Apr 2016 16:28:30 -0500 Subject: [PATCH 896/910] Add elk package --- .../repos/builtin/packages/elk/package.py | 116 ++++++++++++++++++ .../repos/builtin/packages/libxc/package.py | 18 +++ .../builtin/packages/openblas/package.py | 6 + 3 files changed, 140 insertions(+) create mode 100644 var/spack/repos/builtin/packages/elk/package.py create mode 100644 var/spack/repos/builtin/packages/libxc/package.py diff --git a/var/spack/repos/builtin/packages/elk/package.py b/var/spack/repos/builtin/packages/elk/package.py new file mode 100644 index 00000000000..28ebb89db38 --- /dev/null +++ b/var/spack/repos/builtin/packages/elk/package.py @@ -0,0 +1,116 @@ +from spack import * +import spack + +class Elk(Package): + '''An all-electron full-potential linearised augmented-plane wave + (FP-LAPW) code with many advanced features.''' + + homepage = 'http://elk.sourceforge.net/' + url = 'https://sourceforge.net/projects/elk/files/elk-3.3.17.tgz' + + version('3.3.17', 'f57f6230d14f3b3b558e5c71f62f0592') + + # Elk provides these libraries, but allows you to specify your own + variant('blas', default=True, description='Build with custom BLAS library') + variant('lapack', default=True, description='Build with custom LAPACK library') + variant('fft', default=True, description='Build with custom FFT library') + + # Elk does not provide these libraries, but allows you to use them + variant('mpi', default=True, description='Enable MPI parallelism') + variant('openmp', default=True, description='Enable OpenMP support') + variant('libxc', default=True, description='Link to Libxc functional library') + + depends_on('blas', when='+blas') + depends_on('lapack', when='+lapack') + depends_on('fftw', when='+fft') + depends_on('mpi', when='+mpi') + depends_on('libxc', when='+libxc') + + # Cannot be built in parallel + parallel = False + + + def configure(self, spec): + # Dictionary of configuration options + config = { + 'MAKE': 'make', + 'F90': join_path(spack.build_env_path, 'f90'), + 'F77': join_path(spack.build_env_path, 'f77'), + 'AR': 'ar', + 'LIB_FFT': 'fftlib.a', + 'SRC_MPI': 'mpi_stub.f90', + 'SRC_OMP': 'omp_stub.f90', + 'SRC_libxc': 'libxcifc_stub.f90', + 'SRC_FFT': 'zfftifc.f90' + } + + # Compiler-specific flags + flags = '' + if self.compiler.name == 'intel': + flags = '-O3 -ip -unroll -no-prec-div -openmp' + elif self.compiler.name == 'gcc': + flags = '-O3 -ffast-math -funroll-loops -fopenmp' + elif self.compiler.name == 'pgi': + flags = '-O3 -mp -lpthread' + elif self.compiler.name == 'g95': + flags = '-O3 -fno-second-underscore' + elif self.compiler.name == 'nag': + flags = '-O4 -kind=byte -dusty -dcfuns' + elif self.compiler.name == 'xl': + flags = '-O3 -qsmp=omp' + config['F90_OPTS'] = flags + config['F77_OPTS'] = flags + + # BLAS/LAPACK support + blas = 'blas.a' + lapack = 'lapack.a' + if '+blas' in spec: + blas = join_path(spec['blas'].prefix.lib, 'libblas.so') + if '+lapack' in spec: + lapack = join_path(spec['lapack'].prefix.lib, 'liblapack.so') + config['LIB_LPK'] = ' '.join([lapack, blas]) # lapack must come before blas + + # FFT support + if '+fft' in spec: + config['LIB_FFT'] = join_path(spec['fftw'].prefix.lib, 'libfftw3.so') + config['SRC_FFT'] = 'zfftifc_fftw.f90' + + # MPI support + if '+mpi' in spec: + config.pop('SRC_MPI') + config['F90'] = join_path(spec['mpi'].prefix.bin, 'mpif90') + config['F77'] = join_path(spec['mpi'].prefix.bin, 'mpif77') + + # OpenMP support + if '+openmp' in spec: + config.pop('SRC_OMP') + + # Libxc support + if '+libxc' in spec: + config['LIB_libxc'] = ' '.join([ + join_path(spec['libxc'].prefix.lib, 'libxcf90.so'), + join_path(spec['libxc'].prefix.lib, 'libxc.so') + ]) + config['SRC_libxc'] = ' '.join([ + 'libxc_funcs.f90', + 'libxc.f90', + 'libxcifc.f90' + ]) + + # Write configuration options to include file + with open('make.inc', 'w') as inc: + for key in config: + inc.write('{0} = {1}\n'.format(key, config[key])) + + + def install(self, spec, prefix): + self.configure(spec) + + make() + make('test') + + mkdirp(prefix.bin) + + install('src/elk', prefix.bin) + install('src/eos/eos', prefix.bin) + install('src/spacegroup/spacegroup', prefix.bin) diff --git a/var/spack/repos/builtin/packages/libxc/package.py b/var/spack/repos/builtin/packages/libxc/package.py new file mode 100644 index 00000000000..010a5918c56 --- /dev/null +++ b/var/spack/repos/builtin/packages/libxc/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Libxc(Package): + """Libxc is a library of exchange-correlation functionals for + density-functional theory.""" + + homepage = "http://www.tddft.org/programs/octopus/wiki/index.php/Libxc" + url = "http://www.tddft.org/programs/octopus/down.php?file=libxc/libxc-2.2.2.tar.gz" + + version('2.2.2', 'd9f90a0d6e36df6c1312b6422280f2ec') + + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, + '--enable-shared') + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 9f13d0690be..4522130ccc9 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -12,6 +12,7 @@ class Openblas(Package): version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') variant('shared', default=True, description="Build shared libraries as well as static libs.") + variant('openmp', default=True, description="Enable OpenMP support.") # virtual dependency provides('blas') @@ -37,6 +38,11 @@ def install(self, spec, prefix): if spec.satisfies('@0.2.16'): make_defs += ['BUILD_LAPACK_DEPRECATED=1'] + # Add support for OpenMP + # Note: Make sure your compiler supports OpenMP + if '+openmp' in spec: + make_defs += ['USE_OPENMP=1'] + make_args = make_defs + make_targets make(*make_args) From 01f2dd03d5fe092ae448ee1a234a197babf38613 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 12 Apr 2016 16:51:27 -0500 Subject: [PATCH 897/910] Also install examples directory --- var/spack/repos/builtin/packages/elk/package.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/var/spack/repos/builtin/packages/elk/package.py b/var/spack/repos/builtin/packages/elk/package.py index 28ebb89db38..780c41f726f 100644 --- a/var/spack/repos/builtin/packages/elk/package.py +++ b/var/spack/repos/builtin/packages/elk/package.py @@ -104,13 +104,18 @@ def configure(self, spec): def install(self, spec, prefix): + # Elk only provides an interactive setup script self.configure(spec) make() make('test') + # The Elk Makefile does not provide an install target mkdirp(prefix.bin) install('src/elk', prefix.bin) install('src/eos/eos', prefix.bin) install('src/spacegroup/spacegroup', prefix.bin) + + install_tree('examples', join_path(prefix, 'examples')) + From abe744c5a099fd988ff3fe5eb1d50cca7a633d74 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 12 Apr 2016 16:53:45 -0500 Subject: [PATCH 898/910] Add latest version of PnetCDF --- var/spack/repos/builtin/packages/parallel-netcdf/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/parallel-netcdf/package.py b/var/spack/repos/builtin/packages/parallel-netcdf/package.py index 62a8f7ca0ba..e6f8cf026bf 100644 --- a/var/spack/repos/builtin/packages/parallel-netcdf/package.py +++ b/var/spack/repos/builtin/packages/parallel-netcdf/package.py @@ -8,6 +8,7 @@ class ParallelNetcdf(Package): homepage = "https://trac.mcs.anl.gov/projects/parallel-netcdf" url = "http://cucis.ece.northwestern.edu/projects/PnetCDF/Release/parallel-netcdf-1.6.1.tar.gz" + version('1.7.0', '267eab7b6f9dc78c4d0e6def2def3aea4bc7c9f0') version('1.6.1', '62a094eb952f9d1e15f07d56e535052604f1ac34') depends_on("m4") From 04392babca88089de114864cbe5ad419f6fc212b Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Tue, 12 Apr 2016 15:53:29 -0700 Subject: [PATCH 899/910] add setuptools version 20.7.0 --- var/spack/repos/builtin/packages/py-setuptools/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-setuptools/package.py b/var/spack/repos/builtin/packages/py-setuptools/package.py index 9338a5e1578..2086d0b3fbd 100644 --- a/var/spack/repos/builtin/packages/py-setuptools/package.py +++ b/var/spack/repos/builtin/packages/py-setuptools/package.py @@ -11,6 +11,7 @@ class PySetuptools(Package): version('19.2', '78353b1f80375ca5e088f4b4627ffe03') version('20.5', 'fadc1e1123ddbe31006e5e43e927362b') version('20.6.7', '45d6110f3ec14924e44c33411db64fe6') + version('20.7.0', '5d12b39bf3e75e80fdce54e44b255615') extends('python') From 5717cb981e38979727a9340eb118e88166a9f7e3 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 13 Apr 2016 15:03:03 -0500 Subject: [PATCH 900/910] fortran2003 option has been removed from HDF5 --- var/spack/repos/builtin/packages/hdf5/package.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index f26e225b83e..cce609eb291 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -101,10 +101,10 @@ def install(self, spec, prefix): extra_args.append('--enable-cxx') if '+fortran' in spec: - extra_args.extend([ - '--enable-fortran', - '--enable-fortran2003' - ]) + extra_args.append('--enable-fortran') + # '--enable-fortran2003' no longer exists as of version 1.10.0 + if spec.satisfies('@:1.8.16'): + extra_args.append('--enable-fortran2003') if '+mpi' in spec: # The HDF5 configure script warns if cxx and mpi are enabled From 2cf9ebc62c18414ddafb44bdda2b2487f72a4ec5 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Wed, 13 Apr 2016 15:00:45 -0700 Subject: [PATCH 901/910] Fix compile issues with new libpng versions and qt3 builds by using an older version of libpng. In libpng, make older versions available. --- var/spack/repos/builtin/packages/libpng/package.py | 3 +++ var/spack/repos/builtin/packages/qt/package.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/libpng/package.py b/var/spack/repos/builtin/packages/libpng/package.py index 73c8c623414..9d5782896e9 100644 --- a/var/spack/repos/builtin/packages/libpng/package.py +++ b/var/spack/repos/builtin/packages/libpng/package.py @@ -8,6 +8,9 @@ class Libpng(Package): version('1.6.16', '1a4ad377919ab15b54f6cb6a3ae2622d') version('1.6.15', '829a256f3de9307731d4f52dc071916d') version('1.6.14', '2101b3de1d5f348925990f9aa8405660') + version('1.5.26', '3ca98347a5541a2dad55cd6d07ee60a9') + version('1.4.19', '89bcbc4fc8b31f4a403906cf4f662330') + version('1.2.56', '9508fc59d10a1ffadd9aae35116c19ee') depends_on('zlib') diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 8cb88e6c854..93688fb777f 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -29,7 +29,8 @@ class Qt(Package): depends_on("zlib") depends_on("dbus", when='@4:') depends_on("libtiff") - depends_on("libpng") + depends_on("libpng@1.2.56", when='@3') + depends_on("libpng", when='@4:') depends_on("libmng") depends_on("jpeg") @@ -120,6 +121,8 @@ def common_config_args(self): @when('@3') def configure(self): + # An user report that this was necessary to link Qt3 on ubuntu + os.environ['LD_LIBRARY_PATH'] = os.getcwd()+'/lib' configure('-prefix', self.prefix, '-v', '-thread', From f9aafeee1e6a33a76dc1904e97f117194915f4d0 Mon Sep 17 00:00:00 2001 From: Jim Galarowicz Date: Wed, 13 Apr 2016 21:58:57 -0700 Subject: [PATCH 902/910] Loosen the boost requirement from a specific version that was specified in my initial commit. --- var/spack/repos/builtin/packages/mrnet/package.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/mrnet/package.py b/var/spack/repos/builtin/packages/mrnet/package.py index b8cbb7e8f9b..a3abb71285d 100644 --- a/var/spack/repos/builtin/packages/mrnet/package.py +++ b/var/spack/repos/builtin/packages/mrnet/package.py @@ -15,13 +15,10 @@ class Mrnet(Package): variant('krellpatch', default=False, description="Build MRNet with krell openspeedshop based patch.") patch('krell-5.0.1.patch', when='@5.0.1+krellpatch') - - variant('lwthreads', default=False, description="Also build the MRNet LW threadsafe libraries") parallel = False - #depends_on("boost") - depends_on("boost@1.53.0") + depends_on("boost") def install(self, spec, prefix): # Build the MRNet LW thread safe libraries when the krelloptions variant is present From 01fab24c156e82f2886b464a2474fe3474d5312a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 13 Apr 2016 14:02:25 +0200 Subject: [PATCH 903/910] binutils depends on flex, bison, and m4 --- var/spack/repos/builtin/packages/binutils/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/repos/builtin/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py index 897539a4391..b8064093d27 100644 --- a/var/spack/repos/builtin/packages/binutils/package.py +++ b/var/spack/repos/builtin/packages/binutils/package.py @@ -12,6 +12,10 @@ class Binutils(Package): version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e') version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764') + depends_on('m4') + depends_on('flex') + depends_on('bison') + # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell variant('krellpatch', default=False, description="build with openspeedshop based patch.") variant('gold', default=True, description="build the gold linker") From cf1914fc00bd98a1885edc4d4190f9c424b0588b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 14 Apr 2016 10:18:06 -0500 Subject: [PATCH 904/910] Also install species directory --- var/spack/repos/builtin/packages/elk/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/elk/package.py b/var/spack/repos/builtin/packages/elk/package.py index 780c41f726f..1d9216fd1a9 100644 --- a/var/spack/repos/builtin/packages/elk/package.py +++ b/var/spack/repos/builtin/packages/elk/package.py @@ -118,4 +118,5 @@ def install(self, spec, prefix): install('src/spacegroup/spacegroup', prefix.bin) install_tree('examples', join_path(prefix, 'examples')) + install_tree('species', join_path(prefix, 'species')) From a25e33d96ecbd3db690cd8f0bc5119f506310dab Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 17 Apr 2016 00:50:15 -0400 Subject: [PATCH 905/910] Create lib directory --- var/spack/repos/builtin/packages/cereal/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/repos/builtin/packages/cereal/package.py b/var/spack/repos/builtin/packages/cereal/package.py index a83927456f1..6acbf666c81 100644 --- a/var/spack/repos/builtin/packages/cereal/package.py +++ b/var/spack/repos/builtin/packages/cereal/package.py @@ -1,4 +1,5 @@ from spack import * +import os import shutil class Cereal(Package): @@ -30,5 +31,8 @@ def install(self, spec, prefix): # Install shutil.rmtree(join_path(prefix, 'doc'), ignore_errors=True) shutil.rmtree(join_path(prefix, 'include'), ignore_errors=True) + shutil.rmtree(join_path(prefix, 'lib'), ignore_errors=True) shutil.copytree('doc', join_path(prefix, 'doc'), symlinks=True) shutil.copytree('include', join_path(prefix, 'include'), symlinks=True) + # Create empty directory to avoid linker warnings later + os.mkdir(join_path(prefix, 'lib')) From 523be30974c4630450e779a9814d685716dd312e Mon Sep 17 00:00:00 2001 From: Kelly Thompson Date: Sat, 16 Apr 2016 23:11:19 -0600 Subject: [PATCH 906/910] + Add CMake version 3.5.2. (#787) --- var/spack/repos/builtin/packages/cmake/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index 91a4e3b415f..2493cf0a130 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -30,6 +30,7 @@ class Cmake(Package): homepage = 'https://www.cmake.org' url = 'https://cmake.org/files/v3.4/cmake-3.4.3.tar.gz' + version('3.5.2', '701386a1b5ec95f8d1075ecf96383e02') version('3.5.1', 'ca051f4a66375c89d1a524e726da0296') version('3.5.0', '33c5d09d4c33d4ffcc63578a6ba8777e') version('3.4.3', '4cb3ff35b2472aae70f542116d616e63') From fb4f361cd0a8f12c606013b382d7e4fa5706dbfb Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 17 Apr 2016 01:12:52 -0400 Subject: [PATCH 907/910] Update to version 4.1.0 (#785) --- var/spack/repos/builtin/packages/jemalloc/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/repos/builtin/packages/jemalloc/package.py b/var/spack/repos/builtin/packages/jemalloc/package.py index 8cec9ea75ba..9cb0fd1f40a 100644 --- a/var/spack/repos/builtin/packages/jemalloc/package.py +++ b/var/spack/repos/builtin/packages/jemalloc/package.py @@ -5,6 +5,7 @@ class Jemalloc(Package): homepage = "http://www.canonware.com/jemalloc/" url = "https://github.com/jemalloc/jemalloc/releases/download/4.0.4/jemalloc-4.0.4.tar.bz2" + version('4.1.0', 'c4e53c947905a533d5899e5cc3da1f94') version('4.0.4', '687c5cc53b9a7ab711ccd680351ff988') variant('stats', default=False, description='Enable heap statistics') @@ -20,5 +21,8 @@ def install(self, spec, prefix): configure(*configure_args) + # Don't use -Werror + filter_file(r'-Werror=\S*', '', 'Makefile') + make() make("install") From 375de085e271f2926ef2381a2e22f40bd5541177 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 17 Apr 2016 01:14:34 -0400 Subject: [PATCH 908/910] Make PAPI build on Darwin (#772) * Make PAPI build on Darwin - don't use - run install_name_tool * Use fix_darwin_install_name --- .../repos/builtin/packages/papi/package.py | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/papi/package.py b/var/spack/repos/builtin/packages/papi/package.py index 53d69e28d90..74b3ea9ef9e 100644 --- a/var/spack/repos/builtin/packages/papi/package.py +++ b/var/spack/repos/builtin/packages/papi/package.py @@ -1,5 +1,8 @@ from spack import * +import glob import os +import sys +from llnl.util.filesystem import fix_darwin_install_name class Papi(Package): """PAPI provides the tool designer and application engineer with a @@ -18,17 +21,27 @@ class Papi(Package): version('5.3.0', '367961dd0ab426e5ae367c2713924ffb') def install(self, spec, prefix): - os.chdir("src/") + with working_dir("src"): - configure_args=["--prefix=%s" % prefix] + configure_args=["--prefix=%s" % prefix] - # PAPI uses MPI if MPI is present; since we don't require an - # MPI package, we ensure that all attempts to use MPI fail, so - # that PAPI does not get confused - configure_args.append('MPICC=:') + # PAPI uses MPI if MPI is present; since we don't require + # an MPI package, we ensure that all attempts to use MPI + # fail, so that PAPI does not get confused + configure_args.append('MPICC=:') - configure(*configure_args) + configure(*configure_args) - make() - make("install") + # Don't use + for level in [".", "*", "*/*"]: + files = glob.iglob(join_path(level, "*.[ch]")) + filter_file(r"\", "", *files) + make() + make("install") + + # The shared library is not installed correctly on Darwin + if sys.platform == 'darwin': + os.rename(join_path(prefix.lib, 'libpapi.so'), + join_path(prefix.lib, 'libpapi.dylib')) + fix_darwin_install_name(prefix.lib) From 63bade7a0c7350681dd29ce877f2a9fad679d67d Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Sun, 17 Apr 2016 01:15:07 -0400 Subject: [PATCH 909/910] metis: fix OS X install (#764) Without this, the binaries use relative paths for loading which causes them to not be found when running binaries. Not sure why the existing hack wasn't working, but this fixes it the proper way. --- var/spack/repos/builtin/packages/metis/package.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py index 41e3ebb429f..b05f23a3dc2 100644 --- a/var/spack/repos/builtin/packages/metis/package.py +++ b/var/spack/repos/builtin/packages/metis/package.py @@ -136,6 +136,7 @@ def install(self, spec, prefix): source_directory = self.stage.source_path options.append('-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=source_directory)) + options.append('-DCMAKE_INSTALL_NAME_DIR:PATH=%s/lib' % prefix) if '+shared' in spec: options.append('-DSHARED:BOOL=ON') @@ -184,7 +185,3 @@ def install(self, spec, prefix): fs = glob.glob(join_path(source_directory,'GKlib',"*.h")) for f in fs: install(f, GKlib_dist) - - # The shared library is not installed correctly on Darwin; correct this - if (sys.platform == 'darwin') and ('+shared' in spec): - fix_darwin_install_name(prefix.lib) From dd84a575807636159d80809f59962cf799b83fd7 Mon Sep 17 00:00:00 2001 From: Kelly Thompson Date: Sat, 16 Apr 2016 23:15:48 -0600 Subject: [PATCH 910/910] + On some systems, Dia also requires intltool. (#768) --- .../repos/builtin/packages/dia/package.py | 1 + .../builtin/packages/intltool/package.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 var/spack/repos/builtin/packages/intltool/package.py diff --git a/var/spack/repos/builtin/packages/dia/package.py b/var/spack/repos/builtin/packages/dia/package.py index 1cb5910e46d..25d5f08205e 100644 --- a/var/spack/repos/builtin/packages/dia/package.py +++ b/var/spack/repos/builtin/packages/dia/package.py @@ -7,6 +7,7 @@ class Dia(Package): version('0.97.3', '0e744a0f6a6c4cb6a089e4d955392c3c') + depends_on('intltool') depends_on('gtkplus@2.6.0:') depends_on('cairo') #depends_on('libart') # optional dependency, not yet supported by spack. diff --git a/var/spack/repos/builtin/packages/intltool/package.py b/var/spack/repos/builtin/packages/intltool/package.py new file mode 100644 index 00000000000..9b3c095378e --- /dev/null +++ b/var/spack/repos/builtin/packages/intltool/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Intltool(Package): + """intltool is a set of tools to centralize translation of many different file formats using GNU gettext-compatible PO files.""" + homepage = 'https://freedesktop.org/wiki/Software/intltool/' + + version('0.51.0', '12e517cac2b57a0121cda351570f1e63') + + def url_for_version(self, version): + """Handle version-based custom URLs.""" + return 'https://launchpad.net/intltool/trunk/%s/+download/intltool-%s.tar.gz' % (version, version) + + def install(self, spec, prefix): + + # configure, build, install: + options = ['--prefix=%s' % prefix ] + configure(*options) + make() + make('install')