From 0a191d6b09e86d5b6eb66e97dde3cad197708f29 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 21 Dec 2015 15:02:48 +0100 Subject: [PATCH 01/23] scalasca : updated package build (depends on scorep, does not build twice). --- var/spack/packages/scalasca/package.py | 96 +++++++++++++------------- 1 file changed, 47 insertions(+), 49 deletions(-) diff --git a/var/spack/packages/scalasca/package.py b/var/spack/packages/scalasca/package.py index cf7a40c1f53..595b7ad0b85 100644 --- a/var/spack/packages/scalasca/package.py +++ b/var/spack/packages/scalasca/package.py @@ -1,65 +1,63 @@ -# FIXME: Add copyright +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## from spack import * + class Scalasca(Package): - """Scalasca is a software tool that supports the performance optimization - of parallel programs by measuring and analyzing their runtime behavior. - The analysis identifies potential performance bottlenecks - in - particular those concerning communication and synchronization - and - offers guidance in exploring their causes.""" + """ + Scalasca is a software tool that supports the performance optimization of parallel programs by measuring and + analyzing their runtime behavior. The analysis identifies potential performance bottlenecks - in particular those + concerning communication and synchronization - and offers guidance in exploring their causes. + """ - # FIXME: add a proper url for your package's homepage here. homepage = "http://www.scalasca.org" - url = "http://apps.fz-juelich.de/scalasca/releases/scalasca/2.1/dist/scalasca-2.1.tar.gz" + url = "http://apps.fz-juelich.de/scalasca/releases/scalasca/2.1/dist/scalasca-2.1.tar.gz" - version('2.1', 'bab9c2b021e51e2ba187feec442b96e6', - url = 'http://apps.fz-juelich.de/scalasca/releases/scalasca/2.1/dist/scalasca-2.1.tar.gz' ) + version('2.2.2', '2bafce988b0522d18072f7771e491ab9', + url='http://apps.fz-juelich.de/scalasca/releases/scalasca/2.2/dist/scalasca-2.2.2.tar.gz') + + version('2.1', 'bab9c2b021e51e2ba187feec442b96e6', + url='http://apps.fz-juelich.de/scalasca/releases/scalasca/2.1/dist/scalasca-2.1.tar.gz') depends_on("mpi") - depends_on("otf2@1.4") - depends_on("cube@4.2.3") - - backend_user_provided = """\ -CC=cc -CXX=c++ -F77=f77 -FC=f90 -CFLAGS=-fPIC -CXXFLAGS=-fPIC -""" - frontend_user_provided = """\ -CC_FOR_BUILD=cc -CXX_FOR_BUILD=c++ -F77_FOR_BUILD=f70 -FC_FOR_BUILD=f90 -CFLAGS_FOR_BUILD=-fPIC -CXXFLAGS_FOR_BUILD=-fPIC -""" - mpi_user_provided = """\ -MPICC=mpicc -MPICXX=mpicxx -MPIF77=mpif77 -MPIFC=mpif90 -MPI_CFLAGS=-fPIC -MPI_CXXFLAGS=-fPIC -""" + ########## + # Fix dependencies for Scalasca according to what stated in the release page + # The OTF2 library path should be detected automatically from SCOREP + # SCALASCA 2.2.2 + depends_on("scorep@1.4:", when='@2.2.2') + depends_on("cube@4.3:", when='@2.2.2') + # SCALASCA 2.1 + depends_on("scorep@1.3", when='@2.1') + depends_on("cube@4.2:", when='@2.1') + ########## def install(self, spec, prefix): configure_args = ["--prefix=%s" % prefix, - "--with-custom-compilers", - "--with-otf2=%s" % spec['otf2'].prefix.bin, "--with-cube=%s" % spec['cube'].prefix.bin, "--enable-shared"] - configure(*configure_args) - make() - make("install") - - # FIXME: Modify the configure line to suit your build system here. - configure("--prefix=%s" % prefix) - - # FIXME: Add logic to build and install here - make() - make("install") + make("install") \ No newline at end of file From a411c497abe3dbc0e98f4278a2d5908058802e38 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 21 Dec 2015 16:10:55 +0100 Subject: [PATCH 02/23] scalasca + dependencies : simplified build after #255 --- var/spack/packages/cube/package.py | 77 +++++++++++------------ var/spack/packages/opari2/package.py | 81 ++++++++++-------------- var/spack/packages/otf2/package.py | 85 ++++++++++---------------- var/spack/packages/scalasca/package.py | 2 +- var/spack/packages/scorep/package.py | 20 ++---- 5 files changed, 106 insertions(+), 159 deletions(-) diff --git a/var/spack/packages/cube/package.py b/var/spack/packages/cube/package.py index cc1c684594f..17d388c33d4 100644 --- a/var/spack/packages/cube/package.py +++ b/var/spack/packages/cube/package.py @@ -1,58 +1,55 @@ -# FIXME: Add copyright statement +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. # +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + from spack import * -from contextlib import closing + class Cube(Package): - """Cube the profile viewer for Score-P and Scalasca profiles. It - displays a multi-dimensional performance space consisting - of the dimensions (i) performance metric, (ii) call path, - and (iii) system resource.""" + """ + Cube the profile viewer for Score-P and Scalasca profiles. It displays a multi-dimensional performance space + consisting of the dimensions: + - performance metric + - call path + - system resource + """ homepage = "http://www.scalasca.org/software/cube-4.x/download.html" - url = "http://apps.fz-juelich.de/scalasca/releases/cube/4.2/dist/cube-4.2.3.tar.gz" + url = "http://apps.fz-juelich.de/scalasca/releases/cube/4.2/dist/cube-4.2.3.tar.gz" version('4.3.3', '07e109248ed8ffc7bdcce614264a2909', url='http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3.3.tar.gz') - version('4.2.3', '8f95b9531f5a8f8134f279c2767c9b20') + version('4.2.3', '8f95b9531f5a8f8134f279c2767c9b20', + url="http://apps.fz-juelich.de/scalasca/releases/cube/4.2/dist/cube-4.2.3.tar.gz") - version('4.3TP1', 'a2090fbc7b2ba394bd5c09ba971e237f', - url = 'http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz') - - # Using CC as C++ compiler provides quirky workaround for a Score-P build system attempt - # to guess a matching C compiler when configuring scorep-score - backend_user_provided = """\ -CC=cc -CXX=CC -F77=f77 -FC=f90 -#CFLAGS=-fPIC -#CXXFLAGS=-fPIC -""" - frontend_user_provided = """\ -CC_FOR_BUILD=cc -CXX_FOR_BUILD=CC -F77_FOR_BUILD=f70 -FC_FOR_BUILD=f90 -""" + # TODO : add variant that builds GUI on top of Qt def install(self, spec, prefix): - # Use a custom compiler configuration, otherwise the score-p - # build system messes with spack's compiler settings. - # Create these three files in the build directory - - with closing(open("vendor/common/build-config/platforms/platform-backend-user-provided", "w")) as backend_file: - backend_file.write(self.backend_user_provided) - with closing(open("vendor/common/build-config/platforms/platform-frontend-user-provided", "w")) as frontend_file: - frontend_file.write(self.frontend_user_provided) - configure_args = ["--prefix=%s" % prefix, - "--with-custom-compilers", - "--without-paraver", + "--without-paraver", "--without-gui"] - configure(*configure_args) - make(parallel=False) make("install", parallel=False) diff --git a/var/spack/packages/opari2/package.py b/var/spack/packages/opari2/package.py index 3f8c65377df..c68978f5c0a 100644 --- a/var/spack/packages/opari2/package.py +++ b/var/spack/packages/opari2/package.py @@ -1,18 +1,38 @@ -# FIXME: Add copyright statement here +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## from spack import * -from contextlib import closing class Opari2(Package): - """OPARI2 is a source-to-source instrumentation tool for OpenMP and - hybrid codes. It surrounds OpenMP directives and runtime library - calls with calls to the POMP2 measurement interface. - OPARI2 will provide you with a new initialization method that allows - for multi-directory and parallel builds as well as the usage of - pre-instrumented libraries. Furthermore, an efficient way of - tracking parent-child relationships was added. Additionally, we - extended OPARI2 to support instrumentation of OpenMP 3.0 - tied tasks. """ + """ + OPARI2 is a source-to-source instrumentation tool for OpenMP and hybrid codes. It surrounds OpenMP directives and + runtime library calls with calls to the POMP2 measurement interface. OPARI2 will provide you with a new + initialization method that allows for multi-directory and parallel builds as well as the usage of pre-instrumented + libraries. Furthermore, an efficient way of tracking parent-child relationships was added. Additionally, we extended + OPARI2 to support instrumentation of OpenMP 3.0 tied tasks. + """ homepage = "http://www.vi-hps.org/projects/score-p" url = "http://www.vi-hps.org/upload/packages/opari2/opari2-1.1.2.tar.gz" @@ -21,47 +41,8 @@ class Opari2(Package): url='http://www.vi-hps.org/upload/packages/opari2/opari2-1.1.4.tar.gz') version('1.1.2', '9a262c7ca05ff0ab5f7775ae96f3539e') - backend_user_provided = """\ -CC=cc -CXX=c++ -F77=f77 -FC=f90 -CFLAGS=-fPIC -CXXFLAGS=-fPIC -""" - frontend_user_provided = """\ -CC_FOR_BUILD=cc -CXX_FOR_BUILD=c++ -F77_FOR_BUILD=f70 -FC_FOR_BUILD=f90 -CFLAGS_FOR_BUILD=-fPIC -CXXFLAGS_FOR_BUILD=-fPIC -""" - mpi_user_provided = """\ -MPICC=mpicc -MPICXX=mpicxx -MPIF77=mpif77 -MPIFC=mpif90 -MPI_CFLAGS=-fPIC -MPI_CXXFLAGS=-fPIC -""" - def install(self, spec, prefix): - # Use a custom compiler configuration, otherwise the score-p - # build system messes with spack's compiler settings. - # Create these three files in the build directory - with closing(open("platform-backend-user-provided", "w")) as backend_file: - backend_file.write(self.backend_user_provided) - with closing(open("platform-frontend-user-provided", "w")) as frontend_file: - frontend_file.write(self.frontend_user_provided) - with closing(open("platform-mpi-user-provided", "w")) as mpi_file: - mpi_file.write(self.mpi_user_provided) - - # FIXME: Modify the configure line to suit your build system here. configure("--prefix=%s" % prefix, - "--with-custom-compilers", "--enable-shared") - - # FIXME: Add logic to build and install here make() make("install") diff --git a/var/spack/packages/otf2/package.py b/var/spack/packages/otf2/package.py index b3d3a5b6633..c3d61bc2283 100644 --- a/var/spack/packages/otf2/package.py +++ b/var/spack/packages/otf2/package.py @@ -1,12 +1,35 @@ -# FIXME: Add copyright +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## from spack import * -from contextlib import closing -import os + class Otf2(Package): - """The Open Trace Format 2 is a highly scalable, memory efficient event - trace data format plus support library.""" + """ + The Open Trace Format 2 is a highly scalable, memory efficient event trace data format plus support library. + """ homepage = "http://www.vi-hps.org/score-p" url = "http://www.vi-hps.org/upload/packages/otf2/otf2-1.4.tar.gz" @@ -22,57 +45,11 @@ class Otf2(Package): version('1.2.1', '8fb3e11fb7489896596ae2c7c83d7fc8', url="http://www.vi-hps.org/upload/packages/otf2/otf2-1.2.1.tar.gz") - backend_user_provided = """\ -CC=cc -CXX=c++ -F77=f77 -FC=f90 -CFLAGS=-fPIC -CXXFLAGS=-fPIC -""" - frontend_user_provided = """\ -CC_FOR_BUILD=cc -CXX_FOR_BUILD=c++ -F77_FOR_BUILD=f70 -FC_FOR_BUILD=f90 -CFLAGS_FOR_BUILD=-fPIC -CXXFLAGS_FOR_BUILD=-fPIC -""" - mpi_user_provided = """\ -MPICC=cc -MPICXX=c++ -MPIF77=f77 -MPIFC=f90 -MPI_CFLAGS=-fPIC -MPI_CXXFLAGS=-fPIC -""" - - @when('@:1.2.1') - def version_specific_args(self): - return ["--with-platform=disabled", "CC=cc", "CXX=c++", "F77=f77", "F90=f90", "CFLAGS=-fPIC", "CXXFLAGS=-fPIC"] - - @when('@1.3:') - def version_specific_args(self): - # TODO: figure out what scorep's build does as of otf2 1.3 - return ["--with-custom-compilers"] - def install(self, spec, prefix): - # Use a custom compiler configuration, otherwise the score-p - # build system messes with spack's compiler settings. - # Create these three files in the build directory - with closing(open("platform-backend-user-provided", "w")) as backend_file: - backend_file.write(self.backend_user_provided) - with closing(open("platform-frontend-user-provided", "w")) as frontend_file: - frontend_file.write(self.frontend_user_provided) - with closing(open("platform-mpi-user-provided", "w")) as mpi_file: - mpi_file.write(self.mpi_user_provided) - configure_args=["--prefix=%s" % prefix, - "--enable-shared"] - - configure_args.extend(self.version_specific_args()) - + "--enable-shared", + "CFLAGS=-fPIC", + "CXXFLAGS=-fPIC"] configure(*configure_args) - make() make("install") diff --git a/var/spack/packages/scalasca/package.py b/var/spack/packages/scalasca/package.py index 595b7ad0b85..6de14564b29 100644 --- a/var/spack/packages/scalasca/package.py +++ b/var/spack/packages/scalasca/package.py @@ -44,7 +44,7 @@ class Scalasca(Package): depends_on("mpi") ########## - # Fix dependencies for Scalasca according to what stated in the release page + # Hard-code dependencies for Scalasca according to what stated in the release page # The OTF2 library path should be detected automatically from SCOREP # SCALASCA 2.2.2 depends_on("scorep@1.4:", when='@2.2.2') diff --git a/var/spack/packages/scorep/package.py b/var/spack/packages/scorep/package.py index 0820f2d8ac3..5127e814b68 100644 --- a/var/spack/packages/scorep/package.py +++ b/var/spack/packages/scorep/package.py @@ -28,9 +28,9 @@ class Scorep(Package): """ - The Score-P measurement infrastructure is a highly scalable and - easy-to-use tool suite for profiling, event tracing, and online - analysis of HPC applications.""" + The Score-P measurement infrastructure is a highly scalable and easy-to-use tool suite for profiling, event + tracing, and online analysis of HPC applications. + """ homepage = "http://www.vi-hps.org/projects/score-p" url = "http://www.vi-hps.org/upload/packages/scorep/scorep-1.2.3.tar.gz" @@ -55,15 +55,6 @@ class Scorep(Package): depends_on("mpi") depends_on("papi") - def get_compiler_config_line(self): - backend_user_provided = ['CC=%s' % self.compiler.cc_names[0], - 'CXX=%s' % self.compiler.cxx_names[0], - 'F77=%s' % self.compiler.f77_names[0] if len(self.compiler.f77_names) else "", - 'FC=%s' % self.compiler.fc_names[0] if len(self.compiler.fc_names) else "", - 'CFLAGS=-fPIC %s' % self.rpath_args, - 'CXXFLAGS=-fPIC %s'% self.rpath_args] - return backend_user_provided - def install(self, spec, prefix): configure = Executable( join_path(self.stage.source_path, 'configure') ) with working_dir('spack-build', create=True): @@ -73,8 +64,9 @@ def install(self, spec, prefix): "--with-cube=%s" % spec['cube'].prefix.bin, "--with-papi-header=%s" % spec['papi'].prefix.include, "--with-papi-lib=%s" % spec['papi'].prefix.lib, - "--enable-shared"] - configure_args.extend(self.get_compiler_config_line()) + "--enable-shared", + "CFLAGS=-fPIC", + "CXXFLAGS=-fPIC"] configure(*configure_args) make() make("install") From 6dc1fc03c13027f3fd1553af4d23ed00ea929985 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 21 Dec 2015 10:34:41 -0500 Subject: [PATCH 03/23] Update hwloc; don't require libpciaccess on OS X - hwloc 1.11.2 is available. - libpciaccess is not supported on OS X; don't require it there. --- var/spack/packages/hwloc/package.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/hwloc/package.py b/var/spack/packages/hwloc/package.py index 452a7d7ce3b..35a526ceb87 100644 --- a/var/spack/packages/hwloc/package.py +++ b/var/spack/packages/hwloc/package.py @@ -15,15 +15,17 @@ class Hwloc(Package): homepage = "http://www.open-mpi.org/projects/hwloc/" url = "http://www.open-mpi.org/software/hwloc/v1.9/downloads/hwloc-1.9.tar.gz" + version('1.11.2', '486169cbe111cdea57be12638828ebbf', + url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.2.tar.bz2') version('1.11.1', '002742efd3a8431f98d6315365a2b543', url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.1.tar.bz2') version('1.9', '1f9f9155682fe8946a97c08896109508') - depends_on('libpciaccess') + # libpciaccess is not supported on OS X + depends_on('libpciaccess', when='=linux') def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() make("install") - From 6831ee6f0a98bfdc2c0847d0a0dea6c91fcb1d9b Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 22 Dec 2015 16:24:05 -0500 Subject: [PATCH 04/23] Skip installing libpciaccess on Darwin --- var/spack/packages/hwloc/package.py | 3 +-- var/spack/packages/libpciaccess/package.py | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/hwloc/package.py b/var/spack/packages/hwloc/package.py index 35a526ceb87..7ebede76a3b 100644 --- a/var/spack/packages/hwloc/package.py +++ b/var/spack/packages/hwloc/package.py @@ -21,8 +21,7 @@ class Hwloc(Package): url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.1.tar.bz2') version('1.9', '1f9f9155682fe8946a97c08896109508') - # libpciaccess is not supported on OS X - depends_on('libpciaccess', when='=linux') + depends_on('libpciaccess') def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/packages/libpciaccess/package.py b/var/spack/packages/libpciaccess/package.py index 6022fc34a30..403bafbbd2c 100644 --- a/var/spack/packages/libpciaccess/package.py +++ b/var/spack/packages/libpciaccess/package.py @@ -1,4 +1,5 @@ from spack import * +import os.path class Libpciaccess(Package): """Generic PCI access library.""" @@ -13,6 +14,12 @@ class Libpciaccess(Package): depends_on('libtool') def install(self, spec, prefix): + # libpciaccess does not support OS X + if spec.satisfies('=darwin-x86_64'): + # create a dummy directory + mkdir(prefix.lib) + return + from subprocess import call call(["./autogen.sh"]) configure("--prefix=%s" % prefix) From 552d8ca5f083daa467cf3a852fec76be21eec8ee Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 22 Dec 2015 16:40:09 -0500 Subject: [PATCH 05/23] Update rsync to 3.1.2 --- var/spack/packages/rsync/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/packages/rsync/package.py b/var/spack/packages/rsync/package.py index 8ae21b1cb9b..76aec3096dd 100644 --- a/var/spack/packages/rsync/package.py +++ b/var/spack/packages/rsync/package.py @@ -5,6 +5,7 @@ class Rsync(Package): homepage = "https://rsync.samba.org" url = "https://download.samba.org/pub/rsync/rsync-3.1.1.tar.gz" + version('3.1.2', '0f758d7e000c0f7f7d3792610fad70cb') version('3.1.1', '43bd6676f0b404326eee2d63be3cdcfe') def install(self, spec, prefix): From 60ec79eac3f185f6389166ff801a767c66c7470e Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 22 Dec 2015 18:53:43 -0500 Subject: [PATCH 06/23] New package PCRE2 This is a major update of the package PCRE. It seems both PCRE and PCRE2 can be installed at the same time; their APIs are different. --- var/spack/packages/pcre2/package.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 var/spack/packages/pcre2/package.py diff --git a/var/spack/packages/pcre2/package.py b/var/spack/packages/pcre2/package.py new file mode 100644 index 00000000000..6a0244a15e9 --- /dev/null +++ b/var/spack/packages/pcre2/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Pcre2(Package): + """The PCRE2 package contains Perl Compatible Regular Expression + libraries. These are useful for implementing regular expression + pattern matching using the same syntax and semantics as Perl 5.""" + homepage = "http://www.pcre.org""" + url = "ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre2-10.20.tar.bz2" + + version('10.20', 'dcd027c57ecfdc8a6c3af9d0acf5e3f7') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") From 2b89d9b1db620c3b4a0e391ffc6621717aa03382 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 22 Dec 2015 16:54:41 -0800 Subject: [PATCH 07/23] More consistent URL parsing when finding versions. Versions found by wildcard URLs are different from versions found by parse_version, etc. The wildcards are constructed more haphazardly than the very specific URL patterns in url.py, so they can get things wrong. e.g., for this URL: https://software.lanl.gov/MeshTools/trac/attachment/wiki/WikiStart/mstk-2.25rc1.tgz We miss the 'rc' and only return 2.25r as the version if we ONLY use URL wildcards. Future: Maybe use the regexes from url.py to scrape web pages, and then compare them for similarity with the original URL, instead of trying to make a structured wildcard URL pattern? This might yield better results. --- lib/spack/spack/package.py | 12 +++++++++--- lib/spack/spack/url.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b95afb073df..2e7678b4b03 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1200,6 +1200,8 @@ def find_versions_of_archive(*archive_urls, **kwargs): for aurl in archive_urls: list_urls.add(spack.url.find_list_url(aurl)) + print list_urls + # Grab some web pages to scrape. page_map = {} for lurl in list_urls: @@ -1224,9 +1226,13 @@ def find_versions_of_archive(*archive_urls, **kwargs): for page_url, content in page_map.iteritems(): # extract versions from matches. for regex in regexes: - versions.update( - (Version(m.group(1)), urljoin(page_url, m.group(0))) - for m in re.finditer(regex, content)) + print regex + print + + for m in re.finditer(regex, content): + url = urljoin(page_url, m.group(0)) + ver = spack.url.parse_version(url) + versions[ver] = url return versions diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 02c0b83e260..ed5a6d2a44f 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -210,7 +210,7 @@ def parse_version_offset(path): (r'-((\d+\.)*\d+)$', stem), # e.g. foobar-4.5.1b, foobar4.5RC, foobar.v4.5.1b - (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP?)\d*)$', stem), + (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP)?\d*)$', stem), # e.g. foobar-4.5.0-beta1, or foobar-4.50-beta (r'-((\d+\.)*\d+-beta(\d+)?)$', stem), From 0d4b77a24b883d15ee2c297a983b8a8dc807de6f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 22 Dec 2015 17:02:14 -0800 Subject: [PATCH 08/23] revert accidental print statements. --- lib/spack/spack/package.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 2e7678b4b03..6673e4f3929 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1200,8 +1200,6 @@ def find_versions_of_archive(*archive_urls, **kwargs): for aurl in archive_urls: list_urls.add(spack.url.find_list_url(aurl)) - print list_urls - # Grab some web pages to scrape. page_map = {} for lurl in list_urls: @@ -1226,9 +1224,6 @@ def find_versions_of_archive(*archive_urls, **kwargs): for page_url, content in page_map.iteritems(): # extract versions from matches. for regex in regexes: - print regex - print - for m in re.finditer(regex, content): url = urljoin(page_url, m.group(0)) ver = spack.url.parse_version(url) From d63cb8b537a9fc12a2a1ee5b22f2b20e19d90dc1 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 22 Dec 2015 17:05:23 -0800 Subject: [PATCH 09/23] Fix bug in URL regex. --- lib/spack/spack/url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index ed5a6d2a44f..02c0b83e260 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -210,7 +210,7 @@ def parse_version_offset(path): (r'-((\d+\.)*\d+)$', stem), # e.g. foobar-4.5.1b, foobar4.5RC, foobar.v4.5.1b - (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP)?\d*)$', stem), + (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP?)\d*)$', stem), # e.g. foobar-4.5.0-beta1, or foobar-4.50-beta (r'-((\d+\.)*\d+-beta(\d+)?)$', stem), From 5ca5884ad6285fd766a2f704bceb40b1cf63750f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 23 Dec 2015 13:13:07 -0800 Subject: [PATCH 10/23] Fix #235: Smarter web spidering -- use parsed links instead of reconstructing. - Move `find_versions_of_archive` from spack.package to `spack.util.web`. - `spider` funciton now just uses the link parsing it already does to return links. We evaluate actual links found in the scraped pages instead of trying to reconstruct them naively. - Add `spack url-parse` command, which you can use to show how Spack interprets the name and version in a URL. --- lib/spack/spack/cmd/create.py | 4 +- lib/spack/spack/cmd/url-parse.py | 75 ++++++++++++++++ lib/spack/spack/package.py | 46 +--------- lib/spack/spack/util/web.py | 148 +++++++++++++++++++++++-------- 4 files changed, 187 insertions(+), 86 deletions(-) create mode 100644 lib/spack/spack/cmd/url-parse.py diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 5e42860f3e7..9ecb709110c 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -34,8 +34,8 @@ import spack import spack.cmd import spack.cmd.checksum -import spack.package import spack.url +import spack.util.web from spack.util.naming import * import spack.util.crypto as crypto @@ -166,7 +166,7 @@ def create(parser, args): tty.msg("This looks like a URL for %s version %s." % (name, version)) tty.msg("Creating template for package %s" % name) - versions = spack.package.find_versions_of_archive(url) + versions = spack.util.web.find_versions_of_archive(url) rkeys = sorted(versions.keys(), reverse=True) versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys))) diff --git a/lib/spack/spack/cmd/url-parse.py b/lib/spack/spack/cmd/url-parse.py new file mode 100644 index 00000000000..077c793d2ee --- /dev/null +++ b/lib/spack/spack/cmd/url-parse.py @@ -0,0 +1,75 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import sys + +import llnl.util.tty as tty + +import spack +import spack.url +from spack.util.web import find_versions_of_archive + +description = "Show parsing of a URL, optionally spider web for other versions." + +def setup_parser(subparser): + subparser.add_argument('url', help="url of a package archive") + subparser.add_argument( + '-s', '--spider', action='store_true', help="Spider the source page for versions.") + + +def print_name_and_version(url): + name, ns, nl, ntup, ver, vs, vl, vtup = spack.url.substitution_offsets(url) + underlines = [" "] * max(ns+nl, vs+vl) + for i in range(ns, ns+nl): + underlines[i] = '-' + for i in range(vs, vs+vl): + underlines[i] = '~' + + print " %s" % url + print " %s" % ''.join(underlines) + + +def url_parse(parser, args): + url = args.url + + ver, vs, vl = spack.url.parse_version_offset(url) + name, ns, nl = spack.url.parse_name_offset(url, ver) + + tty.msg("Parsing URL:") + try: + print_name_and_version(url) + except spack.url.UrlParseError as e: + tty.error(str(e)) + + print + tty.msg("Substituting version 9.9.9b:") + newurl = spack.url.substitute_version(url, '9.9.9b') + print_name_and_version(newurl) + + if args.spider: + print + tty.msg("Spidering for versions:") + versions = find_versions_of_archive(url) + for v in sorted(versions): + print "%-20s%s" % (v, versions[v]) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 6673e4f3929..b44554e4183 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1164,7 +1164,7 @@ def fetch_remote_versions(self): raise VersionFetchError(self.__class__) try: - return find_versions_of_archive( + return spack.util.web.find_versions_of_archive( *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) except spack.error.NoNetworkConnectionError, e: tty.die("Package.fetch_versions couldn't connect to:", @@ -1188,50 +1188,6 @@ def rpath_args(self): return " ".join("-Wl,-rpath=%s" % p for p in self.rpath) -def find_versions_of_archive(*archive_urls, **kwargs): - list_url = kwargs.get('list_url', None) - list_depth = kwargs.get('list_depth', 1) - - # Generate a list of list_urls based on archive urls and any - # explicitly listed list_url in the package - list_urls = set() - if list_url: - list_urls.add(list_url) - for aurl in archive_urls: - list_urls.add(spack.url.find_list_url(aurl)) - - # Grab some web pages to scrape. - page_map = {} - for lurl in list_urls: - pages = spack.util.web.get_pages(lurl, depth=list_depth) - page_map.update(pages) - - # Scrape them for archive URLs - regexes = [] - for aurl in archive_urls: - # This creates a regex from the URL with a capture group for - # the version part of the URL. The capture group is converted - # to a generic wildcard, so we can use this to extract things - # on a page that look like archive URLs. - url_regex = spack.url.wildcard_version(aurl) - - # We'll be a bit more liberal and just look for the archive - # part, not the full path. - regexes.append(os.path.basename(url_regex)) - - # Build a version list from all the matches we find - versions = {} - for page_url, content in page_map.iteritems(): - # extract versions from matches. - for regex in regexes: - for m in re.finditer(regex, content): - url = urljoin(page_url, m.group(0)) - ver = spack.url.parse_version(url) - versions[ver] = url - - return versions - - def validate_package_url(url_string): """Determine whether spack can handle a particular URL or not.""" url = urlparse(url_string) diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 94384e9c86f..e26daef2964 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import re +import os import sys import subprocess import urllib2, cookielib @@ -70,7 +71,9 @@ def _spider(args): """ url, visited, root, opener, depth, max_depth, raise_on_error = args - pages = {} + pages = {} # dict from page URL -> text content. + links = set() # set of all links seen on visited pages. + try: # Make a HEAD request first to check the content type. This lets # us ignore tarballs and gigantic files. @@ -99,42 +102,45 @@ def _spider(args): page = response.read() pages[response_url] = page - # If we're not at max depth, parse out the links in the page + # Parse out the links in the page + link_parser = LinkParser() + subcalls = [] + link_parser.feed(page) + + while link_parser.links: + raw_link = link_parser.links.pop() + abs_link = urlparse.urljoin(response_url, raw_link) + + links.add(abs_link) + + # Skip stuff that looks like an archive + if any(raw_link.endswith(suf) for suf in ALLOWED_ARCHIVE_TYPES): + continue + + # Skip things outside the root directory + if not abs_link.startswith(root): + continue + + # Skip already-visited links + if abs_link in visited: + continue + + # If we're not at max depth, follow links. if depth < max_depth: - link_parser = LinkParser() - subcalls = [] - link_parser.feed(page) + subcalls.append((abs_link, visited, root, None, + depth+1, max_depth, raise_on_error)) + visited.add(abs_link) - while link_parser.links: - raw_link = link_parser.links.pop() - - # Skip stuff that looks like an archive - if any(raw_link.endswith(suf) for suf in ALLOWED_ARCHIVE_TYPES): - continue - - # Evaluate the link relative to the page it came from. - abs_link = urlparse.urljoin(response_url, raw_link) - - # Skip things outside the root directory - if not abs_link.startswith(root): - continue - - # Skip already-visited links - if abs_link in visited: - continue - - subcalls.append((abs_link, visited, root, None, depth+1, max_depth, raise_on_error)) - visited.add(abs_link) - - if subcalls: - try: - pool = Pool(processes=len(subcalls)) - dicts = pool.map(_spider, subcalls) - for d in dicts: - pages.update(d) - finally: - pool.terminate() - pool.join() + if subcalls: + try: + pool = Pool(processes=len(subcalls)) + results = pool.map(_spider, subcalls) + for sub_pages, sub_links in results: + pages.update(sub_pages) + links.update(sub_links) + finally: + pool.terminate() + pool.join() except urllib2.URLError, e: tty.debug(e) @@ -155,10 +161,10 @@ def _spider(args): # Other types of errors are completely ignored, except in debug mode. tty.debug("Error in _spider: %s" % e) - return pages + return pages, links -def get_pages(root_url, **kwargs): +def spider(root_url, **kwargs): """Gets web pages from a root URL. If depth is specified (e.g., depth=2), then this will also fetches pages linked from the root and its children up to depth. @@ -167,5 +173,69 @@ def get_pages(root_url, **kwargs): performance over a sequential fetch. """ max_depth = kwargs.setdefault('depth', 1) - pages = _spider((root_url, set(), root_url, None, 1, max_depth, False)) - return pages + pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False)) + return pages, links + + +def find_versions_of_archive(*archive_urls, **kwargs): + """Scrape web pages for new versions of a tarball. + + Arguments: + archive_urls: + URLs for different versions of a package. Typically these + are just the tarballs from the package file itself. By + default, this searches the parent directories of archives. + + Keyword Arguments: + list_url: + + URL for a listing of archives. Spack wills scrape these + pages for download links that look like the archive URL. + + list_depth: + Max depth to follow links on list_url pages. + + """ + list_url = kwargs.get('list_url', None) + list_depth = kwargs.get('list_depth', 1) + + # Generate a list of list_urls based on archive urls and any + # explicitly listed list_url in the package + list_urls = set() + if list_url: + list_urls.add(list_url) + for aurl in archive_urls: + list_urls.add(spack.url.find_list_url(aurl)) + + # Grab some web pages to scrape. + pages = {} + links = set() + for lurl in list_urls: + p, l = spider(lurl, depth=list_depth) + pages.update(p) + links.update(l) + + # Scrape them for archive URLs + regexes = [] + for aurl in archive_urls: + # This creates a regex from the URL with a capture group for + # the version part of the URL. The capture group is converted + # to a generic wildcard, so we can use this to extract things + # on a page that look like archive URLs. + url_regex = spack.url.wildcard_version(aurl) + + # We'll be a bit more liberal and just look for the archive + # part, not the full path. + regexes.append(os.path.basename(url_regex)) + + # Build a dict version -> URL from any links that match the wildcards. + versions = {} + for url in links: + if any(re.search(r, url) for r in regexes): + try: + ver = spack.url.parse_version(url) + versions[ver] = url + except spack.url.UndetectableVersionError as e: + continue + + return versions From 309877088d81dcac3ed4e45327ab83a99957e93b Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 22 Dec 2015 18:54:47 -0500 Subject: [PATCH 11/23] Build Julia --- var/spack/packages/julia/package.py | 55 +++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 var/spack/packages/julia/package.py diff --git a/var/spack/packages/julia/package.py b/var/spack/packages/julia/package.py new file mode 100644 index 00000000000..2c67b8e4439 --- /dev/null +++ b/var/spack/packages/julia/package.py @@ -0,0 +1,55 @@ +from spack import * + +class Julia(Package): + """The Julia Language: A fresh approach to technical computing""" + homepage = "http://julialang.org" + url = "http://github.com/JuliaLang/julia/releases/download/v0.4.2/julia-0.4.2.tar.gz" + + version('0.4.2', 'ccfeb4f4090c8b31083f5e1ccb03eb06') + + # Build-time dependencies + # depends_on("cmake") + # depends_on("awk") + # depends_on("m4") + # depends_on("pkg-config") + + # Run-time dependencies + #TODO depends_on("arpack") + #TODO depends_on("fftw") + #TODO depends_on("git") + #TODO depends_on("gmp") + # depends_on("libedit") + #TODO depends_on("mpfr") + #TODO depends_on("pcre2") + # depends_on("zlib") + + # BLAS and LAPACK: Julia prefers 64-bit versions on 64-bit systems. OpenBLAS + # has an option for this; make it available as variant. + + # LLVM: Julia works only with specific versions, and might require patches. + # Thus we let Julia install its own LLVM. + + # Other possible dependencies: + # USE_SYSTEM_OPENLIBM=0 + # USE_SYSTEM_OPENSPECFUN=0 + # USE_SYSTEM_DSFMT=0 + # USE_SYSTEM_SUITESPARSE=0 + # USE_SYSTEM_UTF8PROC=0 + # USE_SYSTEM_LIBGIT2=0 + + def install(self, spec, prefix): + options = ["CC=cc", + "CXX=c++", + "FC=fc", + #TODO "USE_SYSTEM_ARPACK=1", + #TODO "USE_SYSTEM_FFTW=1", + #TODO "USE_SYSTEM_GMP=1", + #TODO "USE_SYSTEM_MPFR=1", + #TODO "USE_SYSTEM_PCRE=1", + "prefix=%s" % prefix] + with open('Make.user', 'w') as f: + f.write('\n'.join(options) + '\n') + which('env')() + which('env')('which', 'cc') + make() + make("install") From 51f546fe92e46b9737ecf779d2f30787105c95ff Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 23 Dec 2015 08:40:55 -0500 Subject: [PATCH 12/23] Add comments to package file --- var/spack/packages/julia/package.py | 45 ++++++++++++++++++----------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/var/spack/packages/julia/package.py b/var/spack/packages/julia/package.py index 2c67b8e4439..9fd946c9054 100644 --- a/var/spack/packages/julia/package.py +++ b/var/spack/packages/julia/package.py @@ -1,4 +1,5 @@ from spack import * +import os class Julia(Package): """The Julia Language: A fresh approach to technical computing""" @@ -13,19 +14,27 @@ class Julia(Package): # depends_on("m4") # depends_on("pkg-config") + # I think that Julia requires the dependencies above, but it builds find (on + # my system) without these. We should enable them as necessary. + # Run-time dependencies - #TODO depends_on("arpack") - #TODO depends_on("fftw") - #TODO depends_on("git") - #TODO depends_on("gmp") - # depends_on("libedit") - #TODO depends_on("mpfr") - #TODO depends_on("pcre2") - # depends_on("zlib") + # depends_on("arpack") + # depends_on("fftw +float") + # depends_on("gmp") + # depends_on("mpfr") + # depends_on("pcre2") + + # ARPACK: Requires BLAS and LAPACK; needs to use the same version as Julia. # BLAS and LAPACK: Julia prefers 64-bit versions on 64-bit systems. OpenBLAS # has an option for this; make it available as variant. + # FFTW: Something doesn't work when using a pre-installed FFTW library; need + # to investigate. + + # GMP, MPFR: Something doesn't work when using a pre-installed FFTW library; + # need to investigate. + # LLVM: Julia works only with specific versions, and might require patches. # Thus we let Julia install its own LLVM. @@ -38,18 +47,20 @@ class Julia(Package): # USE_SYSTEM_LIBGIT2=0 def install(self, spec, prefix): - options = ["CC=cc", - "CXX=c++", - "FC=fc", - #TODO "USE_SYSTEM_ARPACK=1", - #TODO "USE_SYSTEM_FFTW=1", - #TODO "USE_SYSTEM_GMP=1", - #TODO "USE_SYSTEM_MPFR=1", + # Explicitly setting CC, CXX, or FC breaks building libuv, one of + # Julia's dependencies. This might be a Darwin-specific problem. Given + # how Spack sets up compilers, Julia should still use Spack's compilers, + # even if we don't specify them explicitly. + options = [#"CC=cc", + #"CXX=c++", + #"FC=fc", + #"USE_SYSTEM_ARPACK=1", + #"USE_SYSTEM_FFTW=1", + #"USE_SYSTEM_GMP=1", + #"USE_SYSTEM_MPFR=1", #TODO "USE_SYSTEM_PCRE=1", "prefix=%s" % prefix] with open('Make.user', 'w') as f: f.write('\n'.join(options) + '\n') - which('env')() - which('env')('which', 'cc') make() make("install") From f37f872e5f4920f9dc22237deed5895b94700136 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 23 Dec 2015 16:23:58 -0800 Subject: [PATCH 13/23] Fix #85 and #228: errors fetching VCS packages from a mirror. - Stage and fetcher were not being set up properly when fetching using a different fetch strategy than the default one for the package. - This is fixed but fetch/stage/mirror logic is still too complicated and long-term needs a rethink. - Spack will now print a warning when fetching a checksum-less tarball from a mirror -- users should be careful to use https or local filesystem mirrors for this. --- lib/spack/spack/fetch_strategy.py | 4 +-- lib/spack/spack/stage.py | 53 +++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index a9374fb34b9..0657146bf63 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -687,7 +687,7 @@ def for_package_version(pkg, version): class FetchError(spack.error.SpackError): - def __init__(self, msg, long_msg): + def __init__(self, msg, long_msg=None): super(FetchError, self).__init__(msg, long_msg) @@ -705,7 +705,7 @@ def __init__(self, msg, long_msg): class NoDigestError(FetchError): - def __init__(self, msg, long_msg): + def __init__(self, msg, long_msg=None): super(NoDigestError, self).__init__(msg, long_msg) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 754344fc016..76ca7273cb9 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -82,14 +82,18 @@ def __init__(self, url_or_fetch_strategy, **kwargs): stage object later). If name is not provided, then this stage will be given a unique name automatically. """ + # TODO: fetch/stage coupling needs to be reworked -- the logic + # TODO: here is convoluted and not modular enough. if isinstance(url_or_fetch_strategy, basestring): self.fetcher = fs.from_url(url_or_fetch_strategy) elif isinstance(url_or_fetch_strategy, fs.FetchStrategy): self.fetcher = url_or_fetch_strategy else: raise ValueError("Can't construct Stage without url or fetch strategy") - self.fetcher.set_stage(self) + self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. + self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. + self.name = kwargs.get('name') self.mirror_path = kwargs.get('mirror_path') @@ -198,17 +202,18 @@ def _setup(self): @property def archive_file(self): """Path to the source archive within this stage directory.""" - if not isinstance(self.fetcher, fs.URLFetchStrategy): - return None + paths = [] + if isinstance(self.fetcher, fs.URLFetchStrategy): + paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url))) - paths = [os.path.join(self.path, os.path.basename(self.fetcher.url))] if self.mirror_path: paths.append(os.path.join(self.path, os.path.basename(self.mirror_path))) for path in paths: if os.path.exists(path): return path - return None + else: + return None @property @@ -238,23 +243,34 @@ def fetch(self): """Downloads an archive or checks out code from a repository.""" self.chdir() - fetchers = [self.fetcher] + fetchers = [self.default_fetcher] # TODO: move mirror logic out of here and clean it up! + # TODO: Or @alalazo may have some ideas about how to use a + # TODO: CompositeFetchStrategy here. + self.skip_checksum_for_mirror = True if self.mirror_path: urls = ["%s/%s" % (m, self.mirror_path) for m in _get_mirrors()] + # If this archive is normally fetched from a tarball URL, + # then use the same digest. `spack mirror` ensures that + # the checksum will be the same. digest = None - if isinstance(self.fetcher, fs.URLFetchStrategy): - digest = self.fetcher.digest - fetchers = [fs.URLFetchStrategy(url, digest) - for url in urls] + fetchers - for f in fetchers: - f.set_stage(self) + if isinstance(self.default_fetcher, fs.URLFetchStrategy): + digest = self.default_fetcher.digest + + # Have to skip the checkesum for things archived from + # repositories. How can this be made safer? + self.skip_checksum_for_mirror = not bool(digest) + + for url in urls: + fetchers.insert(0, fs.URLFetchStrategy(url, digest)) for fetcher in fetchers: try: - fetcher.fetch() + fetcher.set_stage(self) + self.fetcher = fetcher + self.fetcher.fetch() break except spack.error.SpackError, e: tty.msg("Fetching from %s failed." % fetcher) @@ -262,13 +278,22 @@ def fetch(self): continue else: errMessage = "All fetchers failed for %s" % self.name + self.fetcher = self.default_fetcher raise fs.FetchError(errMessage, None) def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" - self.fetcher.check() + if self.fetcher is not self.default_fetcher and self.skip_checksum_for_mirror: + tty.warn("Fetching from mirror without a checksum!", + "This package is normally checked out from a version " + "control system, but it has been archived on a spack " + "mirror. This means we cannot know a checksum for the " + "tarball in advance. Be sure that your connection to " + "this mirror is secure!.") + else: + self.fetcher.check() def expand_archive(self): From 52625de52c396b995716c0451ba169df0f3a4b1f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 29 Nov 2015 16:02:51 -0800 Subject: [PATCH 14/23] Fix #154 -- better log messages for do_patch() --- lib/spack/spack/package.py | 40 +++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b44554e4183..84bcb15f7fe 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -733,9 +733,10 @@ def do_patch(self): # Construct paths to special files in the archive dir used to # keep track of whether patches were successfully applied. - archive_dir = self.stage.source_path - good_file = join_path(archive_dir, '.spack_patched') - bad_file = join_path(archive_dir, '.spack_patch_failed') + archive_dir = self.stage.source_path + good_file = join_path(archive_dir, '.spack_patched') + no_patches_file = join_path(archive_dir, '.spack_no_patches') + bad_file = join_path(archive_dir, '.spack_patch_failed') # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. @@ -749,29 +750,46 @@ def do_patch(self): if os.path.isfile(good_file): tty.msg("Already patched %s" % self.name) return + elif os.path.isfile(no_patches_file): + tty.msg("No patches needed for %s." % self.name) + return # Apply all the patches for specs that match this one + patched = False for spec, patch_list in self.patches.items(): if self.spec.satisfies(spec): for patch in patch_list: - tty.msg('Applying patch %s' % patch.path_or_url) try: patch.apply(self.stage) + tty.msg('Applied patch %s' % patch.path_or_url) + patched = True except: # Touch bad file if anything goes wrong. + tty.msg('Patch %s failed.' % patch.path_or_url) touch(bad_file) raise - # patch succeeded. Get rid of failed file & touch good file so we - # don't try to patch again again next time. + if has_patch_fun: + try: + self.patch() + tty.msg("Ran patch() for %s." % self.name) + patched = True + except: + tty.msg("patch() function failed for %s." % self.name) + touch(bad_file) + raise + + # Get rid of any old failed file -- patches have either succeeded + # or are not needed. This is mostly defensive -- it's needed + # if the restage() method doesn't clean *everything* (e.g., for a repo) if os.path.isfile(bad_file): os.remove(bad_file) - touch(good_file) - if has_patch_fun: - self.patch() - - tty.msg("Patched %s" % self.name) + # touch good or no patches file so that we skip next time. + if patched: + touch(good_file) + else: + touch(no_patches_file) def do_fake_install(self): From d7607973fa37ec2a5325111336be44f02cb95860 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Tue, 29 Dec 2015 09:04:54 -0800 Subject: [PATCH 15/23] fix for array handling in ancient bash fixes #284 --- share/spack/setup-env.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 47202f6087a..4faae0d2c9d 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -55,13 +55,12 @@ # avoids the need to come up with a user-friendly naming scheme for # spack dotfiles. ######################################################################## +arrtest[0]='test' || (echo 'Failure: arrays not supported in this version of bash.' && exit 2) + function spack { # save raw arguments into an array before butchering them - args=() - for a in "$@"; do - # yup, this is awful, blame bash2 compat - args=("${args[@]}" "$a") - done + declare -a args=( "$@" ) + # accumulate initial flags for main spack command _sp_flags="" while [[ "$1" =~ ^- ]]; do From 496e4856916cb881da916539635e9bfa3e1406af Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 10:13:46 -0800 Subject: [PATCH 16/23] removing check as it can cause failures in zsh --- share/spack/setup-env.sh | 1 - var/spack/packages/binutils/package.py | 5 + var/spack/packages/clang/package.py | 48 +++++--- var/spack/packages/gcc/package.py | 32 ++++- var/spack/packages/llvm/package.py | 154 ++++++++++++++++++++++--- 5 files changed, 204 insertions(+), 36 deletions(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 4faae0d2c9d..6503728d8f2 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -55,7 +55,6 @@ # avoids the need to come up with a user-friendly naming scheme for # spack dotfiles. ######################################################################## -arrtest[0]='test' || (echo 'Failure: arrays not supported in this version of bash.' && exit 2) function spack { # save raw arguments into an array before butchering them diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py index 123f4598f69..a79fc5fb699 100644 --- a/var/spack/packages/binutils/package.py +++ b/var/spack/packages/binutils/package.py @@ -11,6 +11,7 @@ class Binutils(Package): # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell variant('krellpatch', default=False, description="build with openspeedshop based patch.") + variant('gold', default=True, description="build the gold linker") patch('binutilskrell-2.24.patch', when='@2.24+krellpatch') variant('libiberty', default=False, description='Also install libiberty.') @@ -26,6 +27,10 @@ def install(self, spec, prefix): '--enable-targets=all', '--with-sysroot=/'] + if '+gold' in spec: + configure_args.append('--enable-gold') + configure_args.append('--enable-plugins') + if '+libiberty' in spec: configure_args.append('--enable-install-libiberty') diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py index e46e08d5f11..99e805d15d0 100644 --- a/var/spack/packages/clang/package.py +++ b/var/spack/packages/clang/package.py @@ -36,20 +36,42 @@ class Clang(Package): homepage = 'http://clang.llvm.org' url = 'http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz' - depends_on('llvm@3.7.0', when='@3.7.0') - depends_on('llvm@3.6.2', when='@3.6.2') - depends_on('llvm@3.5.1', when='@3.5.1') + clang_url = 'http://llvm.org/releases/%(version)s/cfe-%(version)s.src.tar.xz' - version('3.7.0', '8f9d27335e7331cf0a4711e952f21f01', url='http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz') - version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz') - version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz') + resources = { + 'clang-tools-extra' : { + 'url' : 'http://llvm.org/releases/%(version)s/clang-tools-extra-%(version)s.src.tar.xz', + 'destination' : 'tools' + }, + } + releases = [ + { + 'version' : '3.7.0', + 'md5':'8f9d27335e7331cf0a4711e952f21f01', + 'resources' : { 'clang-tools-extra' : 'd5a87dacb65d981a427a536f6964642e' } + }, + { + 'version' : '3.6.2', + 'md5':'ff862793682f714bb7862325b9c06e20', + 'resources' : { 'clang-tools-extra' : '3ebc1dc41659fcec3db1b47d81575e06' } + }, + { + 'version' : '3.5.1', + 'md5':'93f9532f8f7e6f1d8e5c1116907051cb', + 'resources' : { 'clang-tools-extra' : 'f13f31ed3038acadc6fa63fef812a246' } + }, + ] - ########## - # @3.7.0 - resource(name='clang-tools-extra', - url='http://llvm.org/releases/3.7.0/clang-tools-extra-3.7.0.src.tar.xz', - md5='d5a87dacb65d981a427a536f6964642e', destination='tools', when='@3.7.0') - ########## + for release in releases: + version(release['version'], release['md5'], url=clang_url % release) + depends_on('llvm@%(version)s' % release, when='@%(version)s' % release) + + for name, md5 in release['resources'].items(): + resource(name=name, + url=resources[name]['url'] % release, + md5=md5, + destination=resources[name]['destination'], + when='@%(version)s' % release) def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag @@ -92,4 +114,4 @@ def do_link(source_dir, destination_dir): # Link folder and files in lib llvm_dir = clang_lib_dir_at(spec['llvm'].prefix) clang_dir = clang_lib_dir_at(self.prefix) - do_link(llvm_dir, clang_dir) \ No newline at end of file + do_link(llvm_dir, clang_dir) diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py index 7ec160d5957..da90c220237 100644 --- a/var/spack/packages/gcc/package.py +++ b/var/spack/packages/gcc/package.py @@ -52,19 +52,45 @@ class Gcc(Package): depends_on("mpfr") depends_on("gmp") depends_on("mpc") # when @4.5: - depends_on("binutils~libiberty") + depends_on("binutils~libiberty+gold") # Save these until we can do optional deps. depends_on("isl", when=DEPENDS_ON_ISL_PREDICATE) #depends_on("ppl") #depends_on("cloog") + variant('lang_go', default=True, description="Include support for the go language") + variant('lang_fortran', default=True, description="Include support for the Fortran language") + variant('lang_c', default=True, description="Include support for the C language") + variant('lang_cxx', default=True, description="Include support for the C++ language") + variant('lang_ada', default=False, description="Include support for the ada language") + variant('lang_java', default=False, description="Include support for the java language") + variant('lang_objc', default=True, description="Include support for the Objective-C language") + variant('lang_objcxx', default=True, description="Include support for the Objective-C++ language") + variant('jit', default=False, description="Include support for JIT") + def install(self, spec, prefix): # libjava/configure needs a minor fix to install into spack paths. filter_file(r"'@.*@'", "'@[[:alnum:]]*@'", 'libjava/configure', string=True) - enabled_languages = set(('c', 'c++', 'fortran', 'java', 'objc')) - if spec.satisfies("@4.7.1:"): + enabled_languages = set()#('c', 'c++', 'fortran', 'java', 'objc') + if '+lang_fortran' in spec: + enabled_languages.add('fortran') + if '+lang_c' in spec: + enabled_languages.add('c') + if '+lang_cxx' in spec: + enabled_languages.add('c++') + if '+lang_java' in spec: + enabled_languages.add('java') + if '+lang_objc' in spec: + enabled_languages.add('objc') + if '+lang_objcxx' in spec: + enabled_languages.add('obj-c++') + if '+lang_ada' in spec: + enabled_languages.add('ada') + if '+jit' in spec: + enabled_languages.add('jit') + if spec.satisfies("@4.7.1:") and '+lang_go' in spec: enabled_languages.add('go') # Generic options to compile GCC diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index a3307584e08..8a36a873675 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -3,8 +3,9 @@ # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. -# Written by David Beckingsale, david@llnl.gov, All rights reserved. -# LLNL-CODE-647188 +# Written by David Beckingsale, david@llnl.gov +# Heavily modified by others, including Tom Scogland, scogland1@llnl.gov +# All rights reserved. LLNL-CODE-647188 # # For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. @@ -23,6 +24,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * +import os class Llvm(Package): @@ -40,33 +42,147 @@ class Llvm(Package): version('3.5.1', '2d3d8004f38852aa679e5945b8ce0b14', url='http://llvm.org/releases/3.5.1/llvm-3.5.1.src.tar.xz') version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package + variant('debug', default=False, description="Build a debug version of LLVM, this increases binary size considerably") + variant('clang', default=True, description="Build the LLVM C/C++/Objective-C compiler frontend") + variant('clang_extra', default=True, description="Build the clang extra tools") + variant('lldb', default=True, description="Build the LLVM debugger") + variant('internal_unwind', default=True, description="Build and use the libcxxabi built-in unwinder") + variant('compiler-rt', default=False, description="Compiler runtime including sanitizers and unwinding libraries") + variant('gold', default=True, description="Add support for the gold linker and build the gold linker plugin for LTO") + + # Universal dependency depends_on('python@2.7:') - variant('libcxx', default=False, description="Builds the LLVM Standard C++ library targeting C++11") + # lldb dependencies + depends_on('ncurses', when='+lldb') + depends_on('swig', when='+lldb') + depends_on('libedit', when='+lldb') - ########## - # @3.7.0 - resource(name='compiler-rt', - url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5', - destination='projects', when='@3.7.0') - resource(name='openmp', - url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f', - destination='projects', when='@3.7.0') - resource(name='libcxx', - url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd', - destination='projects', placement='libcxx', when='+libcxx@3.7.0') - resource(name='libcxxabi', - url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772', - destination='projects', placement='libcxxabi', when='+libcxx@3.7.0') - ########## + # gold support + depends_on('binutils+gold', when='+gold') + + base_url = 'http://llvm.org/releases/%%(version)s/%(pkg)s-%%(version)s.src.tar.xz' + llvm_url = base_url % { 'pkg' : 'llvm'} + + resources = { + 'compiler-rt' : { + 'url' : base_url % { 'pkg' : 'compiler-rt'}, + 'destination' : 'projects', + 'placement' : 'compiler-rt', + }, + 'openmp' : { + 'url' : base_url % { 'pkg' : 'openmp'}, + 'destination' : 'projects', + 'placement' : 'openmp', + }, + 'libcxx' : { + 'url' : base_url % { 'pkg' : 'libcxx'}, + 'destination' : 'projects', + 'placement' : 'libcxx', + }, + 'libcxxabi' : { + 'url' : base_url % { 'pkg' : 'libcxxabi'}, + 'destination' : 'projects', + 'placement' : 'libcxxabi', + }, + 'clang' : { + 'url' : base_url % { 'pkg' : 'cfe'}, + 'destination' : 'tools', + 'placement' : 'clang', + }, + 'clang-tools-extra' : { + 'url' : base_url % { 'pkg' : 'clang-tools-extra'}, + 'destination' : 'tools/clang/tools', + 'placement' : 'clang-tools-extra', + }, + 'lldb' : { + 'url' : base_url % { 'pkg' : 'lldb'}, + 'destination' : 'tools', + 'placement' : 'lldb', + }, + 'polly' : { + 'url' : base_url % { 'pkg' : 'polly'}, + 'destination' : 'tools', + 'placement' : 'polly', + }, + 'llvm-libunwind' : { + 'url' : base_url % { 'pkg' : 'libunwind'}, + 'destination' : 'projects', + 'placement' : 'libunwind', + }, + } + releases = [ + { + 'version' : '3.7.0', + 'md5':'b98b9495e5655a672d6cb83e1a180f8e', + 'resources' : { + # FIXME: compiler-rt + # 'compiler-rt' : '383c10affd513026f08936b5525523f5', + 'openmp' : 'f482c86fdead50ba246a1a2b0bbf206f', + 'polly' : '32f93ffc9cc7e042df22089761558f8b', + 'libcxx' : '46aa5175cbe1ad42d6e9c995968e56dd', + 'libcxxabi' : '5aa769e2fca79fa5335cfae8f6258772', + 'clang' : '8f9d27335e7331cf0a4711e952f21f01', + 'clang-tools-extra' : 'd5a87dacb65d981a427a536f6964642e', + 'lldb' : 'e5931740400d1dc3e7db4c7ba2ceff68', + 'llvm-libunwind' : '9a75392eb7eb8ed5c0840007e212baf5', + } + }, + { + 'version' : '3.6.2', + 'md5':'0c1ee3597d75280dee603bae9cbf5cc2', + 'resources' : { + 'compiler-rt' : 'e3bc4eb7ba8c39a6fe90d6c988927f3c', + 'openmp' : '65dd5863b9b270960a96817e9152b123', + 'libcxx' : '22214c90697636ef960a49aef7c1823a', + 'libcxxabi' : '17518e361e4e228f193dd91e8ef54ba2', + 'clang' : 'ff862793682f714bb7862325b9c06e20', + 'clang-tools-extra' : '3ebc1dc41659fcec3db1b47d81575e06', + 'lldb' : '51e5eb552f777b950bb0ff326e60d5f0', + } + }, + { + 'version' : '3.5.1', + 'md5':'2d3d8004f38852aa679e5945b8ce0b14', + 'resources' : { + 'compiler-rt' : 'd626cfb8a9712cb92b820798ab5bc1f8', + 'openmp' : '121ddb10167d7fc38b1f7e4b029cf059', + 'libcxx' : '406f09b1dab529f3f7879f4d548329d2', + 'libcxxabi' : 'b22c707e8d474a99865ad3c521c3d464', + 'clang' : '93f9532f8f7e6f1d8e5c1116907051cb', + 'clang-tools-extra' : 'f13f31ed3038acadc6fa63fef812a246', + 'lldb' : 'cc5ea8a414c62c33e760517f8929a204', + } + }, + ] + + for release in releases: + version(release['version'], release['md5'], url=llvm_url % release) + + for name, md5 in release['resources'].items(): + resource(name=name, + url=resources[name]['url'] % release, + md5=md5, + destination=resources[name]['destination'], + when='@%(version)s' % release, + placement=resources[name].get('placement', None)) def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag + build_type = 'Debug' if '+debug' in spec else 'Release' + + if '+clang' not in spec: + if '+clang_extra' in spec: + raise SpackException('The clang_extra variant requires the clang variant to be selected') + if '+lldb' in spec: + raise SpackException('The lldb variant requires the clang variant to be selected') with working_dir('spack-build', create=True): cmake('..', + '-DCMAKE_RELEASE_TYPE=' + build_type, '-DLLVM_REQUIRES_RTTI:BOOL=ON', '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix, *std_cmake_args) - make() + make("VERBOSE=1") make("install") + From 3e3667f254cd9c288fbb2641c51ed871a28f0112 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 30 Dec 2015 10:20:27 -0800 Subject: [PATCH 17/23] Revert "removing check as it can cause failures in zsh" This reverts commit 496e4856916cb881da916539635e9bfa3e1406af. Undo overly large set of changes -- LLVM wasn't ready. Tom can recommit just the zsh stuff. --- share/spack/setup-env.sh | 1 + var/spack/packages/binutils/package.py | 5 - var/spack/packages/clang/package.py | 48 +++----- var/spack/packages/gcc/package.py | 32 +---- var/spack/packages/llvm/package.py | 154 +++---------------------- 5 files changed, 36 insertions(+), 204 deletions(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 6503728d8f2..4faae0d2c9d 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -55,6 +55,7 @@ # avoids the need to come up with a user-friendly naming scheme for # spack dotfiles. ######################################################################## +arrtest[0]='test' || (echo 'Failure: arrays not supported in this version of bash.' && exit 2) function spack { # save raw arguments into an array before butchering them diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py index a79fc5fb699..123f4598f69 100644 --- a/var/spack/packages/binutils/package.py +++ b/var/spack/packages/binutils/package.py @@ -11,7 +11,6 @@ class Binutils(Package): # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell variant('krellpatch', default=False, description="build with openspeedshop based patch.") - variant('gold', default=True, description="build the gold linker") patch('binutilskrell-2.24.patch', when='@2.24+krellpatch') variant('libiberty', default=False, description='Also install libiberty.') @@ -27,10 +26,6 @@ def install(self, spec, prefix): '--enable-targets=all', '--with-sysroot=/'] - if '+gold' in spec: - configure_args.append('--enable-gold') - configure_args.append('--enable-plugins') - if '+libiberty' in spec: configure_args.append('--enable-install-libiberty') diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py index 99e805d15d0..e46e08d5f11 100644 --- a/var/spack/packages/clang/package.py +++ b/var/spack/packages/clang/package.py @@ -36,42 +36,20 @@ class Clang(Package): homepage = 'http://clang.llvm.org' url = 'http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz' - clang_url = 'http://llvm.org/releases/%(version)s/cfe-%(version)s.src.tar.xz' + depends_on('llvm@3.7.0', when='@3.7.0') + depends_on('llvm@3.6.2', when='@3.6.2') + depends_on('llvm@3.5.1', when='@3.5.1') - resources = { - 'clang-tools-extra' : { - 'url' : 'http://llvm.org/releases/%(version)s/clang-tools-extra-%(version)s.src.tar.xz', - 'destination' : 'tools' - }, - } - releases = [ - { - 'version' : '3.7.0', - 'md5':'8f9d27335e7331cf0a4711e952f21f01', - 'resources' : { 'clang-tools-extra' : 'd5a87dacb65d981a427a536f6964642e' } - }, - { - 'version' : '3.6.2', - 'md5':'ff862793682f714bb7862325b9c06e20', - 'resources' : { 'clang-tools-extra' : '3ebc1dc41659fcec3db1b47d81575e06' } - }, - { - 'version' : '3.5.1', - 'md5':'93f9532f8f7e6f1d8e5c1116907051cb', - 'resources' : { 'clang-tools-extra' : 'f13f31ed3038acadc6fa63fef812a246' } - }, - ] + version('3.7.0', '8f9d27335e7331cf0a4711e952f21f01', url='http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz') + version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz') + version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz') - for release in releases: - version(release['version'], release['md5'], url=clang_url % release) - depends_on('llvm@%(version)s' % release, when='@%(version)s' % release) - - for name, md5 in release['resources'].items(): - resource(name=name, - url=resources[name]['url'] % release, - md5=md5, - destination=resources[name]['destination'], - when='@%(version)s' % release) + ########## + # @3.7.0 + resource(name='clang-tools-extra', + url='http://llvm.org/releases/3.7.0/clang-tools-extra-3.7.0.src.tar.xz', + md5='d5a87dacb65d981a427a536f6964642e', destination='tools', when='@3.7.0') + ########## def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag @@ -114,4 +92,4 @@ def do_link(source_dir, destination_dir): # Link folder and files in lib llvm_dir = clang_lib_dir_at(spec['llvm'].prefix) clang_dir = clang_lib_dir_at(self.prefix) - do_link(llvm_dir, clang_dir) + do_link(llvm_dir, clang_dir) \ No newline at end of file diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py index da90c220237..7ec160d5957 100644 --- a/var/spack/packages/gcc/package.py +++ b/var/spack/packages/gcc/package.py @@ -52,45 +52,19 @@ class Gcc(Package): depends_on("mpfr") depends_on("gmp") depends_on("mpc") # when @4.5: - depends_on("binutils~libiberty+gold") + depends_on("binutils~libiberty") # Save these until we can do optional deps. depends_on("isl", when=DEPENDS_ON_ISL_PREDICATE) #depends_on("ppl") #depends_on("cloog") - variant('lang_go', default=True, description="Include support for the go language") - variant('lang_fortran', default=True, description="Include support for the Fortran language") - variant('lang_c', default=True, description="Include support for the C language") - variant('lang_cxx', default=True, description="Include support for the C++ language") - variant('lang_ada', default=False, description="Include support for the ada language") - variant('lang_java', default=False, description="Include support for the java language") - variant('lang_objc', default=True, description="Include support for the Objective-C language") - variant('lang_objcxx', default=True, description="Include support for the Objective-C++ language") - variant('jit', default=False, description="Include support for JIT") - def install(self, spec, prefix): # libjava/configure needs a minor fix to install into spack paths. filter_file(r"'@.*@'", "'@[[:alnum:]]*@'", 'libjava/configure', string=True) - enabled_languages = set()#('c', 'c++', 'fortran', 'java', 'objc') - if '+lang_fortran' in spec: - enabled_languages.add('fortran') - if '+lang_c' in spec: - enabled_languages.add('c') - if '+lang_cxx' in spec: - enabled_languages.add('c++') - if '+lang_java' in spec: - enabled_languages.add('java') - if '+lang_objc' in spec: - enabled_languages.add('objc') - if '+lang_objcxx' in spec: - enabled_languages.add('obj-c++') - if '+lang_ada' in spec: - enabled_languages.add('ada') - if '+jit' in spec: - enabled_languages.add('jit') - if spec.satisfies("@4.7.1:") and '+lang_go' in spec: + enabled_languages = set(('c', 'c++', 'fortran', 'java', 'objc')) + if spec.satisfies("@4.7.1:"): enabled_languages.add('go') # Generic options to compile GCC diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index 8a36a873675..a3307584e08 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -3,9 +3,8 @@ # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. -# Written by David Beckingsale, david@llnl.gov -# Heavily modified by others, including Tom Scogland, scogland1@llnl.gov -# All rights reserved. LLNL-CODE-647188 +# Written by David Beckingsale, david@llnl.gov, All rights reserved. +# LLNL-CODE-647188 # # For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. @@ -24,7 +23,6 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os class Llvm(Package): @@ -42,147 +40,33 @@ class Llvm(Package): version('3.5.1', '2d3d8004f38852aa679e5945b8ce0b14', url='http://llvm.org/releases/3.5.1/llvm-3.5.1.src.tar.xz') version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package - variant('debug', default=False, description="Build a debug version of LLVM, this increases binary size considerably") - variant('clang', default=True, description="Build the LLVM C/C++/Objective-C compiler frontend") - variant('clang_extra', default=True, description="Build the clang extra tools") - variant('lldb', default=True, description="Build the LLVM debugger") - variant('internal_unwind', default=True, description="Build and use the libcxxabi built-in unwinder") - variant('compiler-rt', default=False, description="Compiler runtime including sanitizers and unwinding libraries") - variant('gold', default=True, description="Add support for the gold linker and build the gold linker plugin for LTO") - - # Universal dependency depends_on('python@2.7:') - # lldb dependencies - depends_on('ncurses', when='+lldb') - depends_on('swig', when='+lldb') - depends_on('libedit', when='+lldb') + variant('libcxx', default=False, description="Builds the LLVM Standard C++ library targeting C++11") - # gold support - depends_on('binutils+gold', when='+gold') - - base_url = 'http://llvm.org/releases/%%(version)s/%(pkg)s-%%(version)s.src.tar.xz' - llvm_url = base_url % { 'pkg' : 'llvm'} - - resources = { - 'compiler-rt' : { - 'url' : base_url % { 'pkg' : 'compiler-rt'}, - 'destination' : 'projects', - 'placement' : 'compiler-rt', - }, - 'openmp' : { - 'url' : base_url % { 'pkg' : 'openmp'}, - 'destination' : 'projects', - 'placement' : 'openmp', - }, - 'libcxx' : { - 'url' : base_url % { 'pkg' : 'libcxx'}, - 'destination' : 'projects', - 'placement' : 'libcxx', - }, - 'libcxxabi' : { - 'url' : base_url % { 'pkg' : 'libcxxabi'}, - 'destination' : 'projects', - 'placement' : 'libcxxabi', - }, - 'clang' : { - 'url' : base_url % { 'pkg' : 'cfe'}, - 'destination' : 'tools', - 'placement' : 'clang', - }, - 'clang-tools-extra' : { - 'url' : base_url % { 'pkg' : 'clang-tools-extra'}, - 'destination' : 'tools/clang/tools', - 'placement' : 'clang-tools-extra', - }, - 'lldb' : { - 'url' : base_url % { 'pkg' : 'lldb'}, - 'destination' : 'tools', - 'placement' : 'lldb', - }, - 'polly' : { - 'url' : base_url % { 'pkg' : 'polly'}, - 'destination' : 'tools', - 'placement' : 'polly', - }, - 'llvm-libunwind' : { - 'url' : base_url % { 'pkg' : 'libunwind'}, - 'destination' : 'projects', - 'placement' : 'libunwind', - }, - } - releases = [ - { - 'version' : '3.7.0', - 'md5':'b98b9495e5655a672d6cb83e1a180f8e', - 'resources' : { - # FIXME: compiler-rt - # 'compiler-rt' : '383c10affd513026f08936b5525523f5', - 'openmp' : 'f482c86fdead50ba246a1a2b0bbf206f', - 'polly' : '32f93ffc9cc7e042df22089761558f8b', - 'libcxx' : '46aa5175cbe1ad42d6e9c995968e56dd', - 'libcxxabi' : '5aa769e2fca79fa5335cfae8f6258772', - 'clang' : '8f9d27335e7331cf0a4711e952f21f01', - 'clang-tools-extra' : 'd5a87dacb65d981a427a536f6964642e', - 'lldb' : 'e5931740400d1dc3e7db4c7ba2ceff68', - 'llvm-libunwind' : '9a75392eb7eb8ed5c0840007e212baf5', - } - }, - { - 'version' : '3.6.2', - 'md5':'0c1ee3597d75280dee603bae9cbf5cc2', - 'resources' : { - 'compiler-rt' : 'e3bc4eb7ba8c39a6fe90d6c988927f3c', - 'openmp' : '65dd5863b9b270960a96817e9152b123', - 'libcxx' : '22214c90697636ef960a49aef7c1823a', - 'libcxxabi' : '17518e361e4e228f193dd91e8ef54ba2', - 'clang' : 'ff862793682f714bb7862325b9c06e20', - 'clang-tools-extra' : '3ebc1dc41659fcec3db1b47d81575e06', - 'lldb' : '51e5eb552f777b950bb0ff326e60d5f0', - } - }, - { - 'version' : '3.5.1', - 'md5':'2d3d8004f38852aa679e5945b8ce0b14', - 'resources' : { - 'compiler-rt' : 'd626cfb8a9712cb92b820798ab5bc1f8', - 'openmp' : '121ddb10167d7fc38b1f7e4b029cf059', - 'libcxx' : '406f09b1dab529f3f7879f4d548329d2', - 'libcxxabi' : 'b22c707e8d474a99865ad3c521c3d464', - 'clang' : '93f9532f8f7e6f1d8e5c1116907051cb', - 'clang-tools-extra' : 'f13f31ed3038acadc6fa63fef812a246', - 'lldb' : 'cc5ea8a414c62c33e760517f8929a204', - } - }, - ] - - for release in releases: - version(release['version'], release['md5'], url=llvm_url % release) - - for name, md5 in release['resources'].items(): - resource(name=name, - url=resources[name]['url'] % release, - md5=md5, - destination=resources[name]['destination'], - when='@%(version)s' % release, - placement=resources[name].get('placement', None)) + ########## + # @3.7.0 + resource(name='compiler-rt', + url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5', + destination='projects', when='@3.7.0') + resource(name='openmp', + url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f', + destination='projects', when='@3.7.0') + resource(name='libcxx', + url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd', + destination='projects', placement='libcxx', when='+libcxx@3.7.0') + resource(name='libcxxabi', + url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772', + destination='projects', placement='libcxxabi', when='+libcxx@3.7.0') + ########## def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag - build_type = 'Debug' if '+debug' in spec else 'Release' - - if '+clang' not in spec: - if '+clang_extra' in spec: - raise SpackException('The clang_extra variant requires the clang variant to be selected') - if '+lldb' in spec: - raise SpackException('The lldb variant requires the clang variant to be selected') with working_dir('spack-build', create=True): cmake('..', - '-DCMAKE_RELEASE_TYPE=' + build_type, '-DLLVM_REQUIRES_RTTI:BOOL=ON', '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix, *std_cmake_args) - make("VERBOSE=1") + make() make("install") - From 8726d2fe2e1e6c7b16fafb398fdcea851392c2cd Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 10:29:11 -0800 Subject: [PATCH 18/23] removing ill-fated array check for non-portability --- share/spack/setup-env.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 4faae0d2c9d..6503728d8f2 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -55,7 +55,6 @@ # avoids the need to come up with a user-friendly naming scheme for # spack dotfiles. ######################################################################## -arrtest[0]='test' || (echo 'Failure: arrays not supported in this version of bash.' && exit 2) function spack { # save raw arguments into an array before butchering them From 926efed31c6b8280060188e18466ad7cf7be4ebf Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 10:55:52 -0800 Subject: [PATCH 19/23] another portability fix, this time for zsh/non-bash sh Yay for non-portable declaration syntax. After the previous screwiness I ran this through a number of shells, and found that this is the most portable version I coudl seem to get. --- share/spack/setup-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 6503728d8f2..586a5b836b1 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -58,7 +58,7 @@ function spack { # save raw arguments into an array before butchering them - declare -a args=( "$@" ) + args=( "$@" ) # accumulate initial flags for main spack command _sp_flags="" From 31f674da3c8676fc4ac27327ef735c8a2c12dd48 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 11:13:09 -0800 Subject: [PATCH 20/23] python libraries, libedit, and ninja A pile of libraries and tools, libedit is actually important as a replacement of readline for non-GPL projects. Also ninja may be worthwhile for some of the larger CMake projects, like llvm/clang. --- var/spack/packages/libedit/package.py | 14 ++++++++++++++ var/spack/packages/ninja/package.py | 22 ++++++++++++++++++++++ var/spack/packages/py-blessings/package.py | 15 +++++++++++++++ var/spack/packages/py-coverage/package.py | 16 ++++++++++++++++ var/spack/packages/py-tappy/package.py | 15 +++++++++++++++ var/spack/packages/py-urwid/package.py | 16 ++++++++++++++++ 6 files changed, 98 insertions(+) create mode 100644 var/spack/packages/libedit/package.py create mode 100644 var/spack/packages/ninja/package.py create mode 100644 var/spack/packages/py-blessings/package.py create mode 100644 var/spack/packages/py-coverage/package.py create mode 100644 var/spack/packages/py-tappy/package.py create mode 100644 var/spack/packages/py-urwid/package.py diff --git a/var/spack/packages/libedit/package.py b/var/spack/packages/libedit/package.py new file mode 100644 index 00000000000..bcd5212b9ea --- /dev/null +++ b/var/spack/packages/libedit/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Libedit(Package): + """An autotools compatible port of the NetBSD editline library""" + homepage = "http://thrysoee.dk/editline/" + url = "http://thrysoee.dk/editline/libedit-20150325-3.1.tar.gz" + + version('3.1', '43cdb5df3061d78b5e9d59109871b4f6', url="http://thrysoee.dk/editline/libedit-20150325-3.1.tar.gz") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/packages/ninja/package.py b/var/spack/packages/ninja/package.py new file mode 100644 index 00000000000..9e6bf4e358d --- /dev/null +++ b/var/spack/packages/ninja/package.py @@ -0,0 +1,22 @@ +from spack import * +import os + +class Ninja(Package): + """ A small, fast Make alternative """ + homepage = "https://martine.github.io/ninja/" + url = "https://github.com/martine/ninja/archive/v1.6.0.tar.gz" + + version('1.6.0', '254133059f2da79d8727f654d7198f43') + + extends('python') + + def install(self, spec, prefix): + sh = which('sh') + python('configure.py', '--bootstrap') + + cp = which('cp') + + bindir = os.path.join(prefix, 'bin') + mkdir(bindir) + cp('-a', '-t', bindir, 'ninja') + cp('-ra', 'misc', prefix) diff --git a/var/spack/packages/py-blessings/package.py b/var/spack/packages/py-blessings/package.py new file mode 100644 index 00000000000..f2475a0efd8 --- /dev/null +++ b/var/spack/packages/py-blessings/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyBlessings(Package): + """A nicer, kinder way to write to the terminal """ + homepage = "https://github.com/erikrose/blessings" + url = "https://pypi.python.org/packages/source/b/blessings/blessings-1.6.tar.gz" + + version('1.6', '4f552a8ebcd4982693c92571beb99394') + + depends_on('py-setuptools') + + extends("python") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-coverage/package.py b/var/spack/packages/py-coverage/package.py new file mode 100644 index 00000000000..39b2ac3b016 --- /dev/null +++ b/var/spack/packages/py-coverage/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyCoverage(Package): + """ Testing coverage checker for python """ + # FIXME: add a proper url for your package's homepage here. + homepage = "http://nedbatchelder.com/code/coverage/" + url = "https://pypi.python.org/packages/source/c/coverage/coverage-4.0a6.tar.gz" + + version('4.0a6', '1bb4058062646148965bef0796b61efc') + + depends_on('py-setuptools') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-tappy/package.py b/var/spack/packages/py-tappy/package.py new file mode 100644 index 00000000000..df61a909da0 --- /dev/null +++ b/var/spack/packages/py-tappy/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyTappy(Package): + """Python TAP interface module for unit tests""" + homepage = "https://github.com/mblayman/tappy" + # base https://pypi.python.org/pypi/cffi + url = "https://pypi.python.org/packages/source/t/tap.py/tap.py-1.6.tar.gz" + + version('1.6', 'c8bdb93ad66e05f939905172a301bedf') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-urwid/package.py b/var/spack/packages/py-urwid/package.py new file mode 100644 index 00000000000..aaa11c681d1 --- /dev/null +++ b/var/spack/packages/py-urwid/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyUrwid(Package): + """A full-featured console UI library""" + homepage = "http://urwid.org/" + url = "https://pypi.python.org/packages/source/u/urwid/urwid-1.3.0.tar.gz" + + version('1.3.0', 'a989acd54f4ff1a554add464803a9175') + + depends_on('py-setuptools') + + extends("python") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + From 01f811d5469c8cc059dae7846aa660aa0c5a4ab7 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 11:18:13 -0800 Subject: [PATCH 21/23] legacy mysql bindings for python --- var/spack/packages/py-mysqldb1/package.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 var/spack/packages/py-mysqldb1/package.py diff --git a/var/spack/packages/py-mysqldb1/package.py b/var/spack/packages/py-mysqldb1/package.py new file mode 100644 index 00000000000..fda02b49822 --- /dev/null +++ b/var/spack/packages/py-mysqldb1/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyMysqldb1(Package): + """Legacy mysql bindings for python""" + homepage = "https://github.com/farcepest/MySQLdb1" + url = "https://github.com/farcepest/MySQLdb1/archive/MySQLdb-1.2.5.tar.gz" + + version('1.2.5', '332c8f4955b6bc0c79ea15170bf7321b') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + From 03537a6be414f92114e240435b6ecbff200d8717 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 1 Jan 2016 17:38:42 -0500 Subject: [PATCH 22/23] Support Darwin for Boost Ensure that builds on Darwin use the "darwin" toolset. Also update Boost to 1.60.0. --- var/spack/packages/boost/package.py | 60 +++++++++++++++++------------ 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index 81dadbbf611..9b7c22c53d9 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -14,6 +14,7 @@ class Boost(Package): list_url = "http://sourceforge.net/projects/boost/files/boost/" list_depth = 2 + version('1.60.0', '65a840e1a0b13a558ff19eeb2c4f0cbe') version('1.59.0', '6aa9a5c6a4ca1016edd0ed1178e3cb87') version('1.58.0', 'b8839650e61e9c1c0a89f371dd475546') version('1.57.0', '1be49befbdd9a5ce9def2983ba3e7b76') @@ -48,11 +49,11 @@ class Boost(Package): variant('mpi', default=False, description='Activate the component Boost.MPI') variant('compression', default=True, description='Activate the compression Boost.iostreams') - depends_on('mpi', when='+mpi') depends_on('python', when='+python') - depends_on('zlib', when='+compression') + depends_on('mpi', when='+mpi') depends_on('bzip2', when='+compression') - + depends_on('zlib', when='+compression') + def url_for_version(self, version): """Handle Boost's weird URLs, which write the version two different ways.""" parts = [str(p) for p in Version(version)] @@ -61,20 +62,23 @@ def url_for_version(self, version): return "http://downloads.sourceforge.net/project/boost/boost/%s/boost_%s.tar.bz2" % ( dots, underscores) - def determine_toolset(self): - toolsets = {'gcc': 'gcc', + def determine_toolset(self, spec): + if spec.satisfies("=darwin-x86_64"): + return 'darwin' + + toolsets = {'g++': 'gcc', 'icpc': 'intel', 'clang++': 'clang'} for cc, toolset in toolsets.iteritems(): - if(cc in self.compiler.cxx_names): + if cc in self.compiler.cxx_names: return toolset # fallback to gcc if no toolset found return 'gcc' def determine_bootstrap_options(self, spec, options): - options.append('--with-toolset=%s' % self.determine_toolset()) + options.append('--with-toolset=%s' % self.determine_toolset(spec)) without_libs = [] if '~mpi' in spec: @@ -82,17 +86,20 @@ def determine_bootstrap_options(self, spec, options): if '~python' in spec: without_libs.append('python') else: - options.append('--with-python=%s' % (spec['python'].prefix.bin + '/python')) + options.append('--with-python=%s' % + join_path(spec['python'].prefix.bin, 'python')) if without_libs: options.append('--without-libraries=%s' % ','.join(without_libs)) with open('user-config.jam', 'w') as f: if '+mpi' in spec: - f.write('using mpi : %s ;\n' % (spec['mpi'].prefix.bin + '/mpicxx')) + f.write('using mpi : %s ;\n' % + joinpath(spec['mpi'].prefix.bin, 'mpicxx')) if '+python' in spec: - f.write('using python : %s : %s ;\n' % (spec['python'].version, - (spec['python'].prefix.bin + '/python'))) + f.write('using python : %s : %s ;\n' % + (spec['python'].version, + joinpath(spec['python'].prefix.bin, 'python'))) def determine_b2_options(self, spec, options): if '+debug' in spec: @@ -101,22 +108,26 @@ def determine_b2_options(self, spec, options): options.append('variant=release') if '~compression' in spec: - options.extend(['-s NO_BZIP2=1', - '-s NO_ZLIB=1', - ]) + options.extend([ + '-s', 'NO_BZIP2=1', + '-s', 'NO_ZLIB=1']) if '+compression' in spec: - options.extend(['-s BZIP2_INCLUDE=%s' % spec['bzip2'].prefix.include, - '-s BZIP2_LIBPATH=%s' % spec['bzip2'].prefix.lib, - '-s ZLIB_INCLUDE=%s' % spec['zlib'].prefix.include, - '-s ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib]) + options.extend([ + '-s', 'BZIP2_INCLUDE=%s' % spec['bzip2'].prefix.include, + '-s', 'BZIP2_LIBPATH=%s' % spec['bzip2'].prefix.lib, + '-s', 'ZLIB_INCLUDE=%s' % spec['zlib'].prefix.include, + '-s', 'ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib, + ]) - options.extend(['toolset=%s' % self.determine_toolset(), - 'link=static,shared', - '--layout=tagged']) + options.extend([ + 'toolset=%s' % self.determine_toolset(spec), + 'link=static,shared', + 'threading=single,multi', + '--layout=tagged']) def install(self, spec, prefix): - # to make him find the user-config.jam + # to make Boost find the user-config.jam env['BOOST_BUILD_PATH'] = './' bootstrap = Executable('./bootstrap.sh') @@ -130,9 +141,8 @@ def install(self, spec, prefix): b2name = './b2' if spec.satisfies('@1.47:') else './bjam' b2 = Executable(b2name) - b2_options = ['-j %s' % make_jobs] + b2_options = ['-j', '%s' % make_jobs] self.determine_b2_options(spec, b2_options) - b2('install', 'threading=single', *b2_options) - b2('install', 'threading=multi', *b2_options) + b2('install', *b2_options) From 96e5905bce302007e8be6e3debf6d3efdf77a8cd Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 4 Jan 2016 12:14:42 -0500 Subject: [PATCH 23/23] Build multiple FFTW variants simultaneously FFTW can build only one floating point precision (float, double, long double, quad) at once, but they can all be installed simultaneously as the libraries have different names. It is common packages to decide only at run time which precision FFTW they need, and thus FFTW should offer all precisions at once. --- var/spack/packages/fftw/package.py | 55 +++++++++--------------------- 1 file changed, 17 insertions(+), 38 deletions(-) diff --git a/var/spack/packages/fftw/package.py b/var/spack/packages/fftw/package.py index 5f71762c4ff..4d2b9642426 100644 --- a/var/spack/packages/fftw/package.py +++ b/var/spack/packages/fftw/package.py @@ -39,54 +39,21 @@ class Fftw(Package): version('3.3.4', '2edab8c06b24feeb3b82bbb3ebf3e7b3') - ########## - # Floating point precision - FLOAT = 'float' - LONG_DOUBLE = 'long_double' - QUAD_PRECISION = 'quad' - PRECISION_OPTIONS = { - FLOAT: '--enable-float', - LONG_DOUBLE: '--enable--long-double', - QUAD_PRECISION: '--enable-quad-precision' - } - variant(FLOAT, default=False, description='Produces a single precision version of the library') - variant(LONG_DOUBLE, default=False, description='Produces a long double precision version of the library') - variant(QUAD_PRECISION, default=False, description='Produces a quad precision version of the library (works only with GCC and libquadmath)') - ########## + variant('float', default=True, description='Produces a single precision version of the library') + variant('long_double', default=True, description='Produces a long double precision version of the library') + variant('quad', default=False, description='Produces a quad precision version of the library (works only with GCC and libquadmath)') variant('mpi', default=False, description='Activate MPI support') depends_on('mpi', when='+mpi') - @staticmethod - def enabled(x): - """ - Given a variant name returns the string that means the variant is enabled - - :param x: variant name - """ - # FIXME : duplicated from MVAPICH2 - return '+' + x - - def check_fortran_availability(self, options): - if not self.compiler.f77 or not self.compiler.fc: - options.append("--disable-fortran") - - def set_floating_point_precision(self, spec, options): - l = [option for variant, option in Fftw.PRECISION_OPTIONS.iteritems() if self.enabled(variant) in spec] - if len(l) > 1: - raise RuntimeError('At most one floating point precision variant may activated per build.') - options.extend(l) - def install(self, spec, prefix): - options = ['--prefix=%s' % prefix, '--enable-shared', '--enable-threads', '--enable-openmp'] - self.check_fortran_availability(options) - self.set_floating_point_precision(spec, options) - + if not self.compiler.f77 or not self.compiler.fc: + options.append("--disable-fortran") if '+mpi' in spec: options.append('--enable-mpi') @@ -94,3 +61,15 @@ def install(self, spec, prefix): make() make("install") + if '+float' in spec: + configure('--enable-float', *options) + make() + make("install") + if '+long_double' in spec: + configure('--enable-long-double', *options) + make() + make("install") + if '+quad' in spec: + configure('--enable-quad-precision', *options) + make() + make("install")