From 2220784eda80546415e39c783785f2435986a08a Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 26 May 2016 15:22:17 -0500 Subject: [PATCH 01/14] Add scons support, .zip support, and Cantera package --- lib/spack/spack/build_environment.py | 3 +- lib/spack/spack/cmd/create.py | 160 ++++++++++++------ .../repos/builtin/packages/cantera/package.py | 149 ++++++++++++++++ .../repos/builtin/packages/py-3to2/package.py | 40 +++++ .../builtin/packages/py-unittest2/package.py | 41 +++++ .../packages/py-unittest2py3k/package.py | 42 +++++ .../repos/builtin/packages/serf/package.py | 2 - .../builtin/packages/sundials/package.py | 105 +++++++++++- .../builtin/packages/superlu-mt/package.py | 135 +++++++++++++++ 9 files changed, 615 insertions(+), 62 deletions(-) create mode 100644 var/spack/repos/builtin/packages/cantera/package.py create mode 100644 var/spack/repos/builtin/packages/py-3to2/package.py create mode 100644 var/spack/repos/builtin/packages/py-unittest2/package.py create mode 100644 var/spack/repos/builtin/packages/py-unittest2py3k/package.py create mode 100644 var/spack/repos/builtin/packages/superlu-mt/package.py diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 7c65091d49c..c72dc1a4dd3 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -290,7 +290,7 @@ def set_module_variables_for_package(pkg, module): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. """ - # number of jobs spack will to build with. + # number of jobs spack will build with. jobs = multiprocessing.cpu_count() if not pkg.parallel: jobs = 1 @@ -303,6 +303,7 @@ def set_module_variables_for_package(pkg, module): # TODO: make these build deps that can be installed if not found. m.make = MakeExecutable('make', jobs) m.gmake = MakeExecutable('gmake', jobs) + m.scons = MakeExecutable('scons', jobs) # easy shortcut to os.environ m.env = os.environ diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 41bfa741f60..fa7ffb39230 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -1,4 +1,3 @@ -_copyright = """\ ############################################################################## # Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. @@ -23,10 +22,8 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -""" import string import os -import hashlib import re from ordereddict_backport import OrderedDict @@ -41,16 +38,37 @@ from spack.spec import Spec from spack.util.naming import * from spack.repository import Repo, RepoError -import spack.util.crypto as crypto from spack.util.executable import which -from spack.stage import Stage description = "Create a new package file from an archive URL" -package_template = string.Template( - _copyright + """ +package_template = string.Template("""\ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## # # This is a template package file for Spack. We've put "FIXME" # next to all the things you'll want to change. Once you've handled @@ -68,8 +86,10 @@ # from spack import * + class ${class_name}(Package): ""\"FIXME: put a proper description of your package here.""\" + # FIXME: add a proper url for your package's homepage here. homepage = "http://www.example.com" url = "${url}" @@ -80,12 +100,10 @@ class ${class_name}(Package): # depends_on("foo") def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. + # FIXME: Modify the installation instructions here ${configure} - - # FIXME: Add logic to build and install here - make() - make("install") + ${build} + ${install} """) @@ -120,39 +138,78 @@ def setup_parser(subparser): class ConfigureGuesser(object): def __call__(self, stage): - """Try to guess the type of build system used by the project, and return - an appropriate configure line. - """ - autotools = "configure('--prefix=%s' % prefix)" - cmake = "cmake('.', *std_cmake_args)" - python = "python('setup.py', 'install', '--prefix=%s' % prefix)" - r = "R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file)" + """Try to guess the type of build system used by the project. Set the + appropriate default configure, build, and install instructions.""" - config_lines = ((r'/configure$', 'autotools', autotools), - (r'/CMakeLists.txt$', 'cmake', cmake), - (r'/setup.py$', 'python', python), - (r'/NAMESPACE$', 'r', r)) + # Default configure instructions + configureDict = { + 'autotools': "configure('--prefix={0}'.format(prefix))", + 'cmake': "cmake('.', *std_cmake_args)", + 'scons': "", + 'python': "", + 'r': "", + 'unknown': "# FIXME: Unknown build system" + } - # Peek inside the tarball. - tar = which('tar') - output = tar( - "--exclude=*/*/*", "-tf", stage.archive_file, output=str) - lines = output.split("\n") + # Default build instructions + buildDict = { + 'autotools': "make()", + 'cmake': "make()", + 'scons': "scons('prefix={0}'.format(prefix))", + 'python': "", + 'r': "", + 'unknown': "make()", + } - # Set the configure line to the one that matched. - for pattern, bs, cl in config_lines: + # Default install instructions + installDict = { + 'autotools': "make('install')", + 'cmake': "make('install')", + 'scons': "scons('install')", + 'python': "python('setup.py', 'install', " + + "'--prefix={0}'.format(prefix))", + 'r': "R('CMD', 'INSTALL', '--library={0}'.format(" + + "self.module.r_lib_dir), self.stage.archive_file)", + 'unknown': "make('install')", + } + + # A list of clues that give us an idea of the build system a package + # uses. If the regular expression matches a file contained in the + # archive, the corresponding build system is assumed. + clues = [ + (r'/configure$', 'autotools'), + (r'/CMakeLists.txt$', 'cmake'), + (r'/SConstruct$', 'scons'), + (r'/setup.py$', 'python'), + (r'/NAMESPACE$', 'r') + ] + + # Peek inside the compressed file. + output = '' + if stage.archive_file.endswith(('.tar', '.tar.gz', '.tar.bz2', + '.tgz', '.tbz2')): + tar = which('tar') + output = tar('--exclude=*/*/*', '-tf', + stage.archive_file, output=str) + elif stage.archive_file.endswith('.gz'): + gunzip = which('gunzip') + output = gunzip('-l', stage.archive_file, output=str) + elif stage.archive_file.endswith('.zip'): + unzip = which('unzip') + output = unzip('-l', stage.archive_file, output=str) + lines = output.split('\n') + + # Determine the build system based on the files contained + # in the archive. + build_system = 'unknown' + for pattern, bs in clues: if any(re.search(pattern, l) for l in lines): - config_line = cl build_system = bs - break - else: - # None matched -- just put both, with cmake commented out - config_line = "# FIXME: Spack couldn't guess one, so here are some options:\n" - config_line += " # " + autotools + "\n" - config_line += " # " + cmake - build_system = 'unknown' - self.configure = config_line + self.configure = configureDict[build_system] + self.build = buildDict[build_system] + self.install = installDict[build_system] + self.build_system = build_system @@ -168,7 +225,7 @@ def guess_name_and_version(url, args): else: try: name = spack.url.parse_name(url, version) - except spack.url.UndetectableNameError, e: + except spack.url.UndetectableNameError: # Use a user-supplied name if one is present tty.die("Couldn't guess a name for this package. Try running:", "", "spack create --name ") @@ -182,7 +239,8 @@ def guess_name_and_version(url, args): def find_repository(spec, args): # figure out namespace for spec if spec.namespace and args.namespace and spec.namespace != args.namespace: - tty.die("Namespaces '%s' and '%s' do not match." % (spec.namespace, args.namespace)) + tty.die("Namespaces '%s' and '%s' do not match." % (spec.namespace, + args.namespace)) if not spec.namespace and args.namespace: spec.namespace = args.namespace @@ -193,8 +251,8 @@ def find_repository(spec, args): try: repo = Repo(repo_path) if spec.namespace and spec.namespace != repo.namespace: - tty.die("Can't create package with namespace %s in repo with namespace %s" - % (spec.namespace, repo.namespace)) + tty.die("Can't create package with namespace %s in repo with " + "namespace %s" % (spec.namespace, repo.namespace)) except RepoError as e: tty.die(str(e)) else: @@ -214,11 +272,7 @@ def find_repository(spec, args): def fetch_tarballs(url, name, version): """Try to find versions of the supplied archive by scraping the web. - - Prompts the user to select how many to download if many are found. - - - """ + Prompts the user to select how many to download if many are found.""" versions = spack.util.web.find_versions_of_archive(url) rkeys = sorted(versions.keys(), reverse=True) versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys))) @@ -226,11 +280,11 @@ def fetch_tarballs(url, name, version): archives_to_fetch = 1 if not versions: # If the fetch failed for some reason, revert to what the user provided - versions = { version : url } + versions = {version: url} elif len(versions) > 1: tty.msg("Found %s versions of %s:" % (len(versions), name), *spack.cmd.elide_list( - ["%-10s%s" % (v,u) for v, u in versions.iteritems()])) + ["%-10s%s" % (v, u) for v, u in versions.iteritems()])) print archives_to_fetch = tty.get_number( "Include how many checksums in the package file?", @@ -292,10 +346,12 @@ def create(parser, args): pkg_file.write( package_template.substitute( name=name, - configure=guesser.configure, class_name=mod_to_class(name), url=url, - versions=make_version_calls(ver_hash_tuples))) + versions=make_version_calls(ver_hash_tuples), + configure=guesser.configure, + build=guesser.build, + install=guesser.install)) # If everything checks out, go ahead and edit. spack.editor(pkg_path) diff --git a/var/spack/repos/builtin/packages/cantera/package.py b/var/spack/repos/builtin/packages/cantera/package.py new file mode 100644 index 00000000000..a55f8835601 --- /dev/null +++ b/var/spack/repos/builtin/packages/cantera/package.py @@ -0,0 +1,149 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +import spack +import os + + +class Cantera(Package): + """Cantera is a suite of object-oriented software tools for problems + involving chemical kinetics, thermodynamics, and/or transport processes.""" + + homepage = "http://www.cantera.org/docs/sphinx/html/index.html" + url = "https://github.com/Cantera/cantera/archive/v2.2.1.tar.gz" + + version('2.2.1', '9d1919bdef39ddec54485fc8a741a3aa') + + variant('lapack', default=True, description='Build with external BLAS/LAPACK libraries') + variant('threadsafe', default=True, description='Build threadsafe, requires Boost') + variant('sundials', default=True, description='Build with external Sundials') + variant('python', default=False, description='Build the Cantera Python module') + variant('matlab', default=False, description='Build the Cantera Matlab toolbox') + + # Required dependencies + depends_on('scons') + + # Recommended dependencies + depends_on('blas', when='+lapack') + depends_on('lapack', when='+lapack') + depends_on('boost', when='+threadsafe') + depends_on('sundials', when='+sundials') # must be compiled with -fPIC + + # Python module dependencies + extends('python', when='+python') + depends_on('py-numpy', when='+python') + depends_on('py-scipy', when='+python') + depends_on('py-cython', when='+python') + depends_on('py-3to2', when='+python') + # TODO: these "when" specs don't actually work + #depends_on('py-unittest2', when='+python^python@2.6') + #depends_on('py-unittest2py3k', when='+python^python@3.1') + + # Matlab toolbox dependencies + # TODO: add Matlab package + # TODO: allow packages to extend multiple other packages + #extends('matlab', when='+matlab') + + def install(self, spec, prefix): + # Required options + options = [ + 'prefix={0}'.format(prefix), + 'CC={0}'.format(os.environ['CC']), + 'CXX={0}'.format(os.environ['CXX']), + 'F77={0}'.format(os.environ['F77']), + 'FORTRAN={0}'.format(os.environ['FC']), + 'cc_flags=-fPIC', + # Allow Spack environment variables to propagate through to SCons + 'env_vars=all' + ] + + # BLAS/LAPACK support + if '+lapack' in spec: + options.extend([ + 'blas_lapack_libs=lapack,blas', + 'blas_lapack_dir={0}'.format(spec['lapack'].prefix.lib) + ]) + + # Threadsafe build, requires Boost + if '+threadsafe' in spec: + options.extend([ + 'build_thread_safe=yes', + 'boost_inc_dir={0}'.format(spec['boost'].prefix.include), + 'boost_lib_dir={0}'.format(spec['boost'].prefix.lib), + 'boost_thread_lib=boost_thread-mt' + ]) + else: + options.append('build_thread_safe=no') + + # Sundials support + if '+sundials' in spec: + options.extend([ + 'use_sundials=y', + 'sundials_include={0}'.format(spec['sundials'].prefix.include), + 'sundials_libdir={0}'.format(spec['sundials'].prefix.lib), + 'sundials_license={0}'.format( + join_path(spec['sundials'].prefix, 'LICENSE')) + ]) + else: + options.append('use_sundials=n') + + # Python module + if '+python' in spec: + options.extend([ + 'python_package=full', + 'python_cmd={0}'.format( + join_path(spec['python'].prefix.bin, 'python')), + 'python_array_home={0}'.format(spec['py-numpy'].prefix) + ]) + if spec['python'].satisfies('@3'): + options.extend([ + 'python3_package=y', + 'python3_cmd={0}'.format( + join_path(spec['python'].prefix.bin, 'python')), + 'python3_array_home={0}'.format(spec['py-numpy'].prefix) + ]) + else: + options.append('python3_package=n') + else: + options.append('python_package=none') + options.append('python3_package=n') + + # Matlab toolbox + if '+matlab' in spec: + options.extend([ + 'matlab_toolbox=y', + 'matlab_path={0}'.format(spec['matlab'].prefix) + ]) + else: + options.append('matlab_toolbox=n') + + scons('build', *options) + + if '+python' in spec: + # Tests will always fail if Python dependencies aren't built + #scons('test') # TODO: 3 expected failures, not sure what's wrong + pass + + scons('install') diff --git a/var/spack/repos/builtin/packages/py-3to2/package.py b/var/spack/repos/builtin/packages/py-3to2/package.py new file mode 100644 index 00000000000..577662e5833 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-3to2/package.py @@ -0,0 +1,40 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Py3to2(Package): + """lib3to2 is a set of fixers that are intended to backport code written + for Python version 3.x into Python version 2.x.""" + + homepage = "https://pypi.python.org/pypi/3to2" + url = "https://pypi.python.org/packages/8f/ab/58a363eca982c40e9ee5a7ca439e8ffc5243dde2ae660ba1ffdd4868026b/3to2-1.1.1.zip" + + version('1.1.1', 'cbeed28e350dbdaef86111ace3052824') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-unittest2/package.py b/var/spack/repos/builtin/packages/py-unittest2/package.py new file mode 100644 index 00000000000..c9b0ad5cdf5 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-unittest2/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyUnittest2(Package): + """unittest2 is a backport of the new features added to the unittest + testing framework in Python 2.7 and onwards.""" + + homepage = "https://pypi.python.org/pypi/unittest2" + url = "https://pypi.python.org/packages/7f/c4/2b0e2d185d9d60772c10350d9853646832609d2f299a8300ab730f199db4/unittest2-1.1.0.tar.gz" + + version('1.1.0', 'f72dae5d44f091df36b6b513305ea000') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-unittest2py3k/package.py b/var/spack/repos/builtin/packages/py-unittest2py3k/package.py new file mode 100644 index 00000000000..b2c7510d943 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-unittest2py3k/package.py @@ -0,0 +1,42 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyUnittest2py3k(Package): + """unittest2 is a backport of the new features added to the unittest + testing framework in Python 2.7 and 3.2. This is a Python 3 compatible + version of unittest2.""" + + homepage = "https://pypi.python.org/pypi/unittest2py3k" + url = "https://pypi.python.org/packages/4e/3d/d44421e8d828af1399c1509c196db92e2a58f3764b01a0ee928d7025d1ca/unittest2py3k-0.5.1.tar.gz" + + version('0.5.1', '8824ff92044310d9365f90d892bf0f09') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/serf/package.py b/var/spack/repos/builtin/packages/serf/package.py index 817db682413..074d3d64b21 100644 --- a/var/spack/repos/builtin/packages/serf/package.py +++ b/var/spack/repos/builtin/packages/serf/package.py @@ -41,8 +41,6 @@ class Serf(Package): depends_on('zlib') def install(self, spec, prefix): - scons = which("scons") - options = ['PREFIX=%s' % prefix] options.append('APR=%s' % spec['apr'].prefix) options.append('APU=%s' % spec['apr-util'].prefix) diff --git a/var/spack/repos/builtin/packages/sundials/package.py b/var/spack/repos/builtin/packages/sundials/package.py index ba2340f74c6..a31cf5fde4f 100644 --- a/var/spack/repos/builtin/packages/sundials/package.py +++ b/var/spack/repos/builtin/packages/sundials/package.py @@ -24,16 +24,107 @@ ############################################################################## from spack import * + class Sundials(Package): - """SUNDIALS (SUite of Nonlinear and DIfferential/ALgebraic equation Solvers)""" + """SUNDIALS (SUite of Nonlinear and DIfferential/ALgebraic equation + Solvers)""" + homepage = "http://computation.llnl.gov/casc/sundials/" - url = "http://computation.llnl.gov/casc/sundials/download/code/sundials-2.5.0.tar.gz" + url = "http://computation.llnl.gov/projects/sundials-suite-nonlinear-differential-algebraic-equation-solvers/download/sundials-2.6.2.tar.gz" - version('2.5.0', 'aba8b56eec600de3109cfb967aa3ba0f') + version('2.6.2', '3deeb0ede9f514184c6bd83ecab77d95') - depends_on("mpi") + variant('mpi', default=True, description='Enable MPI support') + variant('lapack', default=True, description='Build with external BLAS/LAPACK libraries') + variant('klu', default=True, description='Build with SuiteSparse KLU libraries') + variant('superlu', default=True, description='Build with SuperLU_MT libraries') + variant('openmp', default=False, description='Enable OpenMP support') + variant('pthread', default=True, description='Enable POSIX threads support') + + depends_on('mpi', when='+mpi') + depends_on('blas', when='+lapack') + depends_on('lapack', when='+lapack') + depends_on('suite-sparse', when='+klu') + depends_on('superlu-mt+openmp', when='+superlu+openmp') + depends_on('superlu-mt+pthread', when='+superlu+pthread') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") + cmake_args = std_cmake_args + cmake_args.extend([ + '-DBUILD_SHARED_LIBS=ON', + '-DCMAKE_C_FLAGS=-fPIC' + ]) + + # MPI support + if '+mpi' in spec: + cmake_args.extend([ + '-DMPI_ENABLE=ON', + '-DMPI_MPICC={0}'.format(spec['mpi'].mpicc), + '-DMPI_MPIF77={0}'.format(spec['mpi'].mpif77) + ]) + else: + cmake_args.append('-DMPI_ENABLE=OFF') + + # Building with LAPACK and BLAS + if '+lapack' in spec: + cmake_args.extend([ + '-DLAPACK_ENABLE=ON', + '-DLAPACK_LIBRARIES={0};{1}'.format( + spec['lapack'].lapack_shared_lib, + spec['blas'].blas_shared_lib + ) + ]) + else: + cmake_args.append('-DLAPACK_ENABLE=OFF') + + # Building with KLU + if '+klu' in spec: + cmake_args.extend([ + '-DKLU_ENABLE=ON', + '-DKLU_INCLUDE_DIR={0}'.format( + spec['suite-sparse'].prefix.include), + '-DKLU_LIBRARY_DIR={0}'.format( + spec['suite-sparse'].prefix.lib) + ]) + else: + cmake_args.append('-DKLU_ENABLE=OFF') + + # Building with SuperLU_MT + if '+superlu' in spec: + cmake_args.extend([ + '-DSUPERLUMT_ENABLE=ON', + '-DSUPERLUMT_INCLUDE_DIR={0}'.format( + spec['superlu-mt'].prefix.include), + '-DSUPERLUMT_LIBRARY_DIR={0}'.format( + spec['superlu-mt'].prefix.lib) + ]) + if '+openmp' in spec: + cmake_args.append('-DSUPERLUMT_THREAD_TYPE=OpenMP') + elif '+pthread' in spec: + cmake_args.append('-DSUPERLUMT_THREAD_TYPE=Pthread') + else: + msg = 'You must choose either +openmp or +pthread when ' + msg += 'building with SuperLU_MT' + raise RuntimeError(msg) + else: + cmake_args.append('-DSUPERLUMT_ENABLE=OFF') + + # OpenMP support + if '+openmp' in spec: + cmake_args.append('-DOPENMP_ENABLE=ON') + else: + cmake_args.append('-DOPENMP_ENABLE=OFF') + + # POSIX threads support + if '+pthread' in spec: + cmake_args.append('-DPTHREAD_ENABLE=ON') + else: + cmake_args.append('-DPTHREAD_ENABLE=OFF') + + with working_dir('build', create=True): + cmake('..', *cmake_args) + + make() + make('install') + + install('LICENSE', prefix) diff --git a/var/spack/repos/builtin/packages/superlu-mt/package.py b/var/spack/repos/builtin/packages/superlu-mt/package.py new file mode 100644 index 00000000000..5eee4bf5d93 --- /dev/null +++ b/var/spack/repos/builtin/packages/superlu-mt/package.py @@ -0,0 +1,135 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +import spack +import glob +import os + + +class SuperluMt(Package): + """SuperLU is a general purpose library for the direct solution of large, + sparse, nonsymmetric systems of linear equations on high performance + machines. SuperLU_MT is designed for shared memory parallel machines.""" + + homepage = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/#superlu_mt" + url = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_mt_3.1.tar.gz" + + version('3.1', '06ac62f1b4b7d17123fffa0d0c315e91') + + variant('blas', default=True, description='Build with external BLAS library') + + # Must choose one or the other + variant('openmp', default=False, description='Build with OpenMP support') + variant('pthread', default=True, description='Build with POSIX threads support') + + # NOTE: must link with a single-threaded BLAS library + depends_on('blas', when='+blas') + + # Cannot be built in parallel + parallel = False + + def configure(self, spec): + # Validate chosen variants + if '+openmp' in spec and '+pthread' in spec: + msg = 'You cannot choose both +openmp and +pthread' + raise RuntimeError(msg) + if '~openmp' in spec and '~pthread' in spec: + msg = 'You must choose either +openmp or +pthread' + raise RuntimeError(msg) + + # List of configuration options + config = [] + + # The machine (platform) identifier to append to the library names + if '+openmp' in spec: + # OpenMP + config.extend([ + 'PLAT = _OPENMP', + 'TMGLIB = libtmglib.a', + 'MPLIB = {0}'.format(self.compiler.openmp_flag), + 'CFLAGS = {0}'.format(self.compiler.openmp_flag), + 'FFLAGS = {0}'.format(self.compiler.openmp_flag) + ]) + elif '+pthread' in spec: + # POSIX threads + config.extend([ + 'PLAT = _PTHREAD', + 'TMGLIB = libtmglib$(PLAT).a', + 'MPLIB = -lpthread' + ]) + + # The BLAS library + # NOTE: must link with a single-threaded BLAS library + if '+blas' in spec: + config.extend([ + 'BLASDEF = -DUSE_VENDOR_BLAS', + 'BLASLIB = -L{0} -lblas'.format(spec['blas'].prefix.lib) + ]) + else: + config.append('BLASLIB = ../lib/libblas$(PLAT).a') + + # Generic options + config.extend([ + # The name of the libraries to be created/linked to + 'SUPERLULIB = libsuperlu_mt$(PLAT).a', + 'MATHLIB = -lm', + # The archiver and the flag(s) to use when building archives + 'ARCH = ar', + 'ARCHFLAGS = cr', + 'RANLIB = {0}'.format('ranlib' if which('ranlib') else 'echo'), + # Definitions used by CPP + 'PREDEFS = -D_$(PLAT)', + # Compilers and flags + 'CC = {0}'.format(os.environ['CC']), + 'CFLAGS += $(PREDEFS) -D_LONGINT', + 'NOOPTS = -O0', + 'FORTRAN = {0}'.format(os.environ['FC']), + 'LOADER = {0}'.format(os.environ['CC']), + # C preprocessor defs for compilation + 'CDEFS = -DAdd_' + ]) + + # Write configuration options to include file + with open('make.inc', 'w') as inc: + for option in config: + inc.write('{0}\n'.format(option)) + + def install(self, spec, prefix): + # Set up make include file manually + self.configure(spec) + + # BLAS needs to be compiled separately if using internal BLAS library + if '+blas' not in spec: + make('blaslib') + + make() + + # Install manually + install_tree('lib', prefix.lib) + + headers = glob.glob(join_path('SRC', '*.h')) + mkdir(prefix.include) + for h in headers: + install(h, prefix.include) From a21e845ce7e4b7050a3ef28149b349ff0a4ec58e Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 26 May 2016 16:19:23 -0500 Subject: [PATCH 02/14] Flake8 --- lib/spack/spack/build_environment.py | 30 +++++++++++-------- .../repos/builtin/packages/cantera/package.py | 9 +++--- .../repos/builtin/packages/serf/package.py | 3 +- .../builtin/packages/superlu-mt/package.py | 1 - 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index c72dc1a4dd3..9e06f9f9ccc 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -63,7 +63,7 @@ import spack from spack.environment import EnvironmentModifications, validate from spack.util.environment import * -from spack.util.executable import Executable, which +from spack.util.executable import Executable # # This can be set by the user to globally disable parallel builds. @@ -88,7 +88,6 @@ dso_suffix = 'dylib' if sys.platform == 'darwin' else 'so' - class MakeExecutable(Executable): """Special callable executable object for make so the user can specify parallel or not on a per-invocation basis. Using @@ -177,11 +176,13 @@ def set_compiler_environment_variables(pkg, env): flags = pkg.spec.compiler_flags # Set compiler variables used by CMake and autotools - assert all(key in compiler.link_paths for key in ('cc', 'cxx', 'f77', 'fc')) + assert all(key in compiler.link_paths for key in ( + 'cc', 'cxx', 'f77', 'fc')) # Populate an object with the list of environment modifications # and return it - # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc. + # TODO : add additional kwargs for better diagnostics, like requestor, + # ttyout, ttyerr, etc. link_dir = spack.build_env_path env.set('CC', join_path(link_dir, compiler.link_paths['cc'])) env.set('CXX', join_path(link_dir, compiler.link_paths['cxx'])) @@ -233,7 +234,8 @@ def set_build_environment_variables(pkg, env): # handled by putting one in the /case-insensitive # directory. Add that to the path too. env_paths = [] - for item in [spack.build_env_path, join_path(spack.build_env_path, pkg.compiler.name)]: + for item in [spack.build_env_path, join_path(spack.build_env_path, + pkg.compiler.name)]: env_paths.append(item) ci = join_path(item, 'case-insensitive') if os.path.isdir(ci): @@ -246,7 +248,8 @@ def set_build_environment_variables(pkg, env): # Prefixes of all of the package's dependencies go in SPACK_DEPENDENCIES dep_prefixes = [d.prefix for d in pkg.spec.traverse(root=False)] env.set_path(SPACK_DEPENDENCIES, dep_prefixes) - env.set_path('CMAKE_PREFIX_PATH', dep_prefixes) # Add dependencies to CMAKE_PREFIX_PATH + # Add dependencies to CMAKE_PREFIX_PATH + env.set_path('CMAKE_PREFIX_PATH', dep_prefixes) # Install prefix env.set(SPACK_PREFIX, pkg.prefix) @@ -262,7 +265,8 @@ def set_build_environment_variables(pkg, env): env.unset('DYLD_LIBRARY_PATH') # Add bin directories from dependencies to the PATH for the build. - bin_dirs = reversed(filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes])) + bin_dirs = reversed(filter(os.path.isdir, + ['%s/bin' % prefix for prefix in dep_prefixes])) for item in bin_dirs: env.prepend_path('PATH', item) @@ -326,7 +330,8 @@ def set_module_variables_for_package(pkg, module): # Set up CMake rpath m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE') - m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg))) + m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % \ + ":".join(get_rpaths(pkg))) # Put spack compiler paths in module scope. link_dir = spack.build_env_path @@ -373,13 +378,13 @@ def get_rpaths(pkg): def parent_class_modules(cls): - """Get list of super class modules that are all descend from spack.Package""" + """Get list of super class modules that all descend from spack.Package""" if not issubclass(cls, spack.Package) or issubclass(spack.Package, cls): return [] result = [] module = sys.modules.get(cls.__module__) if module: - result = [ module ] + result = [module] for c in cls.__bases__: result.extend(parent_class_modules(c)) return result @@ -411,7 +416,8 @@ def setup_package(pkg): # throwaway environment, but it is kind of dirty. # # TODO: Think about how to avoid this fix and do something cleaner. - for s in pkg.spec.traverse(): s.package.spec = s + for s in pkg.spec.traverse(): + s.package.spec = s set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env) @@ -499,7 +505,7 @@ def child_fun(): # message. Just make the parent exit with an error code. pid, returncode = os.waitpid(pid, 0) if returncode != 0: - raise InstallError("Installation process had nonzero exit code.".format(str(returncode))) + raise InstallError("Installation process had nonzero exit code.") class InstallError(spack.error.SpackError): diff --git a/var/spack/repos/builtin/packages/cantera/package.py b/var/spack/repos/builtin/packages/cantera/package.py index a55f8835601..ddaeb2b0706 100644 --- a/var/spack/repos/builtin/packages/cantera/package.py +++ b/var/spack/repos/builtin/packages/cantera/package.py @@ -23,7 +23,6 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import spack import os @@ -58,13 +57,13 @@ class Cantera(Package): depends_on('py-cython', when='+python') depends_on('py-3to2', when='+python') # TODO: these "when" specs don't actually work - #depends_on('py-unittest2', when='+python^python@2.6') - #depends_on('py-unittest2py3k', when='+python^python@3.1') + # depends_on('py-unittest2', when='+python^python@2.6') + # depends_on('py-unittest2py3k', when='+python^python@3.1') # Matlab toolbox dependencies # TODO: add Matlab package # TODO: allow packages to extend multiple other packages - #extends('matlab', when='+matlab') + # extends('matlab', when='+matlab') def install(self, spec, prefix): # Required options @@ -143,7 +142,7 @@ def install(self, spec, prefix): if '+python' in spec: # Tests will always fail if Python dependencies aren't built - #scons('test') # TODO: 3 expected failures, not sure what's wrong + # scons('test') # TODO: 3 expected failures, not sure what's wrong pass scons('install') diff --git a/var/spack/repos/builtin/packages/serf/package.py b/var/spack/repos/builtin/packages/serf/package.py index 074d3d64b21..ff6fd2da9b0 100644 --- a/var/spack/repos/builtin/packages/serf/package.py +++ b/var/spack/repos/builtin/packages/serf/package.py @@ -28,10 +28,11 @@ class Serf(Package): """Apache Serf - a high performance C-based HTTP client library built upon the Apache Portable Runtime (APR) library""" + homepage = 'https://serf.apache.org/' url = 'https://archive.apache.org/dist/serf/serf-1.3.8.tar.bz2' - version('1.3.8', '1d45425ca324336ce2f4ae7d7b4cfbc5567c5446') + version('1.3.8', '1d45425ca324336ce2f4ae7d7b4cfbc5567c5446') depends_on('apr') depends_on('apr-util') diff --git a/var/spack/repos/builtin/packages/superlu-mt/package.py b/var/spack/repos/builtin/packages/superlu-mt/package.py index 5eee4bf5d93..5a9429d6e5e 100644 --- a/var/spack/repos/builtin/packages/superlu-mt/package.py +++ b/var/spack/repos/builtin/packages/superlu-mt/package.py @@ -23,7 +23,6 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import spack import glob import os From 9500f2718b7ab9d430dbff2fc10bca1179c61b7b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 26 May 2016 16:33:35 -0500 Subject: [PATCH 03/14] More Flake8 --- lib/spack/spack/build_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 9e06f9f9ccc..901ba1608dc 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -330,7 +330,7 @@ def set_module_variables_for_package(pkg, module): # Set up CMake rpath m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE') - m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % \ + m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg))) # Put spack compiler paths in module scope. From 81ac3b62fc4d21a677bea654402f3684b7edbc20 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 27 May 2016 12:39:42 -0500 Subject: [PATCH 04/14] Filter compilers and link boost properly --- .../repos/builtin/packages/cantera/package.py | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/cantera/package.py b/var/spack/repos/builtin/packages/cantera/package.py index ddaeb2b0706..bc8c132ffc4 100644 --- a/var/spack/repos/builtin/packages/cantera/package.py +++ b/var/spack/repos/builtin/packages/cantera/package.py @@ -91,7 +91,7 @@ def install(self, spec, prefix): 'build_thread_safe=yes', 'boost_inc_dir={0}'.format(spec['boost'].prefix.include), 'boost_lib_dir={0}'.format(spec['boost'].prefix.lib), - 'boost_thread_lib=boost_thread-mt' + 'boost_thread_lib=boost_thread-mt,boost_system-mt' ]) else: options.append('build_thread_safe=no') @@ -142,7 +142,55 @@ def install(self, spec, prefix): if '+python' in spec: # Tests will always fail if Python dependencies aren't built - # scons('test') # TODO: 3 expected failures, not sure what's wrong - pass + scons('test', parallel=False) scons('install') + + self.filter_compilers() + + def filter_compilers(self): + """Run after install to tell the Makefile and SConstruct files to use + the compilers that Spack built the package with. + + If this isn't done, they'll have CC, CXX, F77, and FC set to Spack's + generic cc, c++, f77, and f90. We want them to be bound to whatever + compiler they were built with.""" + + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} + dirname = os.path.join(self.prefix, 'share/cantera/samples') + + cc_files = [ + 'cxx/rankine/Makefile', 'cxx/NASA_coeffs/Makefile', + 'cxx/kinetics1/Makefile', 'cxx/flamespeed/Makefile', + 'cxx/combustor/Makefile', 'f77/SConstruct' + ] + + cxx_files = [ + 'cxx/rankine/Makefile', 'cxx/NASA_coeffs/Makefile', + 'cxx/kinetics1/Makefile', 'cxx/flamespeed/Makefile', + 'cxx/combustor/Makefile' + ] + + f77_files = [ + 'f77/Makefile', 'f77/SConstruct' + ] + + fc_files = [ + 'f90/Makefile', 'f90/SConstruct' + ] + + for filename in cc_files: + filter_file(os.environ['CC'], self.compiler.cc, + os.path.join(dirname, filename), **kwargs) + + for filename in cxx_files: + filter_file(os.environ['CXX'], self.compiler.cxx, + os.path.join(dirname, filename), **kwargs) + + for filename in f77_files: + filter_file(os.environ['F77'], self.compiler.f77, + os.path.join(dirname, filename), **kwargs) + + for filename in fc_files: + filter_file(os.environ['FC'], self.compiler.fc, + os.path.join(dirname, filename), **kwargs) From 79fae306f6077f637875f2b7df393ae9be95ca9b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 3 Jun 2016 10:50:31 -0500 Subject: [PATCH 05/14] Add extensions for Python/R and more configurable install --- lib/spack/spack/cmd/create.py | 102 +++++++++++++++++++--------------- 1 file changed, 56 insertions(+), 46 deletions(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index fa7ffb39230..62e7f97f2a3 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -88,22 +88,19 @@ class ${class_name}(Package): - ""\"FIXME: put a proper description of your package here.""\" + ""\"FIXME: Put a proper description of your package here.""\" - # FIXME: add a proper url for your package's homepage here. + # FIXME: Add a proper url for your package's homepage here. homepage = "http://www.example.com" url = "${url}" ${versions} - +${extends} # FIXME: Add dependencies if this package requires them. # depends_on("foo") def install(self, spec, prefix): - # FIXME: Modify the installation instructions here - ${configure} - ${build} - ${install} +${install} """) @@ -138,39 +135,47 @@ def setup_parser(subparser): class ConfigureGuesser(object): def __call__(self, stage): - """Try to guess the type of build system used by the project. Set the - appropriate default configure, build, and install instructions.""" + """Try to guess the type of build system used by the project. + Set the appropriate default installation instructions and any + necessary extensions for Python and R.""" - # Default configure instructions - configureDict = { - 'autotools': "configure('--prefix={0}'.format(prefix))", - 'cmake': "cmake('.', *std_cmake_args)", - 'scons': "", - 'python': "", - 'r': "", - 'unknown': "# FIXME: Unknown build system" - } - - # Default build instructions - buildDict = { - 'autotools': "make()", - 'cmake': "make()", - 'scons': "scons('prefix={0}'.format(prefix))", - 'python': "", - 'r': "", - 'unknown': "make()", - } - - # Default install instructions + # Default installation instructions installDict = { - 'autotools': "make('install')", - 'cmake': "make('install')", - 'scons': "scons('install')", - 'python': "python('setup.py', 'install', " + - "'--prefix={0}'.format(prefix))", - 'r': "R('CMD', 'INSTALL', '--library={0}'.format(" + - "self.module.r_lib_dir), self.stage.archive_file)", - 'unknown': "make('install')", + 'autotools': """\ + # FIXME: Modify the configure line to suit your build system here. + configure('--prefix={0}'.format(prefix)) + + # FIXME: Add logic to build and install here. + make() + make('install')""", + + 'cmake': """\ + with working_dir('spack-build', create=True): + # FIXME: Modify the cmake line to suit your build system here. + cmake('..', *std_cmake_args) + + # FIXME: Add logic to build and install here. + make() + make('install')""", + + 'scons': """\ + # FIXME: Add logic to build and install here. + scons('prefix={0}'.format(prefix)) + scons('install')""", + + 'python': """\ + # FIXME: Add logic to build and install here. + python('setup.py', 'install', '--prefix={0}'.format(prefix))""", + + 'R': """\ + # FIXME: Add logic to build and install here. + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir),""" + + " self.stage.archive_file)""", + + 'unknown': """\ + # FIXME: Unknown build system + make() + make('install')""" } # A list of clues that give us an idea of the build system a package @@ -181,7 +186,7 @@ def __call__(self, stage): (r'/CMakeLists.txt$', 'cmake'), (r'/SConstruct$', 'scons'), (r'/setup.py$', 'python'), - (r'/NAMESPACE$', 'r') + (r'/NAMESPACE$', 'R') ] # Peek inside the compressed file. @@ -206,12 +211,18 @@ def __call__(self, stage): if any(re.search(pattern, l) for l in lines): build_system = bs - self.configure = configureDict[build_system] - self.build = buildDict[build_system] - self.install = installDict[build_system] - self.build_system = build_system + # Set any necessary extensions for Python and R + extensions = '' + if build_system in ['python', 'R']: + extensions = "\n extends('{0}')\n".format(build_system) + + self.extends = extensions + + # Set the appropriate default installation instructions + self.install = installDict[build_system] + def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL @@ -331,7 +342,7 @@ def create(parser, args): name = 'py-%s' % name # Prepend 'r-' to R package names, by convention. - if guesser.build_system == 'r': + if guesser.build_system == 'R': name = 'r-%s' % name # Create a directory for the new package. @@ -349,8 +360,7 @@ def create(parser, args): class_name=mod_to_class(name), url=url, versions=make_version_calls(ver_hash_tuples), - configure=guesser.configure, - build=guesser.build, + extends=guesser.extends, install=guesser.install)) # If everything checks out, go ahead and edit. From 5a55bb3f8de9707de43512c5f2ae41072baeb630 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 3 Jun 2016 14:22:56 -0500 Subject: [PATCH 06/14] Modify R installation template --- lib/spack/spack/cmd/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 62e7f97f2a3..c976b98b173 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -170,7 +170,7 @@ def __call__(self, stage): 'R': """\ # FIXME: Add logic to build and install here. R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir),""" + - " self.stage.archive_file)""", + " self.stage.source_path)""", 'unknown': """\ # FIXME: Unknown build system From 7e1ee463ca677e3209c800a6c3b111dd8851b78d Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 6 Jun 2016 13:25:37 -0500 Subject: [PATCH 07/14] Install examples for sundials --- .../repos/builtin/packages/cantera/package.py | 1 + .../builtin/packages/sundials/package.py | 53 +++++++++++++++++-- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/cantera/package.py b/var/spack/repos/builtin/packages/cantera/package.py index bc8c132ffc4..e9e5da44868 100644 --- a/var/spack/repos/builtin/packages/cantera/package.py +++ b/var/spack/repos/builtin/packages/cantera/package.py @@ -142,6 +142,7 @@ def install(self, spec, prefix): if '+python' in spec: # Tests will always fail if Python dependencies aren't built + # In addition, 3 of the tests fail when run in parallel scons('test', parallel=False) scons('install') diff --git a/var/spack/repos/builtin/packages/sundials/package.py b/var/spack/repos/builtin/packages/sundials/package.py index a31cf5fde4f..c55198a8502 100644 --- a/var/spack/repos/builtin/packages/sundials/package.py +++ b/var/spack/repos/builtin/packages/sundials/package.py @@ -23,6 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * +import os class Sundials(Package): @@ -30,14 +31,14 @@ class Sundials(Package): Solvers)""" homepage = "http://computation.llnl.gov/casc/sundials/" - url = "http://computation.llnl.gov/projects/sundials-suite-nonlinear-differential-algebraic-equation-solvers/download/sundials-2.6.2.tar.gz" + url = "http://computation.llnl.gov/projects/sundials-suite-nonlinear-differential-algebraic-equation-solvers/download/sundials-2.6.2.tar.gz" version('2.6.2', '3deeb0ede9f514184c6bd83ecab77d95') variant('mpi', default=True, description='Enable MPI support') variant('lapack', default=True, description='Build with external BLAS/LAPACK libraries') - variant('klu', default=True, description='Build with SuiteSparse KLU libraries') - variant('superlu', default=True, description='Build with SuperLU_MT libraries') + variant('klu', default=False, description='Build with SuiteSparse KLU libraries') + variant('superlu', default=False, description='Build with SuperLU_MT libraries') variant('openmp', default=False, description='Enable OpenMP support') variant('pthread', default=True, description='Enable POSIX threads support') @@ -49,10 +50,14 @@ class Sundials(Package): depends_on('superlu-mt+pthread', when='+superlu+pthread') def install(self, spec, prefix): - cmake_args = std_cmake_args + cmake_args = std_cmake_args[:] cmake_args.extend([ '-DBUILD_SHARED_LIBS=ON', - '-DCMAKE_C_FLAGS=-fPIC' + '-DCMAKE_C_FLAGS=-fPIC', + '-DCMAKE_Fortran_FLAGS=-fPIC', + '-DEXAMPLES_ENABLE=ON', + '-DEXAMPLES_INSTALL=ON', + '-DFCMIX_ENABLE=ON' ]) # MPI support @@ -128,3 +133,41 @@ def install(self, spec, prefix): make('install') install('LICENSE', prefix) + + self.filter_compilers() + + def filter_compilers(self): + """Run after install to tell the Makefiles to use + the compilers that Spack built the package with. + + If this isn't done, they'll have CC, CPP, and F77 set to + Spack's generic cc and f77. We want them to be bound to + whatever compiler they were built with.""" + + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} + dirname = os.path.join(self.prefix, 'examples') + + cc_files = [ + 'arkode/C_serial/Makefile', 'arkode/C_parallel/Makefile', + 'cvode/serial/Makefile', 'cvode/parallel/Makefile', + 'cvodes/serial/Makefile', 'cvodes/parallel/Makefile', + 'ida/serial/Makefile', 'ida/parallel/Makefile', + 'idas/serial/Makefile', 'idas/parallel/Makefile', + 'kinsol/serial/Makefile', 'kinsol/parallel/Makefile', + 'nvector/serial/Makefile', 'nvector/parallel/Makefile', + 'nvector/pthreads/Makefile' + ] + + f77_files = [ + 'arkode/F77_serial/Makefile', 'cvode/fcmix_serial/Makefile', + 'ida/fcmix_serial/Makefile', 'ida/fcmix_pthreads/Makefile', + 'kinsol/fcmix_serial/Makefile' + ] + + for filename in cc_files: + filter_file(os.environ['CC'], self.compiler.cc, + os.path.join(dirname, filename), **kwargs) + + for filename in f77_files: + filter_file(os.environ['F77'], self.compiler.f77, + os.path.join(dirname, filename), **kwargs) From 98d03c74e114137537ee1865244c389d687aa5d6 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 6 Jun 2016 15:31:04 -0500 Subject: [PATCH 08/14] Add support for less common compression schemes --- lib/spack/spack/cmd/create.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index c976b98b173..dc927757fbd 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -190,18 +190,19 @@ def __call__(self, stage): ] # Peek inside the compressed file. - output = '' - if stage.archive_file.endswith(('.tar', '.tar.gz', '.tar.bz2', - '.tgz', '.tbz2')): - tar = which('tar') - output = tar('--exclude=*/*/*', '-tf', + if stage.archive_file.endswith('.zip'): + try: + unzip = which('unzip') + output = unzip('-l', stage.archive_file, output=str) + except: + output = '' + else: + try: + tar = which('tar') + output = tar('--exclude=*/*/*', '-tf', stage.archive_file, output=str) - elif stage.archive_file.endswith('.gz'): - gunzip = which('gunzip') - output = gunzip('-l', stage.archive_file, output=str) - elif stage.archive_file.endswith('.zip'): - unzip = which('unzip') - output = unzip('-l', stage.archive_file, output=str) + except: + output = '' lines = output.split('\n') # Determine the build system based on the files contained From 5dfc2052bdb2f255a02e747eae9874c6645176fb Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 6 Jun 2016 15:42:30 -0500 Subject: [PATCH 09/14] Flake8 change --- lib/spack/spack/cmd/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index dc927757fbd..bc835668c23 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -200,7 +200,7 @@ def __call__(self, stage): try: tar = which('tar') output = tar('--exclude=*/*/*', '-tf', - stage.archive_file, output=str) + stage.archive_file, output=str) except: output = '' lines = output.split('\n') From c9eb5f81735dbb0c79e5d1294c410399cdeeb01a Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 10 Jun 2016 14:48:29 -0500 Subject: [PATCH 10/14] Use non-checksummed PyPi download URLs --- var/spack/repos/builtin/packages/py-3to2/package.py | 2 +- var/spack/repos/builtin/packages/py-unittest2/package.py | 2 +- var/spack/repos/builtin/packages/py-unittest2py3k/package.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-3to2/package.py b/var/spack/repos/builtin/packages/py-3to2/package.py index 577662e5833..1071a3c2097 100644 --- a/var/spack/repos/builtin/packages/py-3to2/package.py +++ b/var/spack/repos/builtin/packages/py-3to2/package.py @@ -30,7 +30,7 @@ class Py3to2(Package): for Python version 3.x into Python version 2.x.""" homepage = "https://pypi.python.org/pypi/3to2" - url = "https://pypi.python.org/packages/8f/ab/58a363eca982c40e9ee5a7ca439e8ffc5243dde2ae660ba1ffdd4868026b/3to2-1.1.1.zip" + url = "https://pypi.python.org/packages/source/3/3to2/3to2-1.1.1.zip" version('1.1.1', 'cbeed28e350dbdaef86111ace3052824') diff --git a/var/spack/repos/builtin/packages/py-unittest2/package.py b/var/spack/repos/builtin/packages/py-unittest2/package.py index c9b0ad5cdf5..f669a500ec8 100644 --- a/var/spack/repos/builtin/packages/py-unittest2/package.py +++ b/var/spack/repos/builtin/packages/py-unittest2/package.py @@ -30,7 +30,7 @@ class PyUnittest2(Package): testing framework in Python 2.7 and onwards.""" homepage = "https://pypi.python.org/pypi/unittest2" - url = "https://pypi.python.org/packages/7f/c4/2b0e2d185d9d60772c10350d9853646832609d2f299a8300ab730f199db4/unittest2-1.1.0.tar.gz" + url = "https://pypi.python.org/packages/source/u/unittest2/unittest2-1.1.0.tar.gz" version('1.1.0', 'f72dae5d44f091df36b6b513305ea000') diff --git a/var/spack/repos/builtin/packages/py-unittest2py3k/package.py b/var/spack/repos/builtin/packages/py-unittest2py3k/package.py index b2c7510d943..ca857395fbe 100644 --- a/var/spack/repos/builtin/packages/py-unittest2py3k/package.py +++ b/var/spack/repos/builtin/packages/py-unittest2py3k/package.py @@ -31,7 +31,7 @@ class PyUnittest2py3k(Package): version of unittest2.""" homepage = "https://pypi.python.org/pypi/unittest2py3k" - url = "https://pypi.python.org/packages/4e/3d/d44421e8d828af1399c1509c196db92e2a58f3764b01a0ee928d7025d1ca/unittest2py3k-0.5.1.tar.gz" + url = "https://pypi.python.org/packages/source/u/unittest2py3k/unittest2py3k-0.5.1.tar.gz" version('0.5.1', '8824ff92044310d9365f90d892bf0f09') From afff40e584364a0707ee1bee54b8f72b0335c6e5 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 15 Jun 2016 10:50:43 -0500 Subject: [PATCH 11/14] Flake8 fix for R templates --- lib/spack/spack/cmd/create.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index bc835668c23..2d3cfbb9caf 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -169,8 +169,8 @@ def __call__(self, stage): 'R': """\ # FIXME: Add logic to build and install here. - R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir),""" + - " self.stage.source_path)""", + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path)""", 'unknown': """\ # FIXME: Unknown build system From 28b2e36230625a26868122b15c18253192b9ba1d Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 15 Jun 2016 10:54:08 -0500 Subject: [PATCH 12/14] Add ctest executable --- lib/spack/spack/build_environment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 901ba1608dc..66faee14096 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -321,6 +321,7 @@ def set_module_variables_for_package(pkg, module): # TODO: Currently, everything is a link dependency, but tools like # TODO: this shouldn't be. m.cmake = Executable('cmake') + m.ctest = Executable('ctest') # standard CMake arguments m.std_cmake_args = ['-DCMAKE_INSTALL_PREFIX=%s' % pkg.prefix, From be407f531e0589dca82a6d5c7c52a59b5e0a8a3b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 15 Jun 2016 11:01:18 -0500 Subject: [PATCH 13/14] Move around extension logic --- lib/spack/spack/cmd/create.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 2d3cfbb9caf..8cbb367f861 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -214,6 +214,9 @@ def __call__(self, stage): self.build_system = build_system + # Set the appropriate default installation instructions + self.install = installDict[build_system] + # Set any necessary extensions for Python and R extensions = '' if build_system in ['python', 'R']: @@ -221,9 +224,6 @@ def __call__(self, stage): self.extends = extensions - # Set the appropriate default installation instructions - self.install = installDict[build_system] - def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL From aa86488fd9809ced16704e4bd4d607c89d6dda75 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 20 Jun 2016 12:47:17 -0500 Subject: [PATCH 14/14] Flake8 --- lib/spack/spack/build_environment.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 66faee14096..c5efa97c7be 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -112,12 +112,13 @@ def __call__(self, *args, **kwargs): return super(MakeExecutable, self).__call__(*args, **kwargs) + def load_module(mod): """Takes a module name and removes modules until it is possible to load that module. It then loads the provided module. Depends on the modulecmd implementation of modules used in cray and lmod. """ - #Create an executable of the module command that will output python code + # Create an executable of the module command that will output python code modulecmd = which('modulecmd') modulecmd.add_default_arg('python') @@ -128,11 +129,13 @@ def load_module(mod): text = modulecmd('show', mod, output=str, error=str).split() for i, word in enumerate(text): if word == 'conflict': - exec(compile(modulecmd('unload', text[i+1], output=str, error=str), '', 'exec')) + exec(compile(modulecmd('unload', text[i + 1], output=str, + error=str), '', 'exec')) # Load the module now that there are no conflicts load = modulecmd('load', mod, output=str, error=str) exec(compile(load, '', 'exec')) + def get_path_from_module(mod): """Inspects a TCL module for entries that indicate the absolute path at which the library supported by said module can be found. @@ -145,7 +148,7 @@ def get_path_from_module(mod): text = modulecmd('show', mod, output=str, error=str).split('\n') # If it lists its package directory, return that for line in text: - if line.find(mod.upper()+'_DIR') >= 0: + if line.find(mod.upper() + '_DIR') >= 0: words = line.split() return words[2] @@ -153,23 +156,24 @@ def get_path_from_module(mod): for line in text: rpath = line.find('-rpath/') if rpath >= 0: - return line[rpath+6:line.find('/lib')] + return line[rpath + 6:line.find('/lib')] # If it lists a -L instruction, use that for line in text: L = line.find('-L/') if L >= 0: - return line[L+2:line.find('/lib')] + return line[L + 2:line.find('/lib')] # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that for line in text: - if line.find('LD_LIBRARY_PATH') >= 0: + if line.find('LD_LIBRARY_PATH') >= 0: words = line.split() path = words[2] return path[:path.find('/lib')] # Unable to find module path return None + def set_compiler_environment_variables(pkg, env): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -281,8 +285,7 @@ def set_build_environment_variables(pkg, env): for directory in ('lib', 'lib64', 'share'): pcdir = join_path(pre, directory, 'pkgconfig') if os.path.isdir(pcdir): - #pkg_config_dirs.append(pcdir) - env.prepend_path('PKG_CONFIG_PATH',pcdir) + env.prepend_path('PKG_CONFIG_PATH', pcdir) if pkg.spec.architecture.target.module_name: load_module(pkg.spec.architecture.target.module_name) @@ -372,7 +375,7 @@ def get_rpaths(pkg): rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) # Second module is our compiler mod name. We use that to get rpaths from - # module show output. + # module show output. if pkg.compiler.modules and len(pkg.compiler.modules) > 1: rpaths.append(get_path_from_module(pkg.compiler.modules[1])) return rpaths @@ -397,7 +400,8 @@ def load_external_modules(pkg): for dep in list(pkg.spec.traverse()): if dep.external_module: load_module(dep.external_module) - + + def setup_package(pkg): """Execute all environment setup routines.""" spack_env = EnvironmentModifications()