From 113afe860ecea247c5ed343b814a633565c2033b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 10 Jul 2014 15:52:24 -0700 Subject: [PATCH 01/10] More robust symbol inclusion for 'from spack import *' - avoid errors where some symbols aren't exported to packages. - reduce the number of places each symbol needs to be mentioned in an __all__ list --- lib/spack/spack/__init__.py | 41 +++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 9eae4342e3b..a3f00a11b18 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -22,20 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - -# -# When packages call 'from spack import *', this is what is brought in. -# -# Spack internal code calls 'import spack' and accesses other -# variables (spack.db, paths, etc.) directly. -# -# TODO: maybe this should be separated out and should go in build_environment.py? -# TODO: it's not clear where all the stuff that needs to be included in packages -# should live. This file is overloaded for spack core vs. for packages. -__all__ = ['Package', 'when', 'provides', 'depends_on', 'version', - 'patch', 'Version', 'working_dir', 'which', 'Executable', - 'filter_file', 'change_sed_delimiter'] - import os import tempfile from llnl.util.filesystem import * @@ -141,11 +127,30 @@ # sys_type = None + # -# Extra imports that should be generally usable from package.py files. +# When packages call 'from spack import *', this extra stuff is brought in. # -from llnl.util.filesystem import working_dir +# Spack internal code should call 'import spack' and accesses other +# variables (spack.db, paths, etc.) directly. +# +# TODO: maybe this should be separated out and should go in build_environment.py? +# TODO: it's not clear where all the stuff that needs to be included in packages +# should live. This file is overloaded for spack core vs. for packages. +# +__all__ = ['Package', 'Version', 'when'] from spack.package import Package -from spack.relations import * -from spack.multimethod import when from spack.version import Version +from spack.multimethod import when + +import llnl.util.filesystem +from llnl.util.filesystem import * +__all__ += llnl.util.filesystem.__all__ + +import spack.relations +from spack.relations import * +__all__ += spack.relations.__all__ + +import spack.util.executable +from spack.util.executable import * +__all__ += spack.util.executable.__all__ From 0bba101ff90f317c81385306c8beaaacbaa62759 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 10 Jul 2014 15:53:31 -0700 Subject: [PATCH 02/10] Allow packages to add a dotkit() method and write custom parts of dotkits. --- lib/spack/spack/hooks/dotkit.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/hooks/dotkit.py b/lib/spack/spack/hooks/dotkit.py index 10b77323534..524145e6c15 100644 --- a/lib/spack/spack/hooks/dotkit.py +++ b/lib/spack/spack/hooks/dotkit.py @@ -33,6 +33,23 @@ import spack +class DotkitFile(object): + def __init__(self, path): + self.dk_file = open(path, 'w') + + def close(self): + self.dk_file.close() + + def write(self, *args): + self.dk_file.write(*args) + + def alter(self, var, path): + self.dk_file.write("dk_alter %s %s\n" % (var, path)) + + def setenv(self, var, value): + self.dk_file.write("dk_setenv %s %s\n" % (var, value)) + + def dotkit_file(pkg): dk_file_name = pkg.spec.format('$_$@$%@$+$=$#') + ".dk" return join_path(spack.dotkit_path, dk_file_name) @@ -51,15 +68,15 @@ def post_install(pkg): ('LD_LIBRARY_PATH', pkg.prefix.lib64)]: if os.path.isdir(path): - alterations.append("dk_alter %s %s\n" % (var, path)) + alterations.append((var, path)) if not alterations: return - alterations.append("dk_alter CMAKE_PREFIX_PATH %s\n" % pkg.prefix) + alterations.append(("CMAKE_PREFIX_PATH", pkg.prefix)) dk_file = dotkit_file(pkg) - with closing(open(dk_file, 'w')) as dk: + with closing(DotkitFile(dk_file)) as dk: # Put everything in the spack category. dk.write('#c spack\n') @@ -72,8 +89,13 @@ def post_install(pkg): dk.write("#h %s\n" % line) # Write alterations - for alter in alterations: - dk.write(alter) + for alt in alterations: + dk.alter(*alt) + + # callback in case package has extensions. + dotkit_fun = getattr(pkg, 'dotkit', None) + if dotkit_fun and hasattr(dotkit_fun, '__call__'): + dotkit_fun(dk) def post_uninstall(pkg): From 0740c576a714281b0449e33f1b8e4b00e5d9d9c0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 10 Jul 2014 15:54:10 -0700 Subject: [PATCH 03/10] Package for postgresql. --- var/spack/packages/postgresql/package.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 var/spack/packages/postgresql/package.py diff --git a/var/spack/packages/postgresql/package.py b/var/spack/packages/postgresql/package.py new file mode 100644 index 00000000000..bf14c3b922a --- /dev/null +++ b/var/spack/packages/postgresql/package.py @@ -0,0 +1,24 @@ +from spack import * + +class Postgresql(Package): + """PostgreSQL is a powerful, open source object-relational + database system. It has more than 15 years of active + development and a proven architecture that has earned it a + strong reputation for reliability, data integrity, and + correctness.""" + homepage = "http://www.postgresql.org/" + url = "http://ftp.postgresql.org/pub/source/v9.3.4/postgresql-9.3.4.tar.bz2" + + version('9.3.4', 'd0a41f54c377b2d2fab4a003b0dac762') + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + configure("--prefix=%s" % prefix) + + # FIXME: Add logic to build and install here + make() + make("install") + + + def dotkit(self, dk): + dk.setenv('PGDATA', "%s/data" %self.prefix) From 5a3803de39a67a42a70b4048039c77c4831633b2 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 25 Jul 2014 19:38:48 -0700 Subject: [PATCH 04/10] Add options to stage to make it just print out stage dir. --- lib/spack/spack/cmd/stage.py | 42 ++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index 1bf1f93c2fb..7b21faa721d 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -23,6 +23,9 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import argparse +import os + +import llnl.util.tty as tty import spack import spack.cmd @@ -33,18 +36,45 @@ def setup_parser(subparser): subparser.add_argument( '-n', '--no-checksum', action='store_true', dest='no_checksum', help="Do not check downloaded packages against checksum") + + dir_parser = subparser.add_mutually_exclusive_group() + dir_parser.add_argument( + '-d', '--stage-dir', action='store_const', dest='print_dir', + const='stage', help="Prints out the stage directory for a spec.") + dir_parser.add_argument( + '-b', '--build-dir', action='store_const', dest='print_dir', + const='build', help="Prints out the expanded archive path for a spec.") + subparser.add_argument( - 'packages', nargs=argparse.REMAINDER, help="specs of packages to stage") + 'specs', nargs=argparse.REMAINDER, help="specs of packages to stage") def stage(parser, args): - if not args.packages: + if not args.specs: tty.die("stage requires at least one package argument") if args.no_checksum: spack.do_checksum = False - specs = spack.cmd.parse_specs(args.packages, concretize=True) - for spec in specs: - package = spack.db.get(spec) - package.do_stage() + specs = spack.cmd.parse_specs(args.specs, concretize=True) + + if args.print_dir: + if len(specs) != 1: + tty.die("--stage-dir and --build-dir options only take one spec.") + + spec = specs[0] + pkg = spack.db.get(spec) + + if args.print_dir == 'stage': + print pkg.stage.path + elif args.print_dir == 'build': + if not os.listdir(pkg.stage.path): + tty.die("Stage directory is empty. Run this first:", + "spack stage " + " ".join(args.specs)) + print pkg.stage.expanded_archive_path + + else: + for spec in specs: + package = spack.db.get(spec) + package.do_stage() + From abc7d401e2d411642a9cc2b1de96990da2d17c63 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 25 Jul 2014 19:39:10 -0700 Subject: [PATCH 05/10] Add "spack cd" shell support to cd directly into the staged archive. --- share/spack/setup-env.bash | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/share/spack/setup-env.bash b/share/spack/setup-env.bash index e22a2591672..53c53dfee55 100755 --- a/share/spack/setup-env.bash +++ b/share/spack/setup-env.bash @@ -65,6 +65,10 @@ function spack { # Filter out use and unuse. For any other commands, just run the # command. case $_spack_subcommand in + "cd") + cd $(spack stage -b "$@") + return + ;; "use") ;; "unuse") ;; *) From 0b68d1292d9b415b74abc63e909c44399ae9a97a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 25 Jul 2014 20:07:05 -0700 Subject: [PATCH 06/10] Add package for openssl, have postgres use it. - Updated version wildcard to include [a-z]|alpha|beta to accommodate all the letter suffixes on openssl. --- var/spack/packages/openssl/package.py | 26 ++++++++++++++++++++++++ var/spack/packages/postgresql/package.py | 8 ++++---- 2 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 var/spack/packages/openssl/package.py diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py new file mode 100644 index 00000000000..c5a8aeb9dc2 --- /dev/null +++ b/var/spack/packages/openssl/package.py @@ -0,0 +1,26 @@ +from spack import * + +class Openssl(Package): + """The OpenSSL Project is a collaborative effort to develop a + robust, commercial-grade, full-featured, and Open Source + toolkit implementing the Secure Sockets Layer (SSL v2/v3) and + Transport Layer Security (TLS v1) protocols as well as a + full-strength general purpose cryptography library.""" + homepage = "http://www.openssl.org" + url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz" + + version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') + + depends_on("zlib") + parallel = False + + def install(self, spec, prefix): + config = Executable("./config") + config("--prefix=%s" % prefix, + "--openssldir=%s/etc/openssl" % prefix, + "zlib", + "no-krb5", + "shared") + + make() + make("install") diff --git a/var/spack/packages/postgresql/package.py b/var/spack/packages/postgresql/package.py index bf14c3b922a..a93f87df804 100644 --- a/var/spack/packages/postgresql/package.py +++ b/var/spack/packages/postgresql/package.py @@ -11,11 +11,11 @@ class Postgresql(Package): version('9.3.4', 'd0a41f54c377b2d2fab4a003b0dac762') - def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. - configure("--prefix=%s" % prefix) + depends_on("openssl") - # FIXME: Add logic to build and install here + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--with-openssl") make() make("install") From 90cd0c7efa9cc6b1e58530bede25deeaaaf9f52e Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 31 Jul 2014 13:26:55 -0700 Subject: [PATCH 07/10] Add Kevin's experimental TAU version --- var/spack/packages/tau/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/packages/tau/package.py b/var/spack/packages/tau/package.py index 8d9dbe1759b..33320622db2 100644 --- a/var/spack/packages/tau/package.py +++ b/var/spack/packages/tau/package.py @@ -11,7 +11,8 @@ class Tau(Package): url = "http://www.cs.uoregon.edu/research/paracomp/tau/tauprofile/dist/tau-2.23.1.tar.gz" version('2.23.1', '6593b47ae1e7a838e632652f0426fe72') - + version('2.23-perfdb', 'c97b404bcd94c7d9b04fa3dc0a32b0d1', + url='http://www.nic.uoregon.edu/~khuck/tau2-latest.tar.gz') def install(self, spec, prefix): # TAU isn't happy with directories that have '@' in the path. Sigh. From 6127b0baa69fa469cf9a02dcd5dc78f674b746b3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 6 Aug 2014 21:58:59 -0700 Subject: [PATCH 08/10] new prototype TAU tarball from Kevin --- var/spack/packages/tau/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/tau/package.py b/var/spack/packages/tau/package.py index 33320622db2..e58acfb41a4 100644 --- a/var/spack/packages/tau/package.py +++ b/var/spack/packages/tau/package.py @@ -10,8 +10,8 @@ class Tau(Package): homepage = "http://www.cs.uoregon.edu/research/tau" url = "http://www.cs.uoregon.edu/research/paracomp/tau/tauprofile/dist/tau-2.23.1.tar.gz" - version('2.23.1', '6593b47ae1e7a838e632652f0426fe72') - version('2.23-perfdb', 'c97b404bcd94c7d9b04fa3dc0a32b0d1', + version('2.23.1', '6593b47ae1e7a838e632652f0426fe72') + version('2.23.2-perfdb', 'f743a65951220f5b46b9d3cf179129d0', url='http://www.nic.uoregon.edu/~khuck/tau2-latest.tar.gz') def install(self, spec, prefix): From fa3b19000db9ed6f3d6cf31895aeee4ac0d4e633 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 11 Aug 2014 22:47:13 -0700 Subject: [PATCH 09/10] update tau tarball --- var/spack/packages/tau/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/tau/package.py b/var/spack/packages/tau/package.py index e58acfb41a4..caafe4e2c92 100644 --- a/var/spack/packages/tau/package.py +++ b/var/spack/packages/tau/package.py @@ -11,7 +11,7 @@ class Tau(Package): url = "http://www.cs.uoregon.edu/research/paracomp/tau/tauprofile/dist/tau-2.23.1.tar.gz" version('2.23.1', '6593b47ae1e7a838e632652f0426fe72') - version('2.23.2-perfdb', 'f743a65951220f5b46b9d3cf179129d0', + version('2.23.2-perfdb', '4048f693eee246d48eb2619c0f05999e', url='http://www.nic.uoregon.edu/~khuck/tau2-latest.tar.gz') def install(self, spec, prefix): From e301d623329d3f484316643c6a50dc3df4806dab Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 20 Aug 2014 11:46:59 -0700 Subject: [PATCH 10/10] Remove development TAU version from package. --- var/spack/packages/tau/package.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/var/spack/packages/tau/package.py b/var/spack/packages/tau/package.py index caafe4e2c92..048fac80aa5 100644 --- a/var/spack/packages/tau/package.py +++ b/var/spack/packages/tau/package.py @@ -10,9 +10,7 @@ class Tau(Package): homepage = "http://www.cs.uoregon.edu/research/tau" url = "http://www.cs.uoregon.edu/research/paracomp/tau/tauprofile/dist/tau-2.23.1.tar.gz" - version('2.23.1', '6593b47ae1e7a838e632652f0426fe72') - version('2.23.2-perfdb', '4048f693eee246d48eb2619c0f05999e', - url='http://www.nic.uoregon.edu/~khuck/tau2-latest.tar.gz') + version('2.23.1', '6593b47ae1e7a838e632652f0426fe72') def install(self, spec, prefix): # TAU isn't happy with directories that have '@' in the path. Sigh.