From fbabfc593d10a7fc35944e72e2eb8b1ebf4818eb Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 6 Apr 2016 16:44:22 -0500 Subject: [PATCH 1/6] Make R extensable and add a couple of packages for verification. Added R as a build system so that the create template will have the correct configure line. Added a regex for version parsing as the R URLs are a little odd. --- lib/spack/spack/cmd/create.py | 8 ++- lib/spack/spack/url.py | 3 ++ var/spack/repos/builtin/packages/R/package.py | 54 +++++++++++++++++++ .../repos/builtin/packages/r-abind/package.py | 15 ++++++ .../repos/builtin/packages/r-magic/package.py | 15 ++++++ 5 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 var/spack/repos/builtin/packages/r-abind/package.py create mode 100644 var/spack/repos/builtin/packages/r-magic/package.py diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index f0cd50b8df6..e3a31806ab8 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -124,10 +124,12 @@ def __call__(self, stage): autotools = "configure('--prefix=%s' % prefix)" cmake = "cmake('.', *std_cmake_args)" python = "python('setup.py', 'install', '--prefix=%s' % prefix)" + r = "R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file)" config_lines = ((r'/configure$', 'autotools', autotools), (r'/CMakeLists.txt$', 'cmake', cmake), - (r'/setup.py$', 'python', python)) + (r'/setup.py$', 'python', python), + (r'/NAMESPACE$', 'r', r)) # Peek inside the tarball. tar = which('tar') @@ -272,6 +274,10 @@ def create(parser, args): if guesser.build_system == 'python': name = 'py-%s' % name + # Prepend 'r-' to R package names, by convention. + if guesser.build_system == 'r': + name = 'r-%s' % name + # Create a directory for the new package. pkg_path = repo.filename_for_package_name(name) if os.path.exists(pkg_path) and not args.force: diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index f51f05cad7d..b4fb70d6fb7 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -206,6 +206,9 @@ def parse_version_offset(path): # e.g. lame-398-1 (r'-((\d)+-\d)', stem), + # e.g. foobar_1.2-3 + (r'_((\d+\.)+\d+(-\d+)?)', stem), + # e.g. foobar-4.5.1 (r'-((\d+\.)*\d+)$', stem), diff --git a/var/spack/repos/builtin/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py index 2471dff09b6..3a76416f27c 100644 --- a/var/spack/repos/builtin/packages/R/package.py +++ b/var/spack/repos/builtin/packages/R/package.py @@ -1,4 +1,14 @@ +import functools +import glob +import inspect +import os +import re +from contextlib import closing + +import spack +from llnl.util.lang import match_predicate from spack import * +from spack.util.environment import * class R(Package): @@ -9,6 +19,8 @@ class R(Package): """ homepage = "https://www.r-project.org" url = "http://cran.cnr.berkeley.edu/src/base/R-3/R-3.1.2.tar.gz" + + extendable = True version('3.2.3', '1ba3dac113efab69e706902810cc2970') version('3.2.2', '57cef5c2e210a5454da1979562a10e5b') @@ -47,3 +59,45 @@ def install(self, spec, prefix): configure(*options) make() make('install') + + # ======================================================================== + # Set up environment to make install easy for R extensions. + # ======================================================================== + + @property + def r_lib_dir(self): + return os.path.join('lib64', 'R', 'library') + + def setup_dependent_environment(self, spack_env, run_env, extension_spec): + # Set R_LIBS to include the library dir for the + # extension and any other R extensions it depends on. + r_libs_path = [] + for d in extension_spec.traverse(): + if d.package.extends(self.spec): + r_libs_path.append(os.path.join(d.prefix, self.r_lib_dir)) + + r_libs_path = ':'.join(r_libs_path) + spack_env.set('R_LIBS', r_libs_path) + + # For run time environment set only the path for extension_spec and prepend it to R_LIBS + if extension_spec.package.extends(self.spec): + run_env.prepend_path('R_LIBS', os.path.join(extension_spec.prefix, self.r_lib_dir)) + + + def setup_dependent_package(self, module, ext_spec): + """ + Called before R modules' install() methods. + + In most cases, extensions will only need to have one line:: + + R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) + """ + # R extension builds can have a global R executable function + module.R = Executable(join_path(self.spec.prefix.bin, 'R')) + + # Add variable for library directry + module.r_lib_dir = os.path.join(ext_spec.prefix, self.r_lib_dir) + + # Make the site packages directory for extensions, if it does not exist already. + if ext_spec.package.is_extension: + mkdirp(module.r_lib_dir) diff --git a/var/spack/repos/builtin/packages/r-abind/package.py b/var/spack/repos/builtin/packages/r-abind/package.py new file mode 100644 index 00000000000..54d399c432f --- /dev/null +++ b/var/spack/repos/builtin/packages/r-abind/package.py @@ -0,0 +1,15 @@ +from spack import * + +class RAbind(Package): + """Combine multidimensional arrays into a single array. This is a generalization of 'cbind' and 'rbind'. Works with vectors, matrices, and higher-dimensional arrays. Also provides functions 'adrop', 'asub', and 'afill' for manipulating, extracting and replacing data in arrays.""" + + homepage = "https://cran.r-project.org/" + url = "https://cran.r-project.org/src/contrib/abind_1.4-3.tar.gz" + + version('1.4-3', '10fcf80c677b991bf263d38be35a1fc5', expand=False) + + extends('R') + + def install(self, spec, prefix): + + R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) diff --git a/var/spack/repos/builtin/packages/r-magic/package.py b/var/spack/repos/builtin/packages/r-magic/package.py new file mode 100644 index 00000000000..5f25b2a1625 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-magic/package.py @@ -0,0 +1,15 @@ +from spack import * + +class RMagic(Package): + """A collection of efficient, vectorized algorithms for the creation and investigation of magic squares and hypercubes, including a variety of functions for the manipulation and analysis of arbitrarily dimensioned arrays.""" + homepage = "https://cran.r-project.org/" + url = "https://cran.r-project.org/src/contrib/magic_1.5-6.tar.gz" + + version('1.5-6', 'a68e5ced253b2196af842e1fc84fd029', expand=False) + + extends('R') + + depends_on('r-abind') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) From 32779ab1f6c2658771e69e58f8d8a451b23043b0 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 6 Apr 2016 17:02:34 -0500 Subject: [PATCH 2/6] Add r-filehash to test version naming. --- .../repos/builtin/packages/r-filehash/package.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-filehash/package.py diff --git a/var/spack/repos/builtin/packages/r-filehash/package.py b/var/spack/repos/builtin/packages/r-filehash/package.py new file mode 100644 index 00000000000..a3b688da10d --- /dev/null +++ b/var/spack/repos/builtin/packages/r-filehash/package.py @@ -0,0 +1,14 @@ +from spack import * + +class RFilehash(Package): + """Implements a simple key-value style database where character string keys are associated with data values that are stored on the disk. A simple interface is provided for inserting, retrieving, and deleting data from the database. Utilities are provided that allow 'filehash' databases to be treated much like environments and lists are already used in R. These utilities are provided to encourage interactive and exploratory analysis on large datasets. Three different file formats for representing the database are currently available and new formats can easily be incorporated by third parties for use in the 'filehash' framework.""" + + homepage = 'https://cran.r-project.org/' + url = "https://cran.r-project.org/src/contrib/filehash_2.3.tar.gz" + + version('2.3', '01fffafe09b148ccadc9814c103bdc2f', expand=False) + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) From b19d6ab9c5e4b7a8289e057826b498dd93ea7e08 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 6 Apr 2016 19:08:01 -0500 Subject: [PATCH 3/6] Check for non-numeric bits in the stem. --- lib/spack/spack/url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index b4fb70d6fb7..ad51da9d47f 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -207,7 +207,7 @@ def parse_version_offset(path): (r'-((\d)+-\d)', stem), # e.g. foobar_1.2-3 - (r'_((\d+\.)+\d+(-\d+)?)', stem), + (r'_((\d+\.)+\d+(-\d+)?[a-z]?)', stem), # e.g. foobar-4.5.1 (r'-((\d+\.)*\d+)$', stem), From 8a7f34665b0b8b95f273401082d862989bb4443c Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 6 Apr 2016 23:05:49 -0500 Subject: [PATCH 4/6] Make sure base bioconductor package can be installed. --- .../builtin/packages/r-BiocGenerics/package.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 var/spack/repos/builtin/packages/r-BiocGenerics/package.py diff --git a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py new file mode 100644 index 00000000000..e2d9bb95941 --- /dev/null +++ b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py @@ -0,0 +1,13 @@ +from spack import * + +class RBiocgenerics(Package): + """S4 generic functions needed by many Bioconductor packages.""" + homepage = 'https://www.bioconductor.org/packages/release/bioc/html/BiocGenerics.html' + url = "https://www.bioconductor.org/packages/release/bioc/src/contrib/BiocGenerics_0.16.1.tar.gz" + + version('0.16.1', 'c2148ffd86fc6f1f819c7f68eb2c744f', expand=False) + + extends('R') + + def install(self, spec, prefix): + R('CMD', 'INSTALL', '--library=%s' % self.module.r_lib_dir, '%s' % self.stage.archive_file) From 62d175c984c5413fd95185d7bb3280bad69e1e33 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 23 Apr 2016 12:30:34 -0500 Subject: [PATCH 5/6] This commit explicitly sets `libdir`. This is necessary because different systems use different defaults. --- var/spack/repos/builtin/packages/R/package.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py index 3a76416f27c..7c4aa3520c8 100644 --- a/var/spack/repos/builtin/packages/R/package.py +++ b/var/spack/repos/builtin/packages/R/package.py @@ -50,9 +50,12 @@ class R(Package): depends_on('tk') def install(self, spec, prefix): + rlibdir = join_path(prefix, 'rlib') options = ['--prefix=%s' % prefix, + '--libdir=%s' % rlibdir, '--enable-R-shlib', - '--enable-BLAS-shlib'] + '--enable-BLAS-shlib', + '--enable-R-framework=no'] if '+external-lapack' in spec: options.extend(['--with-blas', '--with-lapack']) @@ -66,7 +69,7 @@ def install(self, spec, prefix): @property def r_lib_dir(self): - return os.path.join('lib64', 'R', 'library') + return os.path.join('rlib', 'R', 'library') def setup_dependent_environment(self, spack_env, run_env, extension_spec): # Set R_LIBS to include the library dir for the From 2b7b7f6d9758d47bb59b59e92cb2c7e3b785ac51 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 24 Apr 2016 18:11:18 -0500 Subject: [PATCH 6/6] Reformat description text. --- .../repos/builtin/packages/r-BiocGenerics/package.py | 1 + var/spack/repos/builtin/packages/r-abind/package.py | 5 ++++- var/spack/repos/builtin/packages/r-filehash/package.py | 10 +++++++++- var/spack/repos/builtin/packages/r-magic/package.py | 6 +++++- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py index e2d9bb95941..2d92c1c4d88 100644 --- a/var/spack/repos/builtin/packages/r-BiocGenerics/package.py +++ b/var/spack/repos/builtin/packages/r-BiocGenerics/package.py @@ -2,6 +2,7 @@ class RBiocgenerics(Package): """S4 generic functions needed by many Bioconductor packages.""" + homepage = 'https://www.bioconductor.org/packages/release/bioc/html/BiocGenerics.html' url = "https://www.bioconductor.org/packages/release/bioc/src/contrib/BiocGenerics_0.16.1.tar.gz" diff --git a/var/spack/repos/builtin/packages/r-abind/package.py b/var/spack/repos/builtin/packages/r-abind/package.py index 54d399c432f..d06c7e9240a 100644 --- a/var/spack/repos/builtin/packages/r-abind/package.py +++ b/var/spack/repos/builtin/packages/r-abind/package.py @@ -1,7 +1,10 @@ from spack import * class RAbind(Package): - """Combine multidimensional arrays into a single array. This is a generalization of 'cbind' and 'rbind'. Works with vectors, matrices, and higher-dimensional arrays. Also provides functions 'adrop', 'asub', and 'afill' for manipulating, extracting and replacing data in arrays.""" + """Combine multidimensional arrays into a single array. This is a + generalization of 'cbind' and 'rbind'. Works with vectors, matrices, and + higher-dimensional arrays. Also provides functions 'adrop', 'asub', and + 'afill' for manipulating, extracting and replacing data in arrays.""" homepage = "https://cran.r-project.org/" url = "https://cran.r-project.org/src/contrib/abind_1.4-3.tar.gz" diff --git a/var/spack/repos/builtin/packages/r-filehash/package.py b/var/spack/repos/builtin/packages/r-filehash/package.py index a3b688da10d..4911c636b4d 100644 --- a/var/spack/repos/builtin/packages/r-filehash/package.py +++ b/var/spack/repos/builtin/packages/r-filehash/package.py @@ -1,7 +1,15 @@ from spack import * class RFilehash(Package): - """Implements a simple key-value style database where character string keys are associated with data values that are stored on the disk. A simple interface is provided for inserting, retrieving, and deleting data from the database. Utilities are provided that allow 'filehash' databases to be treated much like environments and lists are already used in R. These utilities are provided to encourage interactive and exploratory analysis on large datasets. Three different file formats for representing the database are currently available and new formats can easily be incorporated by third parties for use in the 'filehash' framework.""" + """Implements a simple key-value style database where character string keys + are associated with data values that are stored on the disk. A simple + interface is provided for inserting, retrieving, and deleting data from the + database. Utilities are provided that allow 'filehash' databases to be + treated much like environments and lists are already used in R. These + utilities are provided to encourage interactive and exploratory analysis on + large datasets. Three different file formats for representing the database + are currently available and new formats can easily be incorporated by third + parties for use in the 'filehash' framework.""" homepage = 'https://cran.r-project.org/' url = "https://cran.r-project.org/src/contrib/filehash_2.3.tar.gz" diff --git a/var/spack/repos/builtin/packages/r-magic/package.py b/var/spack/repos/builtin/packages/r-magic/package.py index 5f25b2a1625..e900cdb216e 100644 --- a/var/spack/repos/builtin/packages/r-magic/package.py +++ b/var/spack/repos/builtin/packages/r-magic/package.py @@ -1,7 +1,11 @@ from spack import * class RMagic(Package): - """A collection of efficient, vectorized algorithms for the creation and investigation of magic squares and hypercubes, including a variety of functions for the manipulation and analysis of arbitrarily dimensioned arrays.""" + """A collection of efficient, vectorized algorithms for the creation and + investigation of magic squares and hypercubes, including a variety of + functions for the manipulation and analysis of arbitrarily dimensioned + arrays.""" + homepage = "https://cran.r-project.org/" url = "https://cran.r-project.org/src/contrib/magic_1.5-6.tar.gz"