Introduce static_to_shared_library function (#6092)

The static_to_shared_library function takes an existing static library
and produces a shared library based on it.

lib/spack/spack/build_environment.py

@@ -373,6 +373,104 @@ def set_module_variables_for_package(pkg, module):
     # Platform-specific library suffix.
     m.dso_suffix = dso_suffix
 
+    def static_to_shared_library(static_lib, shared_lib=None, **kwargs):
+        compiler_path = kwargs.get('compiler', m.spack_cc)
+        compiler = Executable(compiler_path)
+
+        return _static_to_shared_library(pkg.spec.architecture, compiler,
+                                         static_lib, shared_lib, **kwargs)
+
+    m.static_to_shared_library = static_to_shared_library
+
+
+def _static_to_shared_library(arch, compiler, static_lib, shared_lib=None,
+                              **kwargs):
+    """
+    Converts a static library to a shared library. The static library has to
+    be built with PIC for the conversion to work.
+
+    Parameters:
+        static_lib (str): Path to the static library.
+        shared_lib (str): Path to the shared library. Default is to derive
+                          from the static library's path.
+
+    Keyword arguments:
+        compiler (str): Path to the compiler. Default is spack_cc.
+        compiler_output: Where to print compiler output to.
+        arguments (str list): Additional arguments for the compiler.
+        version (str): Library version. Default is unspecified.
+        compat_version (str): Library compatibility version. Default is
+                              version.
+    """
+    compiler_output = kwargs.get('compiler_output', None)
+    arguments = kwargs.get('arguments', [])
+    version = kwargs.get('version', None)
+    compat_version = kwargs.get('compat_version', version)
+
+    if not shared_lib:
+        shared_lib = '{0}.{1}'.format(os.path.splitext(static_lib)[0],
+                                      dso_suffix)
+
+    compiler_args = []
+
+    # TODO: Compiler arguments should not be hardcoded but provided by
+    #       the different compiler classes.
+    if 'linux' in arch:
+        soname = os.path.basename(shared_lib)
+
+        if compat_version:
+            soname += '.{0}'.format(compat_version)
+
+        compiler_args = [
+            '-shared',
+            '-Wl,-soname,{0}'.format(soname),
+            '-Wl,--whole-archive',
+            static_lib,
+            '-Wl,--no-whole-archive'
+        ]
+    elif 'darwin' in arch:
+        install_name = shared_lib
+
+        if compat_version:
+            install_name += '.{0}'.format(compat_version)
+
+        compiler_args = [
+            '-dynamiclib',
+            '-install_name {0}'.format(install_name),
+            '-Wl,-force_load,{0}'.format(static_lib)
+        ]
+
+        if compat_version:
+            compiler_args.append('-compatibility_version {0}'.format(
+                compat_version))
+
+        if version:
+            compiler_args.append('-current_version {0}'.format(version))
+
+    if len(arguments) > 0:
+        compiler_args.extend(arguments)
+
+    shared_lib_base = shared_lib
+
+    if version:
+        shared_lib += '.{0}'.format(version)
+    elif compat_version:
+        shared_lib += '.{0}'.format(compat_version)
+
+    compiler_args.extend(['-o', shared_lib])
+
+    # Create symlinks for version and compat_version
+    shared_lib_link = os.path.basename(shared_lib)
+
+    if version or compat_version:
+        os.symlink(shared_lib_link, shared_lib_base)
+
+    if compat_version and compat_version != version:
+        os.symlink(shared_lib_link, '{0}.{1}'.format(shared_lib_base,
+                                                     compat_version))
+
+    return compiler(*compiler_args, output=compiler_output)
+
 
 def get_rpath_deps(pkg):
     """Return immediate or transitive RPATHs depending on the package."""

lib/spack/spack/test/build_environment.py

@@ -0,0 +1,99 @@
+##############################################################################
+# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/spack/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import os
+import pytest
+
+import spack
+from llnl.util.filesystem import join_path
+from spack.build_environment import dso_suffix, _static_to_shared_library
+from spack.util.executable import Executable
+
+
+@pytest.fixture
+def build_environment():
+    cc = Executable(join_path(spack.build_env_path, "cc"))
+    cxx = Executable(join_path(spack.build_env_path, "c++"))
+    fc = Executable(join_path(spack.build_env_path, "fc"))
+
+    realcc = "/bin/mycc"
+    prefix = "/spack-test-prefix"
+
+    os.environ['SPACK_CC'] = realcc
+    os.environ['SPACK_CXX'] = realcc
+    os.environ['SPACK_FC'] = realcc
+
+    os.environ['SPACK_PREFIX'] = prefix
+    os.environ['SPACK_ENV_PATH'] = "test"
+    os.environ['SPACK_DEBUG_LOG_DIR'] = "."
+    os.environ['SPACK_DEBUG_LOG_ID'] = "foo-hashabc"
+    os.environ['SPACK_COMPILER_SPEC'] = "gcc@4.4.7"
+    os.environ['SPACK_SHORT_SPEC'] = (
+        "foo@1.2 arch=linux-rhel6-x86_64 /hashabc")
+
+    os.environ['SPACK_CC_RPATH_ARG']  = "-Wl,-rpath,"
+    os.environ['SPACK_CXX_RPATH_ARG'] = "-Wl,-rpath,"
+    os.environ['SPACK_F77_RPATH_ARG'] = "-Wl,-rpath,"
+    os.environ['SPACK_FC_RPATH_ARG']  = "-Wl,-rpath,"
+
+    if 'SPACK_DEPENDENCIES' in os.environ:
+        del os.environ['SPACK_DEPENDENCIES']
+
+    yield {'cc': cc, 'cxx': cxx, 'fc': fc}
+
+    for name in ('SPACK_CC', 'SPACK_CXX', 'SPACK_FC', 'SPACK_PREFIX',
+                 'SPACK_ENV_PATH', 'SPACK_DEBUG_LOG_DIR',
+                 'SPACK_COMPILER_SPEC', 'SPACK_SHORT_SPEC',
+                 'SPACK_CC_RPATH_ARG', 'SPACK_CXX_RPATH_ARG',
+                 'SPACK_F77_RPATH_ARG', 'SPACK_FC_RPATH_ARG'):
+        del os.environ[name]
+
+
+def test_static_to_shared_library(build_environment):
+    os.environ['SPACK_TEST_COMMAND'] = 'dump-args'
+
+    expected = {
+        'linux': ('/bin/mycc -Wl,-rpath,/spack-test-prefix/lib'
+                  ' -Wl,-rpath,/spack-test-prefix/lib64 -shared'
+                  ' -Wl,-soname,{2} -Wl,--whole-archive {0}'
+                  ' -Wl,--no-whole-archive -o {1}'),
+        'darwin': ('/bin/mycc -Wl,-rpath,/spack-test-prefix/lib'
+                   ' -Wl,-rpath,/spack-test-prefix/lib64 -dynamiclib'
+                   ' -install_name {1} -Wl,-force_load,{0} -o {1}')
+    }
+
+    static_lib = '/spack/libfoo.a'
+
+    for arch in ('linux', 'darwin'):
+        for shared_lib in (None, '/spack/libbar.so'):
+            output = _static_to_shared_library(arch, build_environment['cc'],
+                                               static_lib, shared_lib,
+                                               compiler_output=str).strip()
+
+            if not shared_lib:
+                shared_lib = '{0}.{1}'.format(
+                    os.path.splitext(static_lib)[0], dso_suffix)
+
+            assert output == expected[arch].format(
+                static_lib, shared_lib, os.path.basename(shared_lib))

var/spack/repos/builtin/packages/lua/liblua-shared.patch

@@ -1,64 +0,0 @@
-diff -ru a/Makefile b/Makefile
---- a/Makefile	2017-10-27 23:49:02.821830453 +0200
-+++ b/Makefile	2017-10-27 23:50:45.697634413 +0200
-@@ -41,7 +41,7 @@
- # What to install.
- TO_BIN= lua luac
- TO_INC= lua.h luaconf.h lualib.h lauxlib.h lua.hpp
--TO_LIB= liblua.a
-+TO_LIB= liblua.a $(LUA_DSO).$(R)
- TO_MAN= lua.1 luac.1
- 
- # Lua version and release.
-@@ -52,7 +52,7 @@
- all:	$(PLAT)
- 
- $(PLATS) clean:
--	cd src && $(MAKE) $@
-+	cd src && $(MAKE) $@ V=$(V) R=$(R)
- 
- test:	dummy
- 	src/lua -v
-@@ -63,6 +63,8 @@
- 	cd src && $(INSTALL_DATA) $(TO_INC) $(INSTALL_INC)
- 	cd src && $(INSTALL_DATA) $(TO_LIB) $(INSTALL_LIB)
- 	cd doc && $(INSTALL_DATA) $(TO_MAN) $(INSTALL_MAN)
-+	ln -sf $(LUA_DSO).$(R) $(INSTALL_LIB)/$(LUA_DSO).$(V)
-+	ln -sf $(LUA_DSO).$(R) $(INSTALL_LIB)/$(LUA_DSO)
- 
- uninstall:
- 	cd src && cd $(INSTALL_BIN) && $(RM) $(TO_BIN)
-diff -ru a/src/Makefile b/src/Makefile
---- a/src/Makefile	2017-10-27 23:49:02.823830467 +0200
-+++ b/src/Makefile	2017-10-27 23:50:01.923284014 +0200
-@@ -29,6 +29,8 @@
- PLATS= aix bsd c89 freebsd generic linux macosx mingw posix solaris
- 
- LUA_A=	liblua.a
-+LUA_SO= liblua.so
-+LUA_DYLIB= liblua.dylib
- CORE_O=	lapi.o lcode.o lctype.o ldebug.o ldo.o ldump.o lfunc.o lgc.o llex.o \
- 	lmem.o lobject.o lopcodes.o lparser.o lstate.o lstring.o ltable.o \
- 	ltm.o lundump.o lvm.o lzio.o
-@@ -43,7 +45,7 @@
- LUAC_O=	luac.o
- 
- ALL_O= $(BASE_O) $(LUA_O) $(LUAC_O)
--ALL_T= $(LUA_A) $(LUA_T) $(LUAC_T)
-+ALL_T= $(LUA_A) $(LUA_T) $(LUAC_T) $(LUA_DSO)
- ALL_A= $(LUA_A)
- 
- # Targets start here.
-@@ -59,6 +61,12 @@
- 	$(AR) $@ $(BASE_O)
- 	$(RANLIB) $@
- 
-+$(LUA_SO): $(CORE_O) $(LIB_O)
-+	$(CC) -shared -ldl -Wl,-soname,$(LUA_SO).$(V) -o $@.$(R) $^ -lm $(MYLDFLAGS)
-+
-+$(LUA_DYLIB): $(CORE_O) $(LIB_O)
-+	$(CC) -dynamiclib -install_name @LUA_PREFIX@/lib/$(LUA_DYLIB).$(V) -compatibility_version $(V) -current_version $(R) -o $@.$(R) $^
-+
- $(LUA_T): $(LUA_O) $(LUA_A)
- 	$(CC) -o $@ $(LDFLAGS) $(LUA_O) $(LUA_A) $(LIBS)
- 

var/spack/repos/builtin/packages/lua/package.py

@@ -58,11 +58,6 @@ class Lua(Package):
         destination="luarocks",
         placement='luarocks')
 
-    # Based on patches from Arch Linux and Homebrew:
-    # https://git.archlinux.org/svntogit/packages.git/tree/trunk/liblua.so.patch?h=packages/lua
-    # https://github.com/Homebrew/homebrew-core/blob/master/Formula/lua.rb
-    patch('liblua-shared.patch')
-
     def install(self, spec, prefix):
         if spec.satisfies("platform=darwin"):
             target = 'macosx'
@@ -76,8 +71,6 @@ def install(self, spec, prefix):
                  spec['ncurses'].prefix.lib),
              'MYLIBS=-lncursesw',
              'CC=%s -std=gnu99' % spack_cc,
-             'LUA_DSO=liblua.%s' % (
-                 dso_suffix),
              target)
         make('INSTALL_TOP=%s' % prefix,
              'MYCFLAGS=%s' % (
@@ -87,10 +80,12 @@ def install(self, spec, prefix):
                  spec['ncurses'].prefix.lib),
              'MYLIBS=-lncursesw',
              'CC=%s -std=gnu99' % spack_cc,
-             'LUA_DSO=liblua.%s' % (
-                 dso_suffix),
              'install')
 
+        static_to_shared_library(join_path(prefix.lib, 'liblua.a'),
+                                 arguments=['-lm'], version=self.version,
+                                 compat_version=self.version.up_to(2))
+
         with working_dir(os.path.join('luarocks', 'luarocks')):
             configure('--prefix=' + prefix, '--with-lua=' + prefix)
             make('build')