Merge pull request #293 from trws/llvm-rework

Llvm rework

lib/spack/spack/package.py

@@ -34,7 +34,9 @@
 README.
 """
 import os
+import errno
 import re
+import shutil
 import time
 import itertools
 import subprocess
@@ -696,7 +698,9 @@ def _expand_archive(stage, name=self.name):
         ##########
         # Stage resources in appropriate path
         resources = self._get_resources()
-        for resource in resources:
+        # TODO: this is to allow nested resources, a better solution would be
+        # good
+        for resource in sorted(resources, key=lambda res: len(res.destination)):
             stage = resource.fetcher.stage
             _expand_archive(stage, resource.name)
             # Turn placement into a dict with relative paths
@@ -705,11 +709,23 @@ def _expand_archive(stage, name=self.name):
                 placement = {'': placement}
             # Make the paths in the dictionary absolute and link
             for key, value in placement.iteritems():
-                link_path = join_path(self.stage.source_path, resource.destination, value)
+                target_path = join_path(self.stage.source_path, resource.destination)
+                link_path = join_path(target_path, value)
                 source_path = join_path(stage.source_path, key)
+
+                try:
+                    os.makedirs(target_path)
+                except OSError as err:
+                    if err.errno == errno.EEXIST and os.path.isdir(target_path):
+                        pass
+                    else: raise
+
+                # NOTE: a reasonable fix for the TODO above might be to have
+                # these expand in place, but expand_archive does not offer
+                # this
+
                 if not os.path.exists(link_path):
-                    # Create a symlink
+                    shutil.move(source_path, link_path)
-                    os.symlink(source_path, link_path)
         ##########
         self.stage.chdir_to_source()
 

var/spack/packages/binutils/package.py

@@ -11,6 +11,7 @@ class Binutils(Package):
 
     # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell
     variant('krellpatch', default=False, description="build with openspeedshop based patch.")
+    variant('gold', default=True, description="build the gold linker")
     patch('binutilskrell-2.24.patch', when='@2.24+krellpatch')
 
     variant('libiberty', default=False, description='Also install libiberty.')
@@ -26,6 +27,9 @@ def install(self, spec, prefix):
             '--enable-targets=all',
             '--with-sysroot=/']
 
+        if '+gold' in spec:
+            configure_args.append('--enable-gold')
+
         if '+libiberty' in spec:
             configure_args.append('--enable-install-libiberty')
 

var/spack/packages/clang/package.py

@@ -1,95 +0,0 @@
-##############################################################################
-# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
-# Produced at the Lawrence Livermore National Laboratory.
-#
-# This file is part of Spack.
-# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
-# LLNL-CODE-647188
-#
-# For details, see https://github.com/llnl/spack
-# Please also see the LICENSE file for our notice and the LGPL.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License (as published by
-# the Free Software Foundation) version 2.1 dated February 1999.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
-# conditions of the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this program; if not, write to the Free Software Foundation,
-# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-##############################################################################
-
-
-from spack import *
-
-import os
-import os.path
-
-class Clang(Package):
-    """The goal of the Clang project is to create a new C, C++,
-       Objective C and Objective C++ front-end for the LLVM compiler.
-    """
-    homepage = 'http://clang.llvm.org'
-    url = 'http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz'
-
-    depends_on('llvm@3.7.0', when='@3.7.0')
-    depends_on('llvm@3.6.2', when='@3.6.2')
-    depends_on('llvm@3.5.1', when='@3.5.1')
-
-    version('3.7.0', '8f9d27335e7331cf0a4711e952f21f01', url='http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz')
-    version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz')
-    version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz')
-
-    ##########
-    # @3.7.0
-    resource(name='clang-tools-extra',
-             url='http://llvm.org/releases/3.7.0/clang-tools-extra-3.7.0.src.tar.xz',
-             md5='d5a87dacb65d981a427a536f6964642e', destination='tools', when='@3.7.0')
-    ##########
-
-    def install(self, spec, prefix):
-        env['CXXFLAGS'] = self.compiler.cxx11_flag
-
-        with working_dir('spack-build', create=True):
-
-            options = []
-            if '@3.7.0:' in spec:
-                options.append('-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp')
-            options.extend(std_cmake_args)
-
-            cmake('..',
-                  '-DCLANG_PATH_TO_LLVM_BUILD:PATH=%s' % spec['llvm'].prefix,
-                  '-DLLVM_MAIN_SRC_DIR:PATH=%s' % spec['llvm'].prefix,
-                  *options)
-            make()
-            make("install")
-            # CLang doesn't look in llvm folders for system headers...
-            self.link_llvm_directories(spec)
-
-    def link_llvm_directories(self, spec):
-
-        def clang_include_dir_at(root):
-            return join_path(root, 'include')
-
-        def clang_lib_dir_at(root):
-            return join_path(root, 'lib/clang/', str(self.version), 'include')
-
-        def do_link(source_dir, destination_dir):
-            if os.path.exists(source_dir):
-                for name in os.listdir(source_dir):
-                    source = join_path(source_dir, name)
-                    link = join_path(destination_dir, name)
-                    os.symlink(source, link)
-
-        # Link folder and files in include
-        llvm_dir = clang_include_dir_at(spec['llvm'].prefix)
-        clang_dir = clang_include_dir_at(self.prefix)
-        do_link(llvm_dir, clang_dir)
-        # Link folder and files in lib
-        llvm_dir = clang_lib_dir_at(spec['llvm'].prefix)
-        clang_dir = clang_lib_dir_at(self.prefix)
-        do_link(llvm_dir, clang_dir)

var/spack/packages/gcc/package.py

@@ -49,10 +49,13 @@ class Gcc(Package):
     version('4.6.4', 'b407a3d1480c11667f293bfb1f17d1a4')
     version('4.5.4', '27e459c2566b8209ab064570e1b378f7')
 
+    variant('gold', default=True, description="Build the gold linker plugin for ld-based LTO")
+    
     depends_on("mpfr")
     depends_on("gmp")
     depends_on("mpc")     # when @4.5:
-    depends_on("binutils~libiberty")
+    depends_on("binutils~libiberty", when='~gold')
+    depends_on("binutils~libiberty+gold", when='+gold')
 
     # Save these until we can do optional deps.
     depends_on("isl", when=DEPENDS_ON_ISL_PREDICATE)

var/spack/packages/llvm/package.py

@@ -23,6 +23,7 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 from spack import *
+import os, shutil
 
 
 class Llvm(Package):
@@ -35,38 +36,180 @@ class Llvm(Package):
     homepage = 'http://llvm.org/'
     url = 'http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz'
 
-    version('3.7.0', 'b98b9495e5655a672d6cb83e1a180f8e', url='http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz')
-    version('3.6.2', '0c1ee3597d75280dee603bae9cbf5cc2', url='http://llvm.org/releases/3.6.2/llvm-3.6.2.src.tar.xz')
-    version('3.5.1', '2d3d8004f38852aa679e5945b8ce0b14', url='http://llvm.org/releases/3.5.1/llvm-3.5.1.src.tar.xz')
     version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package
 
+    variant('debug', default=False, description="Build a debug version of LLVM, this increases binary size by an order of magnitude, make sure you have 20-30gb of space available to build this")
+    variant('clang', default=True, description="Build the LLVM C/C++/Objective-C compiler frontend")
+    variant('lldb', default=True, description="Build the LLVM debugger")
+    variant('internal_unwind', default=True, description="Build the libcxxabi libunwind")
+    variant('polly', default=True, description="Build the LLVM polyhedral optimization plugin, only builds for 3.7.0+")
+    variant('libcxx', default=True, description="Build the LLVM C++ standard library")
+    variant('compiler-rt', default=True, description="Build the LLVM compiler runtime, including sanitizers")
+    variant('gold', default=True, description="Add support for LTO with the gold linker plugin")
+
+
+    # Universal dependency
     depends_on('python@2.7:')
 
-    variant('libcxx', default=False, description="Builds the LLVM Standard C++ library targeting C++11")
+    # lldb dependencies
+    depends_on('ncurses', when='+lldb')
+    depends_on('swig', when='+lldb')
+    depends_on('libedit', when='+lldb')
 
-    ##########
+    # gold support
-    # @3.7.0
+    depends_on('binutils+gold', when='+gold')
-    resource(name='compiler-rt',
+
-             url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5',
+    # polly plugin
-             destination='projects', when='@3.7.0')
+    depends_on('gmp', when='+polly')
-    resource(name='openmp',
+    depends_on('isl', when='+polly')
-             url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f',
+
-             destination='projects', when='@3.7.0')
+    base_url =  'http://llvm.org/releases/%%(version)s/%(pkg)s-%%(version)s.src.tar.xz'
-    resource(name='libcxx',
+    llvm_url = base_url % { 'pkg' : 'llvm'}
-             url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd',
+
-             destination='projects', placement='libcxx', when='+libcxx@3.7.0')
+    resources = {
-    resource(name='libcxxabi',
+                    'compiler-rt' : {
-             url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772',
+                        'url' : base_url % { 'pkg' : 'compiler-rt'},
-             destination='projects', placement='libcxxabi', when='+libcxx@3.7.0')
+                        'destination' : 'projects',
-    ##########
+                        'placement' : 'compiler-rt',
+                    },
+                    'openmp' : {
+                        'url' : base_url % { 'pkg' : 'openmp'},
+                        'destination' : 'projects',
+                        'placement' : 'openmp',
+                    },
+                    'libcxx' : {
+                        'url' : base_url % { 'pkg' : 'libcxx'},
+                        'destination' : 'projects',
+                        'placement' : 'libcxx',
+                    },
+                    'libcxxabi' : {
+                        'url' :  base_url % { 'pkg' : 'libcxxabi'},
+                        'destination' : 'projects',
+                        'placement' : 'libcxxabi',
+                    },
+                    'clang' : {
+                        'url' :  base_url % { 'pkg' : 'cfe'},
+                        'destination' : 'tools',
+                        'placement' : 'clang',
+                    },
+                    'clang-tools-extra' : {
+                        'url' :  base_url % { 'pkg' : 'clang-tools-extra'},
+                        'destination' : 'tools/clang/tools',
+                        'placement' : 'extra',
+                    },
+                    'lldb' : {
+                        'url' :  base_url % { 'pkg' : 'lldb'},
+                        'destination' : 'tools',
+                        'placement' : 'lldb',
+                    },
+                    'polly' : {
+                        'url' :  base_url % { 'pkg' : 'polly'},
+                        'destination' : 'tools',
+                        'placement' : 'polly',
+                    },
+                    'llvm-libunwind' : {
+                        'url' :  base_url % { 'pkg' : 'libunwind'},
+                        'destination' : 'projects',
+                        'placement' : 'libunwind',
+                    },
+                }
+    releases = [
+                  {
+                    'version' : '3.7.0',
+                    'md5':'b98b9495e5655a672d6cb83e1a180f8e',
+                    'resources' : {
+                        'compiler-rt' : '383c10affd513026f08936b5525523f5',
+                        'openmp' : 'f482c86fdead50ba246a1a2b0bbf206f',
+                        'polly' : '32f93ffc9cc7e042df22089761558f8b',
+                        'libcxx' : '46aa5175cbe1ad42d6e9c995968e56dd',
+                        'libcxxabi' : '5aa769e2fca79fa5335cfae8f6258772',
+                        'clang' : '8f9d27335e7331cf0a4711e952f21f01',
+                        'clang-tools-extra' : 'd5a87dacb65d981a427a536f6964642e',
+                        'lldb' : 'e5931740400d1dc3e7db4c7ba2ceff68',
+                        'llvm-libunwind' : '9a75392eb7eb8ed5c0840007e212baf5',
+                        }
+                  },
+                  {
+                    'version' : '3.6.2',
+                    'md5':'0c1ee3597d75280dee603bae9cbf5cc2',
+                    'resources' : {
+                        'compiler-rt' : 'e3bc4eb7ba8c39a6fe90d6c988927f3c',
+                        'openmp' : '65dd5863b9b270960a96817e9152b123',
+                        'libcxx' : '22214c90697636ef960a49aef7c1823a',
+                        'libcxxabi' : '17518e361e4e228f193dd91e8ef54ba2',
+                        'clang' : 'ff862793682f714bb7862325b9c06e20',
+                        'clang-tools-extra' : '3ebc1dc41659fcec3db1b47d81575e06',
+                        'lldb' : '51e5eb552f777b950bb0ff326e60d5f0',
+                        }
+                  },
+                  {
+                    'version' : '3.5.1',
+                    'md5':'2d3d8004f38852aa679e5945b8ce0b14',
+                    'resources' : {
+                        'compiler-rt' : 'd626cfb8a9712cb92b820798ab5bc1f8',
+                        'openmp' : '121ddb10167d7fc38b1f7e4b029cf059',
+                        'libcxx' : '406f09b1dab529f3f7879f4d548329d2',
+                        'libcxxabi' : 'b22c707e8d474a99865ad3c521c3d464',
+                        'clang' : '93f9532f8f7e6f1d8e5c1116907051cb',
+                        'clang-tools-extra' : 'f13f31ed3038acadc6fa63fef812a246',
+                        'lldb' : 'cc5ea8a414c62c33e760517f8929a204',
+                        }
+                  },
+               ]
+
+    for release in releases:
+        version(release['version'], release['md5'], url=llvm_url % release)
+
+        for name, md5 in release['resources'].items():
+            resource(name=name,
+                     url=resources[name]['url'] % release,
+                     md5=md5,
+                     destination=resources[name]['destination'],
+                     when='@%(version)s' % release,
+                     placement=resources[name].get('placement', None))
 
     def install(self, spec, prefix):
         env['CXXFLAGS'] = self.compiler.cxx11_flag
+        cmake_args = [ arg for arg in std_cmake_args if 'BUILD_TYPE' not in arg ]
+
+        build_type = 'RelWithDebInfo'  if '+debug' in spec else 'Release'
+        cmake_args.extend([
+                '..',
+                '-DCMAKE_BUILD_TYPE=' + build_type,
+                '-DLLVM_REQUIRES_RTTI:BOOL=ON',
+                '-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp',
+                '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix ])
+
+        if '+gold' in spec:
+            cmake_args.append('-DLLVM_BINUTILS_INCDIR=' + os.path.join( spec['binutils'].prefix, 'include'))
+        if '+polly' in spec:
+            cmake_args.append('-DLINK_POLLY_INTO_TOOLS:Bool=ON')
+        else:
+            cmake_args.append('-DLLVM_EXTERNAL_POLLY_BUILD:Bool=OFF')
+
+        if '+clang' not in spec:
+            cmake_args.append('-DLLVM_EXTERNAL_CLANG_BUILD:Bool=OFF')
+        if '+lldb' not in spec:
+            cmake_args.append('-DLLVM_EXTERNAL_LLDB_BUILD:Bool=OFF')
+        if '+internal_unwind' not in spec:
+            cmake_args.append('-DLLVM_EXTERNAL_LIBUNWIND_BUILD:Bool=OFF')
+        if '+libcxx' not in spec:
+            cmake_args.append('-DLLVM_EXTERNAL_LIBCXX_BUILD:Bool=OFF')
+            cmake_args.append('-DLLVM_EXTERNAL_LIBCXXABI_BUILD:Bool=OFF')
+        if '+compiler-rt' not in spec:
+            cmake_args.append('-DLLVM_EXTERNAL_COMPILER_RT_BUILD:Bool=OFF')
+
+        if  '+clang' not in spec:
+            if '+clang_extra' in spec:
+                raise SpackException('The clang_extra variant requires the clang variant to be selected')
+            if '+lldb' in spec:
+                raise SpackException('The lldb variant requires the clang variant to be selected')
 
         with working_dir('spack-build', create=True):
-            cmake('..',
+            cmake(*cmake_args)
-                  '-DLLVM_REQUIRES_RTTI:BOOL=ON',
-                  '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix,
-                  *std_cmake_args)
             make()
             make("install")
+            query_path = os.path.join('bin', 'clang-query')
+            # Manually install clang-query, because llvm doesn't...
+            if os.path.exists(query_path):
+                shutil.copy(query_path, os.path.join(prefix, 'bin'))