significant llvm update

This update significantly reworks the llvm and clang packages.  The llvm
package now includes variants allowing it to build and install any and
all of:

* clang
* lldb
* llvm's libunwind (why, WHY did they name it this?!?)
* polly (including building it directly into the clang tools, 3.7.0 only)
* clang extra tools
* compiler-rt (sanitizers)
* clang lto (the gold linker plugin that allows same to work)
* libcxx/libcxxabi
* libopenmp, also setting the default openmp runtime to same, when
  parameters happen this shoudl be an option of libomp or libgomp

Ideally, this should have rpath setup like the gcc package does, but
clang's driver has no support for specs as such, and no clearly
equivalent mechanism either.  If anyone has ideas on this, they would be
welcome.

One significant note related to gcc though, if you test this on LLNL
systems, or anywhere that has multiple GCCs straddling the dwarf2
boundary and sharing a libstdc++, build a gcc with spack and use that to
build clang.  If you use a gcc4.8+  to build this with an older
libstdc++ it will fail on missing unwind symbols because of the
discrepancy.

Resource handling has been changed slightly to move the unpacked archive
into the target rather than use symlinks, because symlinks break certain
kinds of relative paths, and orders resource staging such that nested
resources are unpacked after outer ones.

lib/spack/spack/package.py

@@ -36,6 +36,7 @@
 import os
 import errno
 import re
+import shutil
 import time
 import itertools
 import subprocess
@@ -711,15 +712,20 @@ def _expand_archive(stage, name=self.name):
                 target_path = join_path(self.stage.source_path, resource.destination)
                 link_path = join_path(target_path, value)
                 source_path = join_path(stage.source_path, key)
+
+                try:
+                    os.makedirs(target_path)
+                except OSError as err:
+                    if err.errno == errno.EEXIST and os.path.isdir(target_path):
+                        pass
+                    else: raise
+
+                # NOTE: a reasonable fix for the TODO above might be to have
+                # these expand in place, but expand_archive does not offer
+                # this
+
                 if not os.path.exists(link_path):
-                    # Create a symlink
+                    shutil.move(source_path, link_path)
-                    try:
-                        os.makedirs(target_path)
-                    except OSError as err:
-                        if err.errno == errno.EEXIST and os.path.isdir(target_path):
-                            pass
-                        else: raise
-                    os.symlink(source_path, link_path)
         ##########
         self.stage.chdir_to_source()
 

var/spack/packages/clang/package.py

@@ -1,95 +0,0 @@
-##############################################################################
-# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
-# Produced at the Lawrence Livermore National Laboratory.
-#
-# This file is part of Spack.
-# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
-# LLNL-CODE-647188
-#
-# For details, see https://github.com/llnl/spack
-# Please also see the LICENSE file for our notice and the LGPL.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License (as published by
-# the Free Software Foundation) version 2.1 dated February 1999.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
-# conditions of the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this program; if not, write to the Free Software Foundation,
-# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-##############################################################################
-
-
-from spack import *
-
-import os
-import os.path
-
-class Clang(Package):
-    """The goal of the Clang project is to create a new C, C++,
-       Objective C and Objective C++ front-end for the LLVM compiler.
-    """
-    homepage = 'http://clang.llvm.org'
-    url = 'http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz'
-
-    depends_on('llvm@3.7.0', when='@3.7.0')
-    depends_on('llvm@3.6.2', when='@3.6.2')
-    depends_on('llvm@3.5.1', when='@3.5.1')
-
-    version('3.7.0', '8f9d27335e7331cf0a4711e952f21f01', url='http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz')
-    version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz')
-    version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz')
-
-    ##########
-    # @3.7.0
-    resource(name='clang-tools-extra',
-             url='http://llvm.org/releases/3.7.0/clang-tools-extra-3.7.0.src.tar.xz',
-             md5='d5a87dacb65d981a427a536f6964642e', destination='tools', when='@3.7.0')
-    ##########
-
-    def install(self, spec, prefix):
-        env['CXXFLAGS'] = self.compiler.cxx11_flag
-
-        with working_dir('spack-build', create=True):
-
-            options = []
-            if '@3.7.0:' in spec:
-                options.append('-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp')
-            options.extend(std_cmake_args)
-
-            cmake('..',
-                  '-DCLANG_PATH_TO_LLVM_BUILD:PATH=%s' % spec['llvm'].prefix,
-                  '-DLLVM_MAIN_SRC_DIR:PATH=%s' % spec['llvm'].prefix,
-                  *options)
-            make()
-            make("install")
-            # CLang doesn't look in llvm folders for system headers...
-            self.link_llvm_directories(spec)
-
-    def link_llvm_directories(self, spec):
-
-        def clang_include_dir_at(root):
-            return join_path(root, 'include')
-
-        def clang_lib_dir_at(root):
-            return join_path(root, 'lib/clang/', str(self.version), 'include')
-
-        def do_link(source_dir, destination_dir):
-            if os.path.exists(source_dir):
-                for name in os.listdir(source_dir):
-                    source = join_path(source_dir, name)
-                    link = join_path(destination_dir, name)
-                    os.symlink(source, link)
-
-        # Link folder and files in include
-        llvm_dir = clang_include_dir_at(spec['llvm'].prefix)
-        clang_dir = clang_include_dir_at(self.prefix)
-        do_link(llvm_dir, clang_dir)
-        # Link folder and files in lib
-        llvm_dir = clang_lib_dir_at(spec['llvm'].prefix)
-        clang_dir = clang_lib_dir_at(self.prefix)
-        do_link(llvm_dir, clang_dir)

var/spack/packages/llvm/package.py

@@ -35,38 +35,176 @@ class Llvm(Package):
     homepage = 'http://llvm.org/'
     url = 'http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz'
 
-    version('3.7.0', 'b98b9495e5655a672d6cb83e1a180f8e', url='http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz')
-    version('3.6.2', '0c1ee3597d75280dee603bae9cbf5cc2', url='http://llvm.org/releases/3.6.2/llvm-3.6.2.src.tar.xz')
-    version('3.5.1', '2d3d8004f38852aa679e5945b8ce0b14', url='http://llvm.org/releases/3.5.1/llvm-3.5.1.src.tar.xz')
     version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package
 
+    variant('debug', default=False, description="Build a debug version of LLVM, this increases binary size by an order of magnitude, make sure you have 20-30gb of space available to build this")
+    variant('clang', default=True, description="Build the LLVM C/C++/Objective-C compiler frontend")
+    variant('lldb', default=True, description="Build the LLVM debugger")
+    variant('internal_unwind', default=True, description="Build the libcxxabi libunwind")
+    variant('polly', default=True, description="Build the LLVM polyhedral optimization plugin, only builds for 3.7.0+")
+    variant('libcxx', default=True, description="Build the LLVM C++ standard library")
+    variant('compiler-rt', default=True, description="Build the LLVM compiler runtime, including sanitizers")
+    variant('lto', default=True, description="Add support for LTO with the gold linker plugin")
+
+
+    # Universal dependency
     depends_on('python@2.7:')
 
-    variant('libcxx', default=False, description="Builds the LLVM Standard C++ library targeting C++11")
+    # lldb dependencies
+    depends_on('ncurses', when='+lldb')
+    depends_on('swig', when='+lldb')
+    depends_on('libedit', when='+lldb')
 
-    ##########
+    # gold support
-    # @3.7.0
+    depends_on('binutils+gold', when='+lto')
-    resource(name='compiler-rt',
+
-             url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5',
+    # polly plugin
-             destination='projects', when='@3.7.0')
+    depends_on('gmp', when='+polly')
-    resource(name='openmp',
+    depends_on('isl', when='+polly')
-             url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f',
+
-             destination='projects', when='@3.7.0')
+    base_url =  'http://llvm.org/releases/%%(version)s/%(pkg)s-%%(version)s.src.tar.xz'
-    resource(name='libcxx',
+    llvm_url = base_url % { 'pkg' : 'llvm'}
-             url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd',
+
-             destination='projects', placement='libcxx', when='+libcxx@3.7.0')
+    resources = {
-    resource(name='libcxxabi',
+                    'compiler-rt' : {
-             url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772',
+                        'url' : base_url % { 'pkg' : 'compiler-rt'},
-             destination='projects', placement='libcxxabi', when='+libcxx@3.7.0')
+                        'destination' : 'projects',
-    ##########
+                        'placement' : 'compiler-rt',
+                    },
+                    'openmp' : {
+                        'url' : base_url % { 'pkg' : 'openmp'},
+                        'destination' : 'projects',
+                        'placement' : 'openmp',
+                    },
+                    'libcxx' : {
+                        'url' : base_url % { 'pkg' : 'libcxx'},
+                        'destination' : 'projects',
+                        'placement' : 'libcxx',
+                    },
+                    'libcxxabi' : {
+                        'url' :  base_url % { 'pkg' : 'libcxxabi'},
+                        'destination' : 'projects',
+                        'placement' : 'libcxxabi',
+                    },
+                    'clang' : {
+                        'url' :  base_url % { 'pkg' : 'cfe'},
+                        'destination' : 'tools',
+                        'placement' : 'clang',
+                    },
+                    'clang-tools-extra' : {
+                        'url' :  base_url % { 'pkg' : 'clang-tools-extra'},
+                        'destination' : 'tools/clang/tools',
+                        'placement' : 'extra',
+                    },
+                    'lldb' : {
+                        'url' :  base_url % { 'pkg' : 'lldb'},
+                        'destination' : 'tools',
+                        'placement' : 'lldb',
+                    },
+                    'polly' : {
+                        'url' :  base_url % { 'pkg' : 'polly'},
+                        'destination' : 'tools',
+                        'placement' : 'polly',
+                    },
+                    'llvm-libunwind' : {
+                        'url' :  base_url % { 'pkg' : 'libunwind'},
+                        'destination' : 'projects',
+                        'placement' : 'libunwind',
+                    },
+                }
+    releases = [
+                  {
+                    'version' : '3.7.0',
+                    'md5':'b98b9495e5655a672d6cb83e1a180f8e',
+                    'resources' : {
+                        'compiler-rt' : '383c10affd513026f08936b5525523f5',
+                        'openmp' : 'f482c86fdead50ba246a1a2b0bbf206f',
+                        'polly' : '32f93ffc9cc7e042df22089761558f8b',
+                        'libcxx' : '46aa5175cbe1ad42d6e9c995968e56dd',
+                        'libcxxabi' : '5aa769e2fca79fa5335cfae8f6258772',
+                        'clang' : '8f9d27335e7331cf0a4711e952f21f01',
+                        'clang-tools-extra' : 'd5a87dacb65d981a427a536f6964642e',
+                        'lldb' : 'e5931740400d1dc3e7db4c7ba2ceff68',
+                        'llvm-libunwind' : '9a75392eb7eb8ed5c0840007e212baf5',
+                        }
+                  },
+                  {
+                    'version' : '3.6.2',
+                    'md5':'0c1ee3597d75280dee603bae9cbf5cc2',
+                    'resources' : {
+                        'compiler-rt' : 'e3bc4eb7ba8c39a6fe90d6c988927f3c',
+                        'openmp' : '65dd5863b9b270960a96817e9152b123',
+                        'libcxx' : '22214c90697636ef960a49aef7c1823a',
+                        'libcxxabi' : '17518e361e4e228f193dd91e8ef54ba2',
+                        'clang' : 'ff862793682f714bb7862325b9c06e20',
+                        'clang-tools-extra' : '3ebc1dc41659fcec3db1b47d81575e06',
+                        'lldb' : '51e5eb552f777b950bb0ff326e60d5f0',
+                        }
+                  },
+                  {
+                    'version' : '3.5.1',
+                    'md5':'2d3d8004f38852aa679e5945b8ce0b14',
+                    'resources' : {
+                        'compiler-rt' : 'd626cfb8a9712cb92b820798ab5bc1f8',
+                        'openmp' : '121ddb10167d7fc38b1f7e4b029cf059',
+                        'libcxx' : '406f09b1dab529f3f7879f4d548329d2',
+                        'libcxxabi' : 'b22c707e8d474a99865ad3c521c3d464',
+                        'clang' : '93f9532f8f7e6f1d8e5c1116907051cb',
+                        'clang-tools-extra' : 'f13f31ed3038acadc6fa63fef812a246',
+                        'lldb' : 'cc5ea8a414c62c33e760517f8929a204',
+                        }
+                  },
+               ]
+
+    for release in releases:
+        version(release['version'], release['md5'], url=llvm_url % release)
+
+        for name, md5 in release['resources'].items():
+            resource(name=name,
+                     url=resources[name]['url'] % release,
+                     md5=md5,
+                     destination=resources[name]['destination'],
+                     when='@%(version)s' % release,
+                     placement=resources[name].get('placement', None))
 
     def install(self, spec, prefix):
         env['CXXFLAGS'] = self.compiler.cxx11_flag
+        cmake_args = [ arg for arg in std_cmake_args if 'BUILD_TYPE' not in arg ]
+
+        build_type = 'RelWithDebInfo'  if '+debug' in spec else 'Release'
+        cmake_args.extend([
+                '..',
+                '-DCMAKE_BUILD_TYPE=' + build_type,
+                '-DLLVM_REQUIRES_RTTI:BOOL=ON',
+                '-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp',
+                '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix ])
+
+        if '+lto' in spec:
+            cmake_args.append('-DLLVM_BINUTILS_INCDIR=' + os.path.join( spec['binutils'].prefix, 'include'))
+        if '+polly' in spec:
+            cmake_args.append('-DPOLLY_LINK_INTO_TOOLS:Bool=ON')
+        else:
+            cmake_args.append('-DLLVM_EXTERNAL_POLLY_BUILD:Bool=OFF')
+
+        if '+clang' not in spec:
+            cmake_args.append('-DLLVM_EXTERNAL_CLANG_BUILD:Bool=OFF')
+        if '+lldb' not in spec:
+            cmake_args.append('-DLLVM_EXTERNAL_LLDB_BUILD:Bool=OFF')
+        if '+internal_unwind' not in spec:
+            cmake_args.append('-DLLVM_EXTERNAL_LIBUNWIND_BUILD:Bool=OFF')
+        if '+libcxx' not in spec:
+            cmake_args.append('-DLLVM_EXTERNAL_LIBCXX_BUILD:Bool=OFF')
+            cmake_args.append('-DLLVM_EXTERNAL_LIBCXXABI_BUILD:Bool=OFF')
+        if '+compiler-rt' not in spec:
+            cmake_args.append('-DLLVM_EXTERNAL_COMPILER_RT_BUILD:Bool=OFF')
+
+        if  '+clang' not in spec:
+            if '+clang_extra' in spec:
+                raise SpackException('The clang_extra variant requires the clang variant to be selected')
+            if '+lldb' in spec:
+                raise SpackException('The lldb variant requires the clang variant to be selected')
 
         with working_dir('spack-build', create=True):
-            cmake('..',
+            cmake(*cmake_args)
-                  '-DLLVM_REQUIRES_RTTI:BOOL=ON',
-                  '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix,
-                  *std_cmake_args)
             make()
             make("install")