go rework

This commit includes:
* a new go package that uses gccgo to bootstrap the go toolchain
* env support added to Executable
* a new Go fetch strategy that uses `go get` to fetch a package and all
  of its deps
* A platinum searcher package that leverages the new go package and
  fetch strategy

lib/spack/spack/fetch_strategy.py

@@ -381,6 +381,60 @@ def __str__(self):
     def __repr__(self):
         return "%s<%s>" % (self.__class__, self.url)
 
+class GoFetchStrategy(VCSFetchStrategy):
+    """Fetch strategy that employs the `go get` infrastructure
+       Use like this in a package:
+
+           version('name', go='github.com/monochromegane/the_platinum_searcher/...')
+
+       Go get does not natively support versions, they can be faked with git
+    """
+    enabled = True
+    required_attributes = ('go',)
+
+    def __init__(self, **kwargs):
+        # Discards the keywords in kwargs that may conflict with the next call to __init__
+        forwarded_args = copy.copy(kwargs)
+        forwarded_args.pop('name', None)
+
+        super(GoFetchStrategy, self).__init__('go', **forwarded_args)
+        self._go = None
+
+    @property
+    def go_version(self):
+        vstring = self.go('version', output=str).split(' ')[2]
+        return Version(vstring)
+
+    @property
+    def go(self):
+        if not self._go:
+            self._go = which('go', required=True)
+        return self._go
+
+    @_needs_stage
+    def fetch(self):
+        self.stage.chdir()
+
+        tty.msg("Trying to get go resource:", self.url)
+
+        try:
+            os.mkdir('go')
+        except OSError:
+            pass
+        env = dict(os.environ)
+        env['GOPATH'] = os.path.join(os.getcwd(),'go')
+	self.go('get', '-v', '-d', self.url, env=env)
+
+    def archive(self, destination):
+        super(GoFetchStrategy, self).archive(destination, exclude='.git')
+
+    @_needs_stage
+    def reset(self):
+        self.stage.chdir_to_source()
+        self.go('clean')
+
+    def __str__(self):
+        return "[go] %s" % self.url
 
 class GitFetchStrategy(VCSFetchStrategy):
     """Fetch strategy that gets source code from a git repository.

lib/spack/spack/util/executable.py

@@ -105,6 +105,8 @@ def __call__(self, *args, **kwargs):
         fail_on_error = kwargs.pop("fail_on_error", True)
         ignore_errors = kwargs.pop("ignore_errors", ())
 
+	env = kwargs.get('env', None)
+
         # TODO: This is deprecated.  Remove in a future version.
         return_output = kwargs.pop("return_output", False)
 
@@ -149,7 +151,7 @@ def streamify(arg, mode):
 
         try:
             proc = subprocess.Popen(
-                cmd, stdin=istream, stderr=estream, stdout=ostream)
+                cmd, stdin=istream, stderr=estream, stdout=ostream, env=env)
             out, err = proc.communicate()
 
             rc = self.returncode = proc.returncode

var/spack/repos/builtin/packages/gcc/package.py

@@ -1,33 +1,33 @@
 ##############################################################################
-# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory.
 #
 # This file is part of Spack.
-# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
 # LLNL-CODE-647188
 #
 # For details, see https://github.com/llnl/spack
 # Please also see the LICENSE file for our notice and the LGPL.
 #
 # This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License (as
+# it under the terms of the GNU General Public License (as published by
-# published by the Free Software Foundation) version 2.1, February 1999.
+# the Free Software Foundation) version 2.1 dated February 1999.
 #
 # This program is distributed in the hope that it will be useful, but
 # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
-# conditions of the GNU Lesser General Public License for more details.
+# conditions of the GNU General Public License for more details.
 #
-# You should have received a copy of the GNU Lesser General Public
+# You should have received a copy of the GNU Lesser General Public License
-# License along with this program; if not, write to the Free Software
+# along with this program; if not, write to the Free Software Foundation,
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 from spack import *
 
 from contextlib import closing
 from glob import glob
 import sys
-import os
+
 
 class Gcc(Package):
     """The GNU Compiler Collection includes front ends for C, C++,
@@ -50,10 +50,12 @@ class Gcc(Package):
     version('4.6.4', 'b407a3d1480c11667f293bfb1f17d1a4')
     version('4.5.4', '27e459c2566b8209ab064570e1b378f7')
 
-    variant('binutils', default=sys.platform != 'darwin',
+    variant('binutils',
-        description="Build via binutils")
+            default=sys.platform != 'darwin',
-    variant('gold', default=sys.platform != 'darwin',
+            description="Build via binutils")
-        description="Build the gold linker plugin for ld-based LTO")
+    variant('gold',
+            default=sys.platform != 'darwin',
+            description="Build the gold linker plugin for ld-based LTO")
 
     depends_on("mpfr")
     depends_on("gmp")
@@ -62,12 +64,9 @@ class Gcc(Package):
     depends_on("binutils~libiberty", when='+binutils ~gold')
     depends_on("binutils~libiberty+gold", when='+binutils +gold')
 
-    if sys.platform != 'darwin':
-        provides('go_compiler' when='@4.7.1:')
-
     # TODO: integrate these libraries.
-    #depends_on("ppl")
+    # depends_on("ppl")
-    #depends_on("cloog")
+    # depends_on("cloog")
     if sys.platform == 'darwin':
         patch('darwin/gcc-4.9.patch1', when='@4.9.3')
         patch('darwin/gcc-4.9.patch2', when='@4.9.3')
@@ -75,7 +74,7 @@ class Gcc(Package):
     def install(self, spec, prefix):
         # libjava/configure needs a minor fix to install into spack paths.
         filter_file(r"'@.*@'", "'@[[:alnum:]]*@'", 'libjava/configure',
-            string=True)
+                    string=True)
 
         enabled_languages = set(('c', 'c++', 'fortran', 'java', 'objc'))
 
@@ -83,62 +82,59 @@ def install(self, spec, prefix):
             enabled_languages.add('go')
 
         # Generic options to compile GCC
-        options = ["--prefix=%s" % prefix,
+        options = ["--prefix=%s" % prefix, "--libdir=%s/lib64" % prefix,
-                   "--libdir=%s/lib64" % prefix,
                    "--disable-multilib",
                    "--enable-languages=" + ','.join(enabled_languages),
-                   "--with-mpc=%s" % spec['mpc'].prefix,
+                   "--with-mpc=%s" % spec['mpc'].prefix, "--with-mpfr=%s" %
-                   "--with-mpfr=%s" % spec['mpfr'].prefix,
+                   spec['mpfr'].prefix, "--with-gmp=%s" % spec['gmp'].prefix,
-                   "--with-gmp=%s" % spec['gmp'].prefix,
+                   "--enable-lto", "--with-quad"]
-                   "--enable-lto",
-                   "--with-quad"]
         # Binutils
         if spec.satisfies('+binutils'):
             static_bootstrap_flags = "-static-libstdc++ -static-libgcc"
-            binutils_options = ["--with-sysroot=/",
+            binutils_options = [
-                                "--with-stage1-ldflags=%s %s" %
+                "--with-sysroot=/", "--with-stage1-ldflags=%s %s" %
-                                    (self.rpath_args, static_bootstrap_flags),
+                (self.rpath_args, static_bootstrap_flags),
-                                "--with-boot-ldflags=%s %s" %
+                "--with-boot-ldflags=%s %s" %
-                                    (self.rpath_args, static_bootstrap_flags),
+                (self.rpath_args, static_bootstrap_flags), "--with-gnu-ld",
-                                "--with-gnu-ld",
+                "--with-ld=%s/bin/ld" % spec['binutils'].prefix,
-                                "--with-ld=%s/bin/ld" % spec['binutils'].prefix,
+                "--with-gnu-as",
-                                "--with-gnu-as",
+                "--with-as=%s/bin/as" % spec['binutils'].prefix
-                                "--with-as=%s/bin/as" % spec['binutils'].prefix]
+            ]
             options.extend(binutils_options)
         # Isl
         if 'isl' in spec:
             isl_options = ["--with-isl=%s" % spec['isl'].prefix]
             options.extend(isl_options)
 
-        if sys.platform == 'darwin' :
+        if sys.platform == 'darwin':
-            darwin_options = [ "--with-build-config=bootstrap-debug" ]
+            darwin_options = ["--with-build-config=bootstrap-debug"]
             options.extend(darwin_options)
 
         build_dir = join_path(self.stage.path, 'spack-build')
-        configure = Executable( join_path(self.stage.source_path, 'configure') )
+        configure = Executable(join_path(self.stage.source_path, 'configure'))
         with working_dir(build_dir, create=True):
             # Rest of install is straightforward.
             configure(*options)
-            if sys.platform == 'darwin' : make("bootstrap")
+            if sys.platform == 'darwin':
-            else: make()
+                make("bootstrap")
+            else:
+                make()
             make("install")
 
         self.write_rpath_specs()
 
-
     @property
     def spec_dir(self):
         # e.g. lib64/gcc/x86_64-unknown-linux-gnu/4.9.2
         spec_dir = glob("%s/lib64/gcc/*/*" % self.prefix)
         return spec_dir[0] if spec_dir else None
 
-
     def write_rpath_specs(self):
         """Generate a spec file so the linker adds a rpath to the libs
            the compiler used to build the executable."""
         if not self.spec_dir:
             tty.warn("Could not install specs for %s." %
-                self.spec.format('$_$@'))
+                     self.spec.format('$_$@'))
             return
 
         gcc = Executable(join_path(self.prefix.bin, 'gcc'))
@@ -149,5 +145,5 @@ def write_rpath_specs(self):
                 out.write(line + "\n")
                 if line.startswith("*link:"):
                     out.write("-rpath %s/lib:%s/lib64 \\\n" %
-                        (self.prefix, self.prefix))
+                              (self.prefix, self.prefix))
         set_install_permissions(specs_file)

var/spack/repos/builtin/packages/go/package.py

@@ -1,6 +1,9 @@
 import os
+import shutil
+import glob
 from spack import *
 
+
 class Go(Package):
     """The golang compiler and build environment"""
     homepage = "https://golang.org"
@@ -10,43 +13,65 @@ class Go(Package):
 
     # temporary fix until tags are pulled correctly
     version('1.4.2', git='https://go.googlesource.com/go', tag='go1.4.2')
-    version('1.5.0', git='https://go.googlesource.com/go', tag='go1.5.0')
+    version('1.5.4', git='https://go.googlesource.com/go', tag='go1.5.4')
+    version('1.6.2', git='https://go.googlesource.com/go', tag='go1.6.2')
 
+    variant('test',
+            default=True,
+            description="Run tests as part of build, a good idea but quite"
+            " time consuming")
 
-    provides('go_compiler')
+    provides('golang')
-    # to-do, make non-c self-hosting compilers possible
+
+    # to-do, make non-c self-hosting compilers feasible without backflips
     # should be go_compiler, but that creates an infinite loop
-    depends_on('gcc', when='@1.5:')
+    depends_on('gcc@5:', when='@1.5:')
+    depends_on('git')
 
     def install(self, spec, prefix):
-      os.environ['GOROOT'] = os.getcwd()
+        bash = which('bash')
-      os.environ['GOROOT_FINAL'] = prefix
+        with working_dir('src'):
-      bash = which('bash')
+            if '+test' in spec:
-      bash('-c', 'env')
+                bash('all.bash')
-      bash('-c', 'pwd')
+            else:
-      with working_dir('src'):
+                bash('make.bash')
-        #TODO: crutch until the read-only-filesystem bug is fixed upstream
-        bash('all.bash', fail_on_error=False)
-      cp = which('cp')
-      bash('-c', 'cp -r ./* "{}"'.format(prefix))
 
-    def setup_dependent_environment(self, module, spec, ext_spec):
+        try:
+            os.makedirs(prefix)
+        except OSError:
+            pass
+        for f in glob.glob('*'):
+            if os.path.isdir(f):
+                shutil.copytree(f, os.path.join(prefix, f))
+            else:
+                shutil.copy2(f, os.path.join(prefix, f))
+
+    def setup_environment(self, spack_env, run_env):
+        # spack_env.set("GOROOT", self.spec.prefix)
+        # run_env.set("GOROOT", self.spec.prefix)
+        spack_env.set('GOROOT_FINAL', self.spec.prefix)
+        spack_env.set('GOROOT_BOOTSTRAP', self.spec['gcc'].prefix)
+
+    def setup_dependent_package(self, module, ext_spec):
+        #  Add a go command/compiler for extensions
+        module.go = Executable(join_path(self.spec.prefix.bin, 'go'))
+
+    def setup_dependent_environment(self, spack_env, run_env, ext_spec):
         """Called before go modules' install() methods.
 
         In most cases, extensions will only need to have one line::
 
-            go('get', '<package>')
+        go('get', '<package>')
         """
-        #  Add a go command for extensions
+        # spack_env.set("GOROOT", self.spec.prefix)
-        module.go = Executable(join_path(spec.prefix.bin, 'go'))
+        # run_env.set("GOROOT", self.spec.prefix)
-        os.environ['GOROOT'] = spec.prefix
-
-        stage_path = os.path.realpath(ext_spec.package.stage.source_path)
-        print "PREFIX: {}".format(stage_path)
-        go_paths = [stage_path]
-        for d in ext_spec.traverse():
-            if d.package.extends(self.spec):
-                go_paths.append(d.prefix)
-        os.environ['GOPATH'] = ':'.join(go_paths)
 
+        # stage_path = os.path.realpath(ext_spec.package.stage.source_path)
+        # print "PREFIX: {}".format(stage_path)
+        # go_paths = [stage_path]
+        # for d in ext_spec.traverse():
+        #     if d.package.extends(self.spec):
+        #         go_paths.append(d.prefix)
+        # spack_env.prepend_path('GOPATH',  ':'.join(go_paths))
 
+        spack_env.set('GOPATH', ext_spec.package.stage.source_path)

var/spack/repos/builtin/packages/the_platinum_searcher/package.py

@@ -1,19 +1,20 @@
 from spack import *
+import os
+import shutil
+
 
 class ThePlatinumSearcher(Package):
     """Fast parallel recursive grep alternative"""
     # FIXME: add a proper url for your package's homepage here.
     homepage = "https://github.com/monochromegane/the_platinum_searcher"
-    url      = "https://github.com/monochromegane/the_platinum_searcher/archive/v1.7.7.tar.gz"
+    url      = "https://github.com/monochromegane/the_platinum_searcher"
 
-    version('1.7.7', '08d7265e101bc1427d5d4b9903aa1166')
+    package = 'github.com/monochromegane/the_platinum_searcher/...'
 
-    depends_on("go")
+    version('head', go=package)
+
+    extends("go")
 
     def install(self, spec, prefix):
-        env = which('env')
+        go('install', self.package)
-        env()
+        shutil.copytree('bin', os.path.join(prefix, 'bin'))
-        # Fetch all dependencies
-        go('get', './...')
-        # Build pt
-        go('build')