zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

go rework

Browse Source 
This commit includes:
* a new go package that uses gccgo to bootstrap the go toolchain
* env support added to Executable
* a new Go fetch strategy that uses `go get` to fetch a package and all
  of its deps
* A platinum searcher package that leverages the new go package and
  fetch strategy
This commit is contained in:
Tom Scogland 2016-05-14 22:35:55 -07:00
parent 0816e9554d
commit 577657b3f7
5 changed files with 159 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
lib/spack/spack/fetch_strategy.py
View File

@ -381,6 +381,60 @@ def __str__(self):
def __repr__(self):
return "%s<%s>" % (self.__class__, self.url)
class GoFetchStrategy(VCSFetchStrategy):
"""Fetch strategy that employs the `go get` infrastructure
Use like this in a package:
version('name', go='github.com/monochromegane/the_platinum_searcher/...')
Go get does not natively support versions, they can be faked with git
"""
enabled = True
required_attributes = ('go',)
def __init__(self, **kwargs):
# Discards the keywords in kwargs that may conflict with the next call to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(GoFetchStrategy, self).__init__('go', **forwarded_args)
self._go = None
@property
def go_version(self):
vstring = self.go('version', output=str).split(' ')[2]
return Version(vstring)
@property
def go(self):
if not self._go:
self._go = which('go', required=True)
return self._go
@_needs_stage
def fetch(self):
self.stage.chdir()
tty.msg("Trying to get go resource:", self.url)
try:
os.mkdir('go')
except OSError:
pass
env = dict(os.environ)
env['GOPATH'] = os.path.join(os.getcwd(),'go')
self.go('get', '-v', '-d', self.url, env=env)
def archive(self, destination):
super(GoFetchStrategy, self).archive(destination, exclude='.git')
@_needs_stage
def reset(self):
self.stage.chdir_to_source()
self.go('clean')
def __str__(self):
return "[go] %s" % self.url
class GitFetchStrategy(VCSFetchStrategy):
"""Fetch strategy that gets source code from a git repository.

4
lib/spack/spack/util/executable.py
View File

@ -105,6 +105,8 @@ def __call__(self, *args, **kwargs):
fail_on_error = kwargs.pop("fail_on_error", True)
ignore_errors = kwargs.pop("ignore_errors", ())
env = kwargs.get('env', None)
# TODO: This is deprecated. Remove in a future version.
return_output = kwargs.pop("return_output", False)
@ -149,7 +151,7 @@ def streamify(arg, mode):
try:
proc = subprocess.Popen(
cmd, stdin=istream, stderr=estream, stdout=ostream)
cmd, stdin=istream, stderr=estream, stdout=ostream, env=env)
out, err = proc.communicate()
rc = self.returncode = proc.returncode

84
var/spack/repos/builtin/packages/gcc/package.py
View File

@ -1,33 +1,33 @@
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
from contextlib import closing
from glob import glob
import sys
import os
class Gcc(Package):
"""The GNU Compiler Collection includes front ends for C, C++,
@ -50,10 +50,12 @@ class Gcc(Package):
version('4.6.4', 'b407a3d1480c11667f293bfb1f17d1a4')
version('4.5.4', '27e459c2566b8209ab064570e1b378f7')
variant('binutils', default=sys.platform != 'darwin',
description="Build via binutils")
variant('gold', default=sys.platform != 'darwin',
description="Build the gold linker plugin for ld-based LTO")
variant('binutils',
default=sys.platform != 'darwin',
description="Build via binutils")
variant('gold',
default=sys.platform != 'darwin',
description="Build the gold linker plugin for ld-based LTO")
depends_on("mpfr")
depends_on("gmp")
@ -62,12 +64,9 @@ class Gcc(Package):
depends_on("binutils~libiberty", when='+binutils ~gold')
depends_on("binutils~libiberty+gold", when='+binutils +gold')
if sys.platform != 'darwin':
provides('go_compiler' when='@4.7.1:')
# TODO: integrate these libraries.
#depends_on("ppl")
#depends_on("cloog")
# depends_on("ppl")
# depends_on("cloog")
if sys.platform == 'darwin':
patch('darwin/gcc-4.9.patch1', when='@4.9.3')
patch('darwin/gcc-4.9.patch2', when='@4.9.3')
@ -75,7 +74,7 @@ class Gcc(Package):
def install(self, spec, prefix):
# libjava/configure needs a minor fix to install into spack paths.
filter_file(r"'@.*@'", "'@[[:alnum:]]*@'", 'libjava/configure',
string=True)
string=True)
enabled_languages = set(('c', 'c++', 'fortran', 'java', 'objc'))
@ -83,62 +82,59 @@ def install(self, spec, prefix):
enabled_languages.add('go')
# Generic options to compile GCC
options = ["--prefix=%s" % prefix,
"--libdir=%s/lib64" % prefix,
options = ["--prefix=%s" % prefix, "--libdir=%s/lib64" % prefix,
"--disable-multilib",
"--enable-languages=" + ','.join(enabled_languages),
"--with-mpc=%s" % spec['mpc'].prefix,
"--with-mpfr=%s" % spec['mpfr'].prefix,
"--with-gmp=%s" % spec['gmp'].prefix,
"--enable-lto",
"--with-quad"]
"--with-mpc=%s" % spec['mpc'].prefix, "--with-mpfr=%s" %
spec['mpfr'].prefix, "--with-gmp=%s" % spec['gmp'].prefix,
"--enable-lto", "--with-quad"]
# Binutils
if spec.satisfies('+binutils'):
static_bootstrap_flags = "-static-libstdc++ -static-libgcc"
binutils_options = ["--with-sysroot=/",
"--with-stage1-ldflags=%s %s" %
(self.rpath_args, static_bootstrap_flags),
"--with-boot-ldflags=%s %s" %
(self.rpath_args, static_bootstrap_flags),
"--with-gnu-ld",
"--with-ld=%s/bin/ld" % spec['binutils'].prefix,
"--with-gnu-as",
"--with-as=%s/bin/as" % spec['binutils'].prefix]
binutils_options = [
"--with-sysroot=/", "--with-stage1-ldflags=%s %s" %
(self.rpath_args, static_bootstrap_flags),
"--with-boot-ldflags=%s %s" %
(self.rpath_args, static_bootstrap_flags), "--with-gnu-ld",
"--with-ld=%s/bin/ld" % spec['binutils'].prefix,
"--with-gnu-as",
"--with-as=%s/bin/as" % spec['binutils'].prefix
]
options.extend(binutils_options)
# Isl
if 'isl' in spec:
isl_options = ["--with-isl=%s" % spec['isl'].prefix]
options.extend(isl_options)
if sys.platform == 'darwin' :
darwin_options = [ "--with-build-config=bootstrap-debug" ]
if sys.platform == 'darwin':
darwin_options = ["--with-build-config=bootstrap-debug"]
options.extend(darwin_options)
build_dir = join_path(self.stage.path, 'spack-build')
configure = Executable( join_path(self.stage.source_path, 'configure') )
configure = Executable(join_path(self.stage.source_path, 'configure'))
with working_dir(build_dir, create=True):
# Rest of install is straightforward.
configure(*options)
if sys.platform == 'darwin' : make("bootstrap")
else: make()
if sys.platform == 'darwin':
make("bootstrap")
else:
make()
make("install")
self.write_rpath_specs()
@property
def spec_dir(self):
# e.g. lib64/gcc/x86_64-unknown-linux-gnu/4.9.2
spec_dir = glob("%s/lib64/gcc/*/*" % self.prefix)
return spec_dir[0] if spec_dir else None
def write_rpath_specs(self):
"""Generate a spec file so the linker adds a rpath to the libs
the compiler used to build the executable."""
if not self.spec_dir:
tty.warn("Could not install specs for %s." %
self.spec.format('$_$@'))
self.spec.format('$_$@'))
return
gcc = Executable(join_path(self.prefix.bin, 'gcc'))
@ -149,5 +145,5 @@ def write_rpath_specs(self):
out.write(line + "\n")
if line.startswith("*link:"):
out.write("-rpath %s/lib:%s/lib64 \\\n" %
(self.prefix, self.prefix))
(self.prefix, self.prefix))
set_install_permissions(specs_file)

79
var/spack/repos/builtin/packages/go/package.py
View File

@ -1,6 +1,9 @@
import os
import shutil
import glob
from spack import *
class Go(Package):
"""The golang compiler and build environment"""
homepage = "https://golang.org"
@ -10,43 +13,65 @@ class Go(Package):
# temporary fix until tags are pulled correctly
version('1.4.2', git='https://go.googlesource.com/go', tag='go1.4.2')
version('1.5.0', git='https://go.googlesource.com/go', tag='go1.5.0')
version('1.5.4', git='https://go.googlesource.com/go', tag='go1.5.4')
version('1.6.2', git='https://go.googlesource.com/go', tag='go1.6.2')
variant('test',
default=True,
description="Run tests as part of build, a good idea but quite"
" time consuming")
provides('go_compiler')
# to-do, make non-c self-hosting compilers possible
provides('golang')
# to-do, make non-c self-hosting compilers feasible without backflips
# should be go_compiler, but that creates an infinite loop
depends_on('gcc', when='@1.5:')
depends_on('gcc@5:', when='@1.5:')
depends_on('git')
def install(self, spec, prefix):
os.environ['GOROOT'] = os.getcwd()
os.environ['GOROOT_FINAL'] = prefix
bash = which('bash')
bash('-c', 'env')
bash('-c', 'pwd')
with working_dir('src'):
#TODO: crutch until the read-only-filesystem bug is fixed upstream
bash('all.bash', fail_on_error=False)
cp = which('cp')
bash('-c', 'cp -r ./* "{}"'.format(prefix))
bash = which('bash')
with working_dir('src'):
if '+test' in spec:
bash('all.bash')
else:
bash('make.bash')
def setup_dependent_environment(self, module, spec, ext_spec):
try:
os.makedirs(prefix)
except OSError:
pass
for f in glob.glob('*'):
if os.path.isdir(f):
shutil.copytree(f, os.path.join(prefix, f))
else:
shutil.copy2(f, os.path.join(prefix, f))
def setup_environment(self, spack_env, run_env):
# spack_env.set("GOROOT", self.spec.prefix)
# run_env.set("GOROOT", self.spec.prefix)
spack_env.set('GOROOT_FINAL', self.spec.prefix)
spack_env.set('GOROOT_BOOTSTRAP', self.spec['gcc'].prefix)
def setup_dependent_package(self, module, ext_spec):
# Add a go command/compiler for extensions
module.go = Executable(join_path(self.spec.prefix.bin, 'go'))
def setup_dependent_environment(self, spack_env, run_env, ext_spec):
"""Called before go modules' install() methods.
In most cases, extensions will only need to have one line::
go('get', '<package>')
go('get', '<package>')
"""
# Add a go command for extensions
module.go = Executable(join_path(spec.prefix.bin, 'go'))
os.environ['GOROOT'] = spec.prefix
stage_path = os.path.realpath(ext_spec.package.stage.source_path)
print "PREFIX: {}".format(stage_path)
go_paths = [stage_path]
for d in ext_spec.traverse():
if d.package.extends(self.spec):
go_paths.append(d.prefix)
os.environ['GOPATH'] = ':'.join(go_paths)
# spack_env.set("GOROOT", self.spec.prefix)
# run_env.set("GOROOT", self.spec.prefix)
# stage_path = os.path.realpath(ext_spec.package.stage.source_path)
# print "PREFIX: {}".format(stage_path)
# go_paths = [stage_path]
# for d in ext_spec.traverse():
# if d.package.extends(self.spec):
# go_paths.append(d.prefix)
# spack_env.prepend_path('GOPATH', ':'.join(go_paths))
spack_env.set('GOPATH', ext_spec.package.stage.source_path)

19
var/spack/repos/builtin/packages/the_platinum_searcher/package.py
View File

@ -1,19 +1,20 @@
from spack import *
import os
import shutil
class ThePlatinumSearcher(Package):
"""Fast parallel recursive grep alternative"""
# FIXME: add a proper url for your package's homepage here.
homepage = "https://github.com/monochromegane/the_platinum_searcher"
url = "https://github.com/monochromegane/the_platinum_searcher/archive/v1.7.7.tar.gz"
url = "https://github.com/monochromegane/the_platinum_searcher"
version('1.7.7', '08d7265e101bc1427d5d4b9903aa1166')
package = 'github.com/monochromegane/the_platinum_searcher/...'
depends_on("go")
version('head', go=package)
extends("go")
def install(self, spec, prefix):
env = which('env')
env()
# Fetch all dependencies
go('get', './...')
# Build pt
go('build')
go('install', self.package)
shutil.copytree('bin', os.path.join(prefix, 'bin'))