zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

go: refactor bootstrapping process (#35823)

Browse Source 
* Refactor go bootstrapping to include binary or gcc bootstrap
This commit is contained in:
Alec Scott 2023-03-10 16:27:49 -08:00 committed by GitHub
parent 08d7f47278
commit f5228cf59c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 81 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etc/spack/defaults/packages.yaml
View File

@ -28,7 +28,7 @@ packages:
gl: [glx, osmesa] gl: [glx, osmesa]
glu: [mesa-glu, openglu] glu: [mesa-glu, openglu]
golang: [go, gcc] golang: [go, gcc]
go-external-or-gccgo-bootstrap: [go-bootstrap, gcc] go-or-gccgo-bootstrap: [go-bootstrap, gcc]
iconv: [libiconv] iconv: [libiconv]
ipp: [intel-ipp] ipp: [intel-ipp]
java: [openjdk, jdk, ibm-java] java: [openjdk, jdk, ibm-java]

47
var/spack/repos/builtin/packages/gcc/package.py
View File

@ -197,36 +197,33 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage):
depends_on("autogen@5.5.4:", type="test") depends_on("autogen@5.5.4:", type="test")
depends_on("guile@1.4.1:", type="test") depends_on("guile@1.4.1:", type="test")
# See https://golang.org/doc/install/gccgo#Releases # See https://go.dev/doc/install/gccgo#Releases
with when("languages=go"): with when("languages=go"):
provides("golang", when="@4.6:") provides("go-or-gccgo-bootstrap@:1.0", when="@4.7.1:")
provides("golang@:1", when="@4.7.1:") provides("go-or-gccgo-bootstrap@:1.2", when="@4.9:")
provides("golang@:1.1", when="@4.8:") provides("go-or-gccgo-bootstrap@:1.4", when="@5:")
provides("golang@:1.1.2", when="@4.8.2:") provides("go-or-gccgo-bootstrap@:1.6.1", when="@6:")
provides("go-or-gccgo-bootstrap@:1.8.1", when="@7:")
provides("go-or-gccgo-bootstrap@:1.10.1", when="@8:")
provides("go-or-gccgo-bootstrap@:1.12.2", when="@9:")
provides("go-or-gccgo-bootstrap@:1.14.6", when="@10:")
provides("go-or-gccgo-bootstrap@1.16.3:1.16.5", when="@11:")
provides("golang@:1.0", when="@4.7.1:")
provides("golang@:1.2", when="@4.9:") provides("golang@:1.2", when="@4.9:")
provides("golang@:1.4", when="@5:") provides("golang@:1.4", when="@5:")
provides("golang@:1.6.1", when="@6:") provides("golang@:1.6.1", when="@6:")
provides("golang@:1.8", when="@7:") provides("golang@:1.8.1", when="@7:")
provides("golang@:1.10", when="@8:") provides("golang@:1.10.1", when="@8:")
provides("golang@:1.12", when="@9:") provides("golang@:1.12.2", when="@9:")
provides("golang@:1.14", when="@10:") provides("golang@:1.14.6", when="@10:")
provides("golang@:1.16", when="@11:") provides("golang@1.16.3:1.16.5", when="@11:")
provides("golang@:1.18", when="@11:")
# GCC 4.6 added support for the Go programming language. # GCC 4.7.1 added full support for the Go 1.x programming language.
# See https://gcc.gnu.org/gcc-4.6/changes.html conflicts("@:4.7.0")
conflicts("@:4.5", msg="support for Go has been added in GCC 4.6")
# aarch64 machines (including Macs with Apple silicon) can't use
# go-bootstrap because it pre-dates aarch64 support in Go. When not
# using an external go bootstrap go, These machines have to rely on
# Go support in gcc (which may require compiling a version of gcc
# with Go support just to satisfy this requirement). However,
# there's also a bug in some versions of GCC's Go front-end that prevents
# these versions from properly bootstrapping Go. (See issue #47771
# https://github.com/golang/go/issues/47771 ) On the 10.x branch, we need
# at least 10.4. On the 11.x branch, we need at least 11.3:
provides("go-external-or-gccgo-bootstrap", when="gcc@10.4.0:10,11.3.0:target=aarch64:")
# Go is not supported on macOS # Go is not supported on macOS
conflicts("platform=darwin", msg="Go not supported on MacOS") conflicts("platform=darwin", msg="GCC cannot build Go support on MacOS")
# For a list of valid languages for a specific release, # For a list of valid languages for a specific release,
# run the following command in the GCC source directory: # run the following command in the GCC source directory:

109
var/spack/repos/builtin/packages/go-bootstrap/package.py
View File

@ -3,6 +3,7 @@
# #
# SPDX-License-Identifier: (Apache-2.0 OR MIT) # SPDX-License-Identifier: (Apache-2.0 OR MIT)
import platform
import re import re
from spack.package import * from spack.package import *
@ -18,50 +19,54 @@ class GoBootstrap(Package):
"""Old C-bootstrapped go to bootstrap real go""" """Old C-bootstrapped go to bootstrap real go"""
homepage = "https://golang.org" homepage = "https://golang.org"
url = "https://go.dev/dl/go1.20.1.darwin-amd64.tar.gz"
extendable = True extendable = True
# NOTE: Go@1.4.x is the only supported bootstrapping compiler because all maintainers("alecbcs")
# later versions require a Go compiler to build.
# See: https://golang.org/doc/install/source#go14 and
# https://github.com/golang/go/issues/17545 and
# https://github.com/golang/go/issues/16352
version(
"1.4-bootstrap-20171003",
sha256="f4ff5b5eb3a3cae1c993723f3eab519c5bae18866b5e5f96fe1102f0cb5c3e52",
url="https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz",
)
version(
"1.4-bootstrap-20170531",
sha256="49f806f66762077861b7de7081f586995940772d29d4c45068c134441a743fa2",
url="https://storage.googleapis.com/golang/go1.4-bootstrap-20170531.tar.gz",
)
version(
"1.4-bootstrap-20161024",
sha256="398c70d9d10541ba9352974cc585c43220b6d8dbcd804ba2c9bd2fbf35fab286",
url="https://storage.googleapis.com/golang/go1.4-bootstrap-20161024.tar.gz",
)
provides("golang@:1.4-bootstrap-20171003")
depends_on("git", type=("build", "link", "run")) depends_on("git", type=("build", "link", "run"))
conflicts(
"os=monterey",
msg="go-bootstrap won't build on MacOS Monterey: "
"try `brew install go` and `spack external find go`",
)
conflicts("target=aarch64:", msg="Go bootstrap doesn't support aarch64 architectures")
# This virtual package allows a fallback to gccgo for aarch64,
# where go-bootstrap cannot be built(aarch64 was added with Go 1.5)
provides("go-external-or-gccgo-bootstrap")
# Support for aarch64 was added in Go 1.5, use an external package or gccgo instead:
conflicts("@:1.4", when="target=aarch64:")
executables = ["^go$"] executables = ["^go$"]
# List binary go releases for multiple operating systems and architectures.
# These binary versions are not intended to stay up-to-date. Instead we
# should update these binary releases on a yearly schedule as
# bootstrapping requirements are modified by new releases of go.
go_releases = {
"1.17.13": {
"darwin": {
"amd64": "c101beaa232e0f448fab692dc036cd6b4677091ff89c4889cc8754b1b29c6608",
"arm64": "e4ccc9c082d91eaa0b866078b591fc97d24b91495f12deb3dd2d8eda4e55a6ea",
},
"linux": {
"amd64": "4cdd2bc664724dc7db94ad51b503512c5ae7220951cac568120f64f8e94399fc",
"arm64": "914daad3f011cc2014dea799bb7490442677e4ad6de0b2ac3ded6cee7e3f493d",
"ppc64le": "bd0763fb130f8412672ffe1e4a8e65888ebe2419e5caa9a67ac21e8c298aa254",
},
}
}
# Normalize architectures returned by platform to those used by the
# Go project.
go_targets = {
"aarch64": "arm64",
"arm64": "arm64",
"ppc64le": "ppc64le",
"amd64": "amd64",
"x86_64": "amd64",
}
# determine system os and architecture/target
os = platform.system().lower()
target = go_targets[platform.machine().lower()]
# construct releases for current system configuration
for release in go_releases:
if os in go_releases[release] and target in go_releases[release][os]:
version(release, sha256=go_releases[release][os][target])
provides(f"go-or-gccgo-bootstrap@{release}", when=f"@{release}")
# When the user adds a go compiler using ``spack external find go-bootstrap``, # When the user adds a go compiler using ``spack external find go-bootstrap``,
# this lets us get the version for packages.yaml. Then, the solver can avoid # this lets us get the version for packages.yaml. Then, the solver can avoid
# to build the bootstrap go compiler(for aarch64, it's only gccgo) from source: # to build the bootstrap go compiler(for aarch64, it's only gccgo) from source:
@ -72,25 +77,22 @@ def determine_version(cls, exe):
match = re.search(r"go version go(\S+)", output) match = re.search(r"go version go(\S+)", output)
return match.group(1) if match else None return match.group(1) if match else None
def patch(self): def url_for_version(self, version):
if self.spec.satisfies("@:1.4.3"): # allow maintainers to checksum multiple architectures via
# NOTE: Older versions of Go attempt to download external files that have # `spack checksum go-bootstrap@1.18.9-darwin-arm64`
# since been moved while running the test suite. This patch modifies the match = re.search(r"(\S+)-(\S+)-(\S+)", str(version))
# test files so that these tests don't cause false failures. if match:
# See: https://github.com/golang/go/issues/15694 version = match.group(1)
test_suite_file = FileFilter(join_path("src", "run.bash")) os = match.group(2)
test_suite_file.filter(r"^(.*)(\$GOROOT/src/cmd/api/run.go)(.*)$", r"# \1\2\3") target = match.group(3)
else:
os = self.os
target = self.target
# Go uses a hardcoded limit of 4096 bytes for its printf functions. url = "https://go.dev/dl/go{0}.{1}-{2}.tar.gz"
# This can cause environment variables to be truncated. return url.format(version, os, target)
filter_file("char buf[4096];", "char buf[131072];", "src/cmd/dist/unix.c", string=True)
def install(self, spec, prefix): def install(self, spec, prefix):
env["CGO_ENABLED"] = "0"
bash = which("bash")
with working_dir("src"):
bash("{0}.bash".format("all" if self.run_tests else "make"))
install_tree(".", prefix) install_tree(".", prefix)
def setup_dependent_build_environment(self, env, dependent_spec): def setup_dependent_build_environment(self, env, dependent_spec):
@ -101,6 +103,3 @@ def setup_dependent_build_environment(self, env, dependent_spec):
else: else:
goroot = self.spec.prefix goroot = self.spec.prefix
env.set("GOROOT_BOOTSTRAP", goroot) env.set("GOROOT_BOOTSTRAP", goroot)
def setup_build_environment(self, env):
env.set("GOROOT_FINAL", self.spec.prefix)

49
var/spack/repos/builtin/packages/go/package.py
View File

@ -4,7 +4,6 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT) # SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os import os
import platform
import re import re
import llnl.util.tty as tty import llnl.util.tty as tty
@ -26,10 +25,6 @@
# - on CentOS 7 systems (and possibly others) you need to have the # - on CentOS 7 systems (and possibly others) you need to have the
# glibc package installed or various static cgo tests fail. # glibc package installed or various static cgo tests fail.
# #
# - When building on a *large* machine (144 cores, 1.5TB RAM) I need
# to run `ulimit -u 8192` to bump up the max number of user processes.
# Failure to do so results in an explosion in one of the tests and an
# epic stack trace....
class Go(Package): class Go(Package):
@ -57,29 +52,8 @@ class Go(Package):
provides("golang") provides("golang")
depends_on("git", type=("build", "link", "run")) depends_on("git", type=("build", "link", "run"))
depends_on("go-or-gccgo-bootstrap", type="build")
# aarch64 machines (including Macs with Apple silicon) can't use depends_on("go-or-gccgo-bootstrap@1.17.13:", type="build", when="@1.20:")
# go-bootstrap because it pre-dates aarch64 support in Go. These machines
# have to rely on Go support in gcc (which may require compiling a version
# of gcc with Go support just to satisfy this requirement) or external go:
# #27769: On M1/MacOS, platform.machine() may return arm64:
if platform.machine() in ["arm64", "aarch64"]:
# Use an external go compiler from packages.yaml/`spack external find go-bootstrap`,
# but fallback to build go-bootstrap@1.4 or to gcc with languages=go (for aarch64):
depends_on("go-external-or-gccgo-bootstrap", type="build")
else:
depends_on("go-bootstrap", type="build")
# https://github.com/golang/go/issues/17545
patch("time_test.patch", when="@1.6.4:1.7.4")
# https://github.com/golang/go/issues/17986
# The fix for this issue has been merged into the 1.8 tree.
patch("misc-cgo-testcshared.patch", level=0, when="@1.6.4:1.7.5")
# Unrecognized option '-fno-lto'
conflicts("%gcc@:4", when="@1.17:")
@classmethod @classmethod
def determine_version(cls, exe): def determine_version(cls, exe):
@ -87,28 +61,13 @@ def determine_version(cls, exe):
match = re.search(r"go version go(\S+)", output) match = re.search(r"go version go(\S+)", output)
return match.group(1) if match else None return match.group(1) if match else None
# NOTE: Older versions of Go attempt to download external files that have
# since been moved while running the test suite. This patch modifies the
# test files so that these tests don't cause false failures.
# See: https://github.com/golang/go/issues/15694
@when("@:1.4.3")
def patch(self):
test_suite_file = FileFilter(join_path("src", "run.bash"))
test_suite_file.filter(r"^(.*)(\$GOROOT/src/cmd/api/run.go)(.*)$", r"# \1\2\3")
def install(self, spec, prefix): def install(self, spec, prefix):
bash = which("bash") bash = which("bash")
wd = "." with working_dir("src"):
# 1.11.5 directory structure is slightly different
if self.version == Version("1.11.5"):
wd = "go"
with working_dir(join_path(wd, "src")):
bash("{0}.bash".format("all" if self.run_tests else "make")) bash("{0}.bash".format("all" if self.run_tests else "make"))
install_tree(wd, prefix) install_tree(".", prefix)
def setup_build_environment(self, env): def setup_build_environment(self, env):
env.set("GOROOT_FINAL", self.spec.prefix) env.set("GOROOT_FINAL", self.spec.prefix)