zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docker: fix builds, remove extra distros, add ci builds (#11621)

Browse Source 
* fix docker builds/remove extra builds/add ci builds
* preinstall vim in CI builder images
* simplify & streamline docker resources
* restore os-container-mapping.yaml file
This commit is contained in:
Omar Padron 2019-07-20 17:46:06 -04:00 committed by Todd Gamblin
parent 7d954720ff
commit 1d5ab13be8
26 changed files with 383 additions and 643 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

177
share/spack/docker/Dockerfile
View File

@ -1,177 +0,0 @@
ARG BASE
FROM $BASE
MAINTAINER Spack Maintainers <maintainers@spack.io>
ARG BASE
ARG DISTRO
ARG DISTRO_VERSION
ENV DOCKERFILE_BASE=$BASE \
DOCKERFILE_DISTRO=$DISTRO \
DOCKERFILE_DISTRO_VERSION=$DISTRO_VERSION \
SPACK_ROOT=/opt/spack \
FORCE_UNSAFE_CONFIGURE=1 \
DEBIAN_FRONTEND=noninteractive \
CURRENTLY_BUILDING_DOCKER_IMAGE=1 \
container=docker
COPY bin $SPACK_ROOT/bin
COPY etc $SPACK_ROOT/etc
COPY lib $SPACK_ROOT/lib
COPY share $SPACK_ROOT/share
COPY var $SPACK_ROOT/var
RUN mkdir -p $SPACK_ROOT/opt/spack
MASK PUSH
MASK [[ $DISTRO == arch ]]
RUN pacman -Syu --noconfirm \
&& pacman -Sy --noconfirm \
base-devel ca-certificates curl gcc \
gcc-fortran git gnupg2 inetutils \
iproute2 make openssh procps-ng \
python python-pip sudo tcl \
unzip \
&& echo 'nobody ALL=(ALL) NOPASSWD: ALL' > \
/etc/sudoers.d/nobody-sudo \
&& sudo -u nobody git clone \
'https://aur.archlinux.org/lua-std-_debug.git' \
'/tmp/lua-std-_debug' \
&& sudo -u nobody git clone \
'https://aur.archlinux.org/lua-std-normalize.git' \
'/tmp/lua-std-normalize' \
&& sudo -u nobody git clone \
'https://aur.archlinux.org/lua-posix.git' \
'/tmp/lua-posix' \
&& ( cd /tmp/lua-std-_debug \
&& sudo -u nobody makepkg -si --asdeps --noconfirm ) \
&& ( cd /tmp/lua-std-normalize \
&& sudo -u nobody makepkg -si --asdeps --noconfirm ) \
&& ( cd /tmp/lua-posix \
&& sudo -u nobody makepkg -si --asdeps --noconfirm ) \
&& sudo -u nobody git clone \
'https://aur.archlinux.org/lmod.git' '/tmp/lmod' \
&& ( cd /tmp/lmod \
&& sudo -u nobody makepkg -si --noconfirm ) \
&& rm -rf /tmp/lua-std-_debug /tmp/lua-std-normalize \
/tmp/lmod /etc/sudoers.d/nobody-sudo
MASK [[ $DISTRO =~ (centos|rhel.*) ]]
RUN yum update -y
MASK PUSH
MASK [[ $DISTRO =~ rhel.* ]]
RUN ( yum install -y yum-conf-repos.noarch \
|| yum install -y yum-conf-epel.noarch yum-conf-elrepo.noarch ) \
&& yum update -y
MASK POP
RUN yum install -y epel-release \
&& yum update -y \
&& yum --enablerepo epel groupinstall -y "Development Tools" \
&& yum --enablerepo epel install -y \
curl findutils gcc-c++ gcc \
gcc-gfortran git gnupg2 hostname \
iproute Lmod make patch \
openssh-server python python-pip tcl \
unzip \
&& rm -rf /var/cache/yum \
&& yum clean all
MASK [[ $DISTRO == fedora ]]
RUN dnf update -y \
&& dnf group install -y "C Development Tools and Libraries" \
&& dnf install -y \
@development-tools \
curl findutils gcc-c++ gcc \
gcc-gfortran git gnupg2 hostname \
iproute Lmod make patch \
openssh-server python tcl unzip \
&& dnf clean all
MASK [[ $DISTRO == opensuse ]]
RUN zypper -n ref \
&& zypper -n up --skip-interactive --no-recommends \
&& zypper -n install -l --no-recommends --type pattern \
devel_basis devel_C_C++ \
&& zypper -n install -l --no-recommends \
bash bash-completion ca-certificates curl \
findutils gcc gcc-locale gcc-c++ \
gcc-fortran git glibc-locale gpg2 \
hostname iproute lua-lmod make \
patch openssh python python-pip \
python-xml tcl unzip \
&& zypper clean \
&& rm -rf /var/cache/zypp/*
MASK [[ $DISTRO == ubuntu ]]
RUN apt-get -yqq update \
&& apt-get -yqq install \
build-essential ca-certificates curl g++ \
gcc gfortran git gnupg2 \
iproute2 lmod lua-posix make \
openssh-server python python-pip tcl \
unzip
MASK PUSH
MASK [[ $DISTRO_VERSION == 18.04 ]]
# [WORKAROUND]
# https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
RUN ln -s posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
MASK POP
RUN rm -rf /var/lib/apt/lists/*
MASK POP
RUN rm -rf $SPACK_ROOT/.git \
&& pip install boto3 \
&& ( echo ". /usr/share/lmod/lmod/init/bash" \
&& echo ". \\\$SPACK_ROOT/share/spack/setup-env.sh" \
&& echo "if [ \\\"\\\$CURRENTLY_BUILDING_DOCKER_IMAGE\\\" '!=' '1' ]" \
&& echo "then" \
&& echo " . \\\$SPACK_ROOT/share/spack/spack-completion.bash" \
&& echo "fi" ) \
>> /etc/profile.d/spack.sh \
&& ( echo "f=\\\"\\\$SPACK_ROOT/share/spack/docker/handle-ssh.sh\\\"" \
&& echo "if [ -f \\\"\\\$f\\\" ]" \
&& echo "then" \
&& echo " . \\\"\\\$f\\\"" \
&& echo "else" \
&& cat $SPACK_ROOT/share/spack/docker/handle-ssh.sh \
&& echo "fi" ) \
>> /etc/profile.d/handle-ssh.sh \
&& ( echo "f=\\\"\\\$SPACK_ROOT/share/spack/docker/handle-prompt.sh\\\"" \
&& echo "if [ -f \\\"\\\$f\\\" ]" \
&& echo "then" \
&& echo " . \\\"\\\$f\\\"" \
&& echo "else" \
&& cat $SPACK_ROOT/share/spack/docker/handle-prompt.sh \
&& echo "fi" ) \
>> /etc/profile.d/handle-prompt.sh \
&& mkdir -p /root/.spack \
&& cp $SPACK_ROOT/share/spack/docker/modules.yaml \
/root/.spack/modules.yaml \
&& rm -rf /root/*.*
MASK PUSH
MASK [[ $DISTRO_VERSION =~ (centos|fedora|opensuse|rhel.*) ]]
RUN rm -f /run/nologin
MASK POP
# [WORKAROUND]
# https://superuser.com/questions/1241548/
# xubuntu-16-04-ttyname-failed-inappropriate-ioctl-for-device#1253889
RUN [ -f ~/.profile ] \
&& sed -i 's/mesg n/( tty -s \\&\\& mesg n || true )/g' ~/.profile \
|| true
WORKDIR /root
SHELL ["/bin/bash", "-l", "-c"]
# TODO: add a command to Spack that (re)creates the package cache
RUN spack spec hdf5+mpi
ENTRYPOINT ["/bin/bash", "/opt/spack/share/spack/docker/entrypoint.bash"]
CMD ["docker-shell"]

84
share/spack/docker/build-image.sh
View File

@ -1,84 +0,0 @@
#! /usr/bin/env bash
#
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
script="$( basename "$0" )"
cd "$( dirname "$0" )"
if [ -z "$BASE_IMAGE" ] ; then
BASE_IMAGE="ubuntu"
fi
if [ -z "$BASE_TAG" ] ; then
BASE_TAG="latest"
fi
if [ -z "$DISTRO" ] ; then
DISTRO="${BASE_IMAGE}"
fi
if [ -z "$DISTRO_VERSION" ] ; then
DISTRO_VERSION="${BASE_TAG}"
fi
if [ -z "$BASE_NAME" ] ; then
BASE_NAME="${DISTRO}"
fi
if [ "$BASE_TAG" '=' 'latest' ] ; then
BASE_TAG=""
fi
if [ -n "$BASE_TAG" ] ; then
BASE_TAG=":${BASE_TAG}"
fi
TAG="spack/${BASE_NAME}${BASE_TAG}"
export BASE_IMAGE BASE_TAG DISTRO DISTRO_VERSION BASE_NAME TAG
if [ "$script" '=' 'run-image.sh' ] ; then
com="docker run --rm -ti"
if [ -z "$DISABLE_MOUNT" ] ; then
DISABLE_MOUNT=1
if [ -z "$*" ] ; then
DISABLE_MOUNT=0
fi
fi
if [ "$DISABLE_MOUNT" '==' 0 ] ; then
com="${com} -v \"$( readlink -f ../../.. ):/spack\""
fi
eval "exec ${com}" "${TAG}" "$@"
elif [ "$script" '=' 'render-image-template.sh' ] ; then
./dpp.bash Dockerfile
elif [ "$script" '=' 'push-image.sh' ] ; then
docker push "${TAG}"
for tag in ${EXTRA_TAGS} ; do
docker push "spack/${BASE_NAME}:${tag}"
done
else
tag_options="-t ${TAG}"
for tag in ${EXTRA_TAGS} ; do
tag_options="${tag_options} -t spack/${BASE_NAME}:${tag}"
done
cache_options=""
if docker pull "${TAG}" ; then
cache_options="--cache-from ${TAG}"
fi
exec ./render-image-template.sh |
docker build -f - \
${cache_options} \
${tag_options} \
--build-arg BASE="${BASE_IMAGE}${BASE_TAG}" \
--build-arg DISTRO="${DISTRO}" \
--build-arg DISTRO_VERSION="${DISTRO_VERSION}" \
../../..
fi

76
share/spack/docker/centos-6.dockerfile Normal file
View File

@ -0,0 +1,76 @@
FROM centos:6
MAINTAINER Spack Maintainers <maintainers@spack.io>
ENV DOCKERFILE_BASE=centos \
DOCKERFILE_DISTRO=centos \
DOCKERFILE_DISTRO_VERSION=6 \
SPACK_ROOT=/opt/spack \
FORCE_UNSAFE_CONFIGURE=1 \
DEBIAN_FRONTEND=noninteractive \
CURRENTLY_BUILDING_DOCKER_IMAGE=1 \
container=docker
COPY bin $SPACK_ROOT/bin
COPY etc $SPACK_ROOT/etc
COPY lib $SPACK_ROOT/lib
COPY share $SPACK_ROOT/share
COPY var $SPACK_ROOT/var
RUN mkdir -p $SPACK_ROOT/opt/spack
RUN yum update -y \
&& yum install -y epel-release \
&& yum update -y \
&& yum --enablerepo epel groupinstall -y "Development Tools" \
&& yum --enablerepo epel install -y \
curl findutils gcc-c++ gcc \
gcc-gfortran git gnupg2 hostname \
iproute Lmod make patch \
openssh-server python python-pip tcl \
unzip which \
&& pip install boto3 \
&& rm -rf /var/cache/yum \
&& yum clean all
RUN ( echo ". /usr/share/lmod/lmod/init/bash" \
&& echo ". \$SPACK_ROOT/share/spack/setup-env.sh" \
&& echo "if [ \"\$CURRENTLY_BUILDING_DOCKER_IMAGE\" '!=' '1' ]" \
&& echo "then" \
&& echo " . \$SPACK_ROOT/share/spack/spack-completion.bash" \
&& echo "fi" ) \
>> /etc/profile.d/spack.sh \
&& ( echo "f=\"\$SPACK_ROOT/share/spack/docker/handle-ssh.sh\"" \
&& echo "if [ -f \"\$f\" ]" \
&& echo "then" \
&& echo " . \"\$f\"" \
&& echo "else" \
&& cat $SPACK_ROOT/share/spack/docker/handle-ssh.sh \
&& echo "fi" ) \
>> /etc/profile.d/handle-ssh.sh \
&& ( echo "f=\"\$SPACK_ROOT/share/spack/docker/handle-prompt.sh\"" \
&& echo "if [ -f \"\$f\" ]" \
&& echo "then" \
&& echo " . \"\$f\"" \
&& echo "else" \
&& cat $SPACK_ROOT/share/spack/docker/handle-prompt.sh \
&& echo "fi" ) \
>> /etc/profile.d/handle-prompt.sh \
&& mkdir -p /root/.spack \
&& cp $SPACK_ROOT/share/spack/docker/modules.yaml \
/root/.spack/modules.yaml \
&& rm -rf /root/*.* /run/nologin $SPACK_ROOT/.git
# [WORKAROUND]
# https://superuser.com/questions/1241548/
# xubuntu-16-04-ttyname-failed-inappropriate-ioctl-for-device#1253889
RUN [ -f ~/.profile ] \
&& sed -i 's/mesg n/( tty -s \\&\\& mesg n || true )/g' ~/.profile \
|| true
WORKDIR /root
SHELL ["/bin/bash", "-l", "-c"]
# TODO: add a command to Spack that (re)creates the package cache
RUN spack spec hdf5+mpi
ENTRYPOINT ["/bin/bash", "/opt/spack/share/spack/docker/entrypoint.bash"]
CMD ["docker-shell"]

76
share/spack/docker/centos-7.dockerfile Normal file
View File

@ -0,0 +1,76 @@
FROM centos:7
MAINTAINER Spack Maintainers <maintainers@spack.io>
ENV DOCKERFILE_BASE=centos \
DOCKERFILE_DISTRO=centos \
DOCKERFILE_DISTRO_VERSION=7 \
SPACK_ROOT=/opt/spack \
FORCE_UNSAFE_CONFIGURE=1 \
DEBIAN_FRONTEND=noninteractive \
CURRENTLY_BUILDING_DOCKER_IMAGE=1 \
container=docker
COPY bin $SPACK_ROOT/bin
COPY etc $SPACK_ROOT/etc
COPY lib $SPACK_ROOT/lib
COPY share $SPACK_ROOT/share
COPY var $SPACK_ROOT/var
RUN mkdir -p $SPACK_ROOT/opt/spack
RUN yum update -y \
&& yum install -y epel-release \
&& yum update -y \
&& yum --enablerepo epel groupinstall -y "Development Tools" \
&& yum --enablerepo epel install -y \
curl findutils gcc-c++ gcc \
gcc-gfortran git gnupg2 hostname \
iproute Lmod make patch \
openssh-server python python-pip tcl \
unzip which \
&& pip install boto3 \
&& rm -rf /var/cache/yum \
&& yum clean all
RUN ( echo ". /usr/share/lmod/lmod/init/bash" \
&& echo ". \$SPACK_ROOT/share/spack/setup-env.sh" \
&& echo "if [ \"\$CURRENTLY_BUILDING_DOCKER_IMAGE\" '!=' '1' ]" \
&& echo "then" \
&& echo " . \$SPACK_ROOT/share/spack/spack-completion.bash" \
&& echo "fi" ) \
>> /etc/profile.d/spack.sh \
&& ( echo "f=\"\$SPACK_ROOT/share/spack/docker/handle-ssh.sh\"" \
&& echo "if [ -f \"\$f\" ]" \
&& echo "then" \
&& echo " . \"\$f\"" \
&& echo "else" \
&& cat $SPACK_ROOT/share/spack/docker/handle-ssh.sh \
&& echo "fi" ) \
>> /etc/profile.d/handle-ssh.sh \
&& ( echo "f=\"\$SPACK_ROOT/share/spack/docker/handle-prompt.sh\"" \
&& echo "if [ -f \"\$f\" ]" \
&& echo "then" \
&& echo " . \"\$f\"" \
&& echo "else" \
&& cat $SPACK_ROOT/share/spack/docker/handle-prompt.sh \
&& echo "fi" ) \
>> /etc/profile.d/handle-prompt.sh \
&& mkdir -p /root/.spack \
&& cp $SPACK_ROOT/share/spack/docker/modules.yaml \
/root/.spack/modules.yaml \
&& rm -rf /root/*.* /run/nologin $SPACK_ROOT/.git
# [WORKAROUND]
# https://superuser.com/questions/1241548/
# xubuntu-16-04-ttyname-failed-inappropriate-ioctl-for-device#1253889
RUN [ -f ~/.profile ] \
&& sed -i 's/mesg n/( tty -s \\&\\& mesg n || true )/g' ~/.profile \
|| true
WORKDIR /root
SHELL ["/bin/bash", "-l", "-c"]
# TODO: add a command to Spack that (re)creates the package cache
RUN spack spec hdf5+mpi
ENTRYPOINT ["/bin/bash", "/opt/spack/share/spack/docker/entrypoint.bash"]
CMD ["docker-shell"]

16
share/spack/docker/config/arch.bash
View File

@ -1,16 +0,0 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
unset DISTRO
unset DISTRO_VERSION
unset BASE_IMAGE
unset BASE_NAME
unset BASE_TAG
unset TAG
unset EXTRA_TAGS
export BASE_IMAGE="archlinux/base"
export BASE_NAME="archlinux"
export DISTRO="arch"

16
share/spack/docker/config/centos-6.bash
View File

@ -1,16 +0,0 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
unset DISTRO
unset DISTRO_VERSION
unset BASE_IMAGE
unset BASE_NAME
unset BASE_TAG
unset TAG
unset EXTRA_TAGS
export BASE_IMAGE=centos
export BASE_TAG="6"
export EXTRA_TAGS="centos6"

16
share/spack/docker/config/centos-7.bash
View File

@ -1,16 +0,0 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
unset DISTRO
unset DISTRO_VERSION
unset BASE_IMAGE
unset BASE_NAME
unset BASE_TAG
unset TAG
unset EXTRA_TAGS
export BASE_IMAGE=centos
export BASE_TAG="7"
export EXTRA_TAGS="centos7 latest"

15
share/spack/docker/config/fedora-24.bash
View File

@ -1,15 +0,0 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
unset DISTRO
unset DISTRO_VERSION
unset BASE_IMAGE
unset BASE_NAME
unset BASE_TAG
unset TAG
unset EXTRA_TAGS
export BASE_IMAGE=fedora
export BASE_TAG="24"

16
share/spack/docker/config/fedora-29.bash
View File

@ -1,16 +0,0 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
unset DISTRO
unset DISTRO_VERSION
unset BASE_IMAGE
unset BASE_NAME
unset BASE_TAG
unset TAG
unset EXTRA_TAGS
export BASE_IMAGE=fedora
export BASE_TAG="29"
export EXTRA_TAGS="latest"

15
share/spack/docker/config/opensuse-leap.bash
View File

@ -1,15 +0,0 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
unset DISTRO
unset DISTRO_VERSION
unset BASE_IMAGE
unset BASE_NAME
unset BASE_TAG
unset TAG
unset EXTRA_TAGS
export BASE_IMAGE=opensuse
export BASE_TAG="leap"

16
share/spack/docker/config/opensuse-tumbleweed.bash
View File

@ -1,16 +0,0 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
unset DISTRO
unset DISTRO_VERSION
unset BASE_IMAGE
unset BASE_NAME
unset BASE_TAG
unset TAG
unset EXTRA_TAGS
export BASE_IMAGE=opensuse
export BASE_TAG="tumbleweed"
export EXTRA_TAGS="latest"

16
share/spack/docker/config/scilinux-6.bash
View File

@ -1,16 +0,0 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
unset DISTRO
unset BASE_IMAGE
unset BASE_NAME
unset BASE_TAG
unset TAG
unset EXTRA_TAGS
export BASE_IMAGE=sl
export BASE_TAG="6"
export BASE_NAME=scilinux
export DISTRO=rhel6

17
share/spack/docker/config/scilinux-7.bash
View File

@ -1,17 +0,0 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
unset DISTRO
unset BASE_IMAGE
unset BASE_NAME
unset BASE_TAG
unset TAG
unset EXTRA_TAGS
export BASE_IMAGE=sl
export BASE_TAG="7"
export BASE_NAME=scilinux
export DISTRO=rhel7
export EXTRA_TAGS="latest"

15
share/spack/docker/config/ubuntu-1604.bash
View File

@ -1,15 +0,0 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
unset DISTRO
unset BASE_IMAGE
unset BASE_NAME
unset BASE_TAG
unset TAG
unset EXTRA_TAGS
export BASE_IMAGE=ubuntu
export BASE_TAG="16.04"
export EXTRA_TAGS="xenial"

15
share/spack/docker/config/ubuntu-1804.bash
View File

@ -1,15 +0,0 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
unset DISTRO
unset BASE_IMAGE
unset BASE_NAME
unset BASE_TAG
unset TAG
unset EXTRA_TAGS
export BASE_IMAGE=ubuntu
export BASE_TAG="18.04"
export EXTRA_TAGS="bionic latest"

77
share/spack/docker/dpp.bash
View File

@ -1,77 +0,0 @@
#! /usr/bin/env bash
#
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
function prefix_tokens() {
line="$1" ; shift
nprefix="$1"
line="${line::$nprefix} "
echo "${line::$nprefix}"
}
# read file contents, or stdin
cat "$1" |
# remove blank lines
grep -v '^ *$' |
# remove leading whitespace
sed 's/^ *//g' |
# remove comments
grep -v '^#.*' |
# remove trailing whitespace
sed 's/ *$//g' |
# remove extraneous whitespace
sed 's/ */ /g' |
# mask out subsections
(
stack_level=1
mask_level=1
while read LINE ; do
try_print=1
if [ "$( prefix_tokens "$LINE" 10 )" '=' 'MASK PUSH ' ] ; then
tmp="$stack_level"
stack_level="$(( stack_level + 1 ))"
if [ "$mask_level" '=' "$tmp" ] ; then
mask_level="$stack_level"
fi
try_print=0
elif [ "$( prefix_tokens "$LINE" 9 )" '=' 'MASK POP ' ] ; then
stack_level="$(( stack_level - 1 ))"
if [ "$mask_level" -gt "$stack_level" ] ; then
mask_level="$stack_level"
fi
try_print=0
elif [ "$( prefix_tokens "$LINE" 5 )" '=' 'MASK ' ] ; then
if [ "$(( mask_level + 1 ))" -ge "$stack_level" ] ; then
mask_level="$stack_level"
eval "${LINE:5}"
if [ "$?" '!=' 0 ] ; then
mask_level="$(( mask_level - 1 ))"
fi
fi
try_print=0
fi
if [ "$stack_level" -lt 1 ] ; then
stack_level=1
mask_level=0
fi
if [ "$try_print" '=' 1 -a "$mask_level" '=' "$stack_level" ] ; then
echo "$LINE"
fi
done
)

6
share/spack/docker/handle-prompt.sh
View File

@ -61,11 +61,11 @@ __write() {
} }
__revparse_head() { __revparse_head() {
head="`git -C /spack rev-parse $@ HEAD 2>/dev/null`" head="`git -C "$SPACK_ROOT" rev-parse $@ HEAD 2>/dev/null`"
result="$?" result="$?"
if [ "$result" '!=' '0' ] ; then if [ "$result" '!=' '0' ] ; then
head="`git --git-dir=/spack/.git \\ head="`git --git-dir="$SPACK_ROOT"/.git \\
--work-tree=/spack rev-parse $@ HEAD 2>/dev/null`" --work-tree="$SPACK_ROOT" rev-parse $@ HEAD 2>/dev/null`"
result="$?" result="$?"
fi fi

1
share/spack/docker/push-image.sh
View File

@ -1 +0,0 @@
./build-image.sh

1
share/spack/docker/render-image-template.sh
View File

@ -1 +0,0 @@
./build-image.sh

1
share/spack/docker/run-image.sh
View File

@ -1 +0,0 @@
build-image.sh

61
share/spack/docker/spack_builder/Dockerfile-spack_builder_centos_7
View File

@ -1,61 +0,0 @@
#
# To build this image:
#
# cd <path-to-spack-repo>/share/spack/docker/spack_builder
# docker build -f Dockerfile-spack_builder_centos_7 -t spack_builder_centos_7 .
#
from spack/centos:7
RUN yum update -y && \
yum install -y \
gmp-devel \
libmpc-devel \
mpfr-devel \
vim \
which && \
rm -rf /var/cache/yum && yum clean all
# Download, build and install gcc 5.5.0
RUN mkdir -p /home/spackuser/Download/gcc550/build-gcc550 && \
mkdir -p /opt/gcc/gcc-5.5.0 && \
cd /home/spackuser/Download/gcc550 && \
curl -OL https://ftp.gnu.org/gnu/gcc/gcc-5.5.0/gcc-5.5.0.tar.xz && \
tar -xvf gcc-5.5.0.tar.xz && \
cd build-gcc550 && \
../gcc-5.5.0/configure \
--enable-languages=c,c++,fortran \
--disable-multilib \
--prefix=/opt/gcc/gcc-5.5.0 && \
make -j$(nproc) && \
make install && \
cd /home/spackuser && \
rm -rf /home/spackuser/Download
RUN export PATH=/spack/bin:$PATH && \
spack compiler find /opt/gcc/gcc-5.5.0
RUN sed -i 's/f77: null/f77: \/opt\/gcc\/gcc-5.5.0\/bin\/gfortran/g;s/fc: null/fc: \/opt\/gcc\/gcc-5.5.0\/bin\/gfortran/g' ~/.spack/linux/compilers.yaml
RUN mkdir -p /home/spackuser/spackcommand
COPY update_rpaths.py /home/spackuser/spackcommand/update_rpaths.py
RUN spack python /home/spackuser/spackcommand/update_rpaths.py \
--prefix /opt/gcc/gcc-5.5.0 \
--rpaths /opt/gcc/gcc-5.5.0/lib64
RUN export PATH=/spack/bin:$PATH && \
spack install -y llvm@6.0.0%gcc@5.5.0 && \
spack clean -a
RUN export PATH=/spack/bin:$PATH && \
spack compiler find $(spack location -i llvm@6.0.0%gcc@5.5.0)
RUN sed -i 's/f77: null/f77: \/opt\/gcc\/gcc-5.5.0\/bin\/gfortran/g;s/fc: null/fc: \/opt\/gcc\/gcc-5.5.0\/bin\/gfortran/g' ~/.spack/linux/compilers.yaml
RUN spack python /home/spackuser/spackcommand/update_rpaths.py \
--prefix /spack/opt/spack/linux-centos7-x86_64/gcc-5.5.0/llvm-6.0.0-awfpo7kn3k24weu655rrt2erihzd4gii \
--rpaths /spack/opt/spack/linux-centos7-x86_64/gcc-5.5.0/llvm-6.0.0-awfpo7kn3k24weu655rrt2erihzd4gii/lib

23
share/spack/docker/spack_builder/Dockerfile-spack_builder_ubuntu_18.04
View File

@ -1,23 +0,0 @@
#
# To build this image:
#
# cd <path-to-spack-repo>/share/spack/docker/spack_builder
# docker build -f Dockerfile-spack_builder_ubuntu_18.04 -t spack_builder_ubuntu_18.04 .
#
from spack/ubuntu:bionic
RUN apt-get -yqq update && apt-get -yqq install \
clang \
g++-5 \
gcc-5 \
gfortran-5 \
unzip \
vim && \
rm -rf /var/lib/apt/lists/*
RUN export PATH=/spack/bin:$PATH && \
spack compiler find gcc clang
RUN sed -i 's/f77: null/f77: \/usr\/bin\/gfortran/g;s/fc: null/fc: \/usr\/bin\/gfortran/g' ~/.spack/linux/compilers.yaml

34
share/spack/docker/spack_builder/update_rpaths.py
View File

@ -1,34 +0,0 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import argparse
from spack.config import config as spack_config
def update_compiler(prefix, rpaths):
compilers_config = spack_config.get('compilers')
for compiler_entry in compilers_config:
if compiler_entry['compiler']['paths']['cc'].startswith(prefix):
print('found target compiler: {0}'.format(
compiler_entry['compiler']['spec']))
compiler_entry['compiler']['extra_rpaths'].append(rpaths)
spack_config.update_config('compilers', compilers_config)
if __name__ == "__main__":
# Create argument parser
parser = argparse.ArgumentParser(
description="Add extra_rpaths to default system compilers.yaml")
parser.add_argument('-p', '--prefix', default=None,
help="Install prefix of compiler to update")
parser.add_argument('-r', '--rpaths', default=None,
help="Extra rpaths to add to target compiler")
args = parser.parse_args()
update_compiler(args.prefix, args.rpaths)

73
share/spack/docker/ubuntu-1604.dockerfile Normal file
View File

@ -0,0 +1,73 @@
FROM ubuntu:16.04
MAINTAINER Spack Maintainers <maintainers@spack.io>
ENV DOCKERFILE_BASE=ubuntu:16.04 \
DOCKERFILE_DISTRO=ubuntu \
DOCKERFILE_DISTRO_VERSION=16.04 \
SPACK_ROOT=/opt/spack \
FORCE_UNSAFE_CONFIGURE=1 \
DEBIAN_FRONTEND=noninteractive \
CURRENTLY_BUILDING_DOCKER_IMAGE=1 \
container=docker
COPY bin $SPACK_ROOT/bin
COPY etc $SPACK_ROOT/etc
COPY lib $SPACK_ROOT/lib
COPY share $SPACK_ROOT/share
COPY var $SPACK_ROOT/var
RUN mkdir -p $SPACK_ROOT/opt/spack
RUN apt-get -yqq update \
&& apt-get -yqq install \
build-essential ca-certificates curl g++ \
gcc gfortran git gnupg2 \
iproute2 lmod lua-posix make \
openssh-server python python-pip tcl \
unzip \
&& pip install boto3 \
&& rm -rf /var/lib/apt/lists/*
RUN ( echo ". /usr/share/lmod/lmod/init/bash" \
&& echo ". \$SPACK_ROOT/share/spack/setup-env.sh" \
&& echo "if [ \"\$CURRENTLY_BUILDING_DOCKER_IMAGE\" '!=' '1' ]" \
&& echo "then" \
&& echo " . \$SPACK_ROOT/share/spack/spack-completion.bash" \
&& echo "fi" ) \
>> /etc/profile.d/spack.sh \
&& ( echo "f=\"\$SPACK_ROOT/share/spack/docker/handle-ssh.sh\"" \
&& echo "if [ -f \"\$f\" ]" \
&& echo "then" \
&& echo " . \"\$f\"" \
&& echo "else" \
&& cat $SPACK_ROOT/share/spack/docker/handle-ssh.sh \
&& echo "fi" ) \
>> /etc/profile.d/handle-ssh.sh \
&& ( echo "f=\"\$SPACK_ROOT/share/spack/docker/handle-prompt.sh\"" \
&& echo "if [ -f \"\$f\" ]" \
&& echo "then" \
&& echo " . \"\$f\"" \
&& echo "else" \
&& cat $SPACK_ROOT/share/spack/docker/handle-prompt.sh \
&& echo "fi" ) \
>> /etc/profile.d/handle-prompt.sh \
&& mkdir -p /root/.spack \
&& cp $SPACK_ROOT/share/spack/docker/modules.yaml \
/root/.spack/modules.yaml \
&& rm -rf /root/*.* $SPACK_ROOT/.git
# [WORKAROUND]
# https://superuser.com/questions/1241548/
# xubuntu-16-04-ttyname-failed-inappropriate-ioctl-for-device#1253889
RUN [ -f ~/.profile ] \
&& sed -i 's/mesg n/( tty -s \&\& mesg n || true )/g' ~/.profile \
|| true
WORKDIR /root
SHELL ["/bin/bash", "-l", "-c"]
# TODO: add a command to Spack that (re)creates the package cache
RUN spack spec hdf5+mpi
ENTRYPOINT ["/bin/bash", "/opt/spack/share/spack/docker/entrypoint.bash"]
CMD ["docker-shell"]

76
share/spack/docker/ubuntu-1804.dockerfile Normal file
View File

@ -0,0 +1,76 @@
FROM ubuntu:18.04
MAINTAINER Spack Maintainers <maintainers@spack.io>
ENV DOCKERFILE_BASE=ubuntu \
DOCKERFILE_DISTRO=ubuntu \
DOCKERFILE_DISTRO_VERSION=18.04 \
SPACK_ROOT=/opt/spack \
FORCE_UNSAFE_CONFIGURE=1 \
DEBIAN_FRONTEND=noninteractive \
CURRENTLY_BUILDING_DOCKER_IMAGE=1 \
container=docker
COPY bin $SPACK_ROOT/bin
COPY etc $SPACK_ROOT/etc
COPY lib $SPACK_ROOT/lib
COPY share $SPACK_ROOT/share
COPY var $SPACK_ROOT/var
RUN mkdir -p $SPACK_ROOT/opt/spack
RUN apt-get -yqq update \
&& apt-get -yqq install \
build-essential ca-certificates curl g++ \
gcc gfortran git gnupg2 \
iproute2 lmod lua-posix make \
openssh-server python python-pip tcl \
unzip \
&& pip install boto3 \
&& rm -rf /var/lib/apt/lists/*
RUN ( echo ". /usr/share/lmod/lmod/init/bash" \
&& echo ". \$SPACK_ROOT/share/spack/setup-env.sh" \
&& echo "if [ \"\$CURRENTLY_BUILDING_DOCKER_IMAGE\" '!=' '1' ]" \
&& echo "then" \
&& echo " . \$SPACK_ROOT/share/spack/spack-completion.bash" \
&& echo "fi" ) \
>> /etc/profile.d/spack.sh \
&& ( echo "f=\"\$SPACK_ROOT/share/spack/docker/handle-ssh.sh\"" \
&& echo "if [ -f \"\$f\" ]" \
&& echo "then" \
&& echo " . \"\$f\"" \
&& echo "else" \
&& cat $SPACK_ROOT/share/spack/docker/handle-ssh.sh \
&& echo "fi" ) \
>> /etc/profile.d/handle-ssh.sh \
&& ( echo "f=\"\$SPACK_ROOT/share/spack/docker/handle-prompt.sh\"" \
&& echo "if [ -f \"\$f\" ]" \
&& echo "then" \
&& echo " . \"\$f\"" \
&& echo "else" \
&& cat $SPACK_ROOT/share/spack/docker/handle-prompt.sh \
&& echo "fi" ) \
>> /etc/profile.d/handle-prompt.sh \
&& mkdir -p /root/.spack \
&& cp $SPACK_ROOT/share/spack/docker/modules.yaml \
/root/.spack/modules.yaml \
&& rm -rf /root/*.* $SPACK_ROOT/.git
# [WORKAROUND]
# https://superuser.com/questions/1241548/
# xubuntu-16-04-ttyname-failed-inappropriate-ioctl-for-device#1253889
RUN [ -f ~/.profile ] \
&& sed -i 's/mesg n/( tty -s \&\& mesg n || true )/g' ~/.profile \
|| true
# [WORKAROUND]
# https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
RUN ln -s posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
WORKDIR /root
SHELL ["/bin/bash", "-l", "-c"]
# TODO: add a command to Spack that (re)creates the package cache
RUN spack spec hdf5+mpi
ENTRYPOINT ["/bin/bash", "/opt/spack/share/spack/docker/entrypoint.bash"]
CMD ["docker-shell"]

87
share/spack/qa/run-docker-tests
View File

@ -32,19 +32,90 @@ ensure_docker_login() {
return $__login_success return $__login_success
} }
for config in share/spack/docker/config/* ; do build_docker_image() {
source "$config" ; local dockerfile="$1" ; shift
./share/spack/docker/build-image.sh; local build_ctx="$1" ; shift
done local image_name="$1" ; shift
local first_tag="$1"
local tags=""
if [ -n "$*" ] ; then
tags="$( echo " $*" | sed "s| *| -t ${image_name}:|g" )"
tags="${tags:1}"
fi
if [ -n "$first_tag" ] ; then
first_tag=":$first_tag"
fi
local cache=""
if docker pull "${image_name}${first_tag}" ; then
local cache="--cache-from ${image_name}${first_tag}"
fi
( set +e
docker build -f "$dockerfile" $cache $tags "$build_ctx" )
return $?
}
push_docker_image() {
local image_name="$1" ; shift
local result=0
while [ -n "$*" ] ; do
local tag="$1" ; shift
(
set +e
docker push "${image_name}:${tag}"
)
result="$?"
[ "$result" '=' '0' ] || break
done
return $result
}
SPACK_VERSION="$( ../../../bin/spack --version )"
build_docker_image centos-6.dockerfile ../../.. spack/spack \
"${SPACK_VERSION}-centos-6" \
"centos-6"
build_docker_image centos-7.dockerfile ../../.. spack/spack \
"${SPACK_VERSION}-centos-7" \
"${SPACK_VERSION}" \
"centos-7" \
"latest"
build_docker_image ubuntu-1604.dockerfile ../../.. spack/spack \
"${SPACK_VERSION}-ubuntu-16.04" \
"${SPACK_VERSION}-ubuntu-xenial" \
"ubuntu-16.04" \
"ubuntu-xenial"
build_docker_image ubuntu-1804.dockerfile ../../.. spack/spack \
"${SPACK_VERSION}-ubuntu-18.04" \
"${SPACK_VERSION}-ubuntu-bionic" \
"ubuntu-18.04" \
"ubuntu-bionic"
spack list --format version_json > packages.json spack list --format version_json > packages.json
./share/spack/packages/build-image.sh ./share/spack/packages/build-image.sh
if [ "$TEST_SUITE" '=' "docker" -a \ if [ "$TEST_SUITE" '=' "docker" -a \
"$TRAVIS_EVENT_TYPE" != "pull_request" ] && ensure_docker_login ; then "$TRAVIS_EVENT_TYPE" != "pull_request" ] && ensure_docker_login ; then
for config in share/spack/docker/config/* ; do push_docker_image "spack/spack" \
source "$config" "${SPACK_VERSION}-centos-6" \
./share/spack/docker/push-image.sh "${SPACK_VERSION}-centos-7" \
done "${SPACK_VERSION}-ubuntu-16.04" \
"${SPACK_VERSION}-ubuntu-18.04" \
"${SPACK_VERSION}-ubuntu-xenial" \
"${SPACK_VERSION}-ubuntu-bionic" \
"${SPACK_VERSION}" \
"centos-6" \
"centos-7" \
"ubuntu-16.04" \
"ubuntu-18.04" \
"ubuntu-xenial" \
"ubuntu-bionic" \
"latest"
./share/spack/packages/push-image.sh ./share/spack/packages/push-image.sh
fi fi