Package Index: Build in Dockerhub (#13810)

* Package Index: Build in Dockerhub

Prepare to build the package index service, packages.spack.io,
on Dockerhub.

Local build (in spack root dir):
```
docker build -t spack/packages.spack.io:latest -f share/spack/docker/package-index/Dockerfile .
```

Local test:
```
docker run -p 8080:80 spack/packages.spack.io:latest
```

* Travis-CI: Remove Docker

Remove leftover docker stages from Travis-CI.

* Simplify Split Call

.travis.yml

@@ -102,20 +102,11 @@ jobs:
       os: linux
       language: python
       env: [ TEST_SUITE=build, 'SPEC=mpich' ]
-    - python: '3.8'
-      stage: 'docker build'
-      os: linux
-      language: python
-      env: TEST_SUITE=docker
-  allow_failures:
-    - env: TEST_SUITE=docker
 
 stages:
   - 'style checks'
   - 'unit tests + documentation'
   - 'build tests'
-  - name: 'docker build'
-    if: type = push AND branch IN (develop, master)
 
 
 #=============================================================================
@@ -199,9 +190,6 @@ before_script:
 #=============================================================================
 # Building
 #=============================================================================
-services:
-  - docker
-
 script:
   - share/spack/qa/run-$TEST_SUITE-tests
 

share/spack/docker/package-index/Dockerfile

@@ -0,0 +1,30 @@
+# prepare the package index in form of JSON files
+FROM ubuntu:18.04 AS build-env
+
+ENV SPACK_ROOT=/opt/spack             \
+    DEBIAN_FRONTEND=noninteractive
+
+COPY bin     $SPACK_ROOT/bin
+COPY etc     $SPACK_ROOT/etc
+COPY lib     $SPACK_ROOT/lib
+COPY share   $SPACK_ROOT/share
+COPY var     $SPACK_ROOT/var
+
+RUN apt-get -yqq update   \
+ && apt-get -yqq install  \
+        bash jq python    \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /build
+# single, large index file
+RUN $SPACK_ROOT/bin/spack list --format version_json > packages.json
+# individual packages split into a tree of :firstLetter/:packageName.json
+RUN $SPACK_ROOT/share/spack/docker/package-index/split.sh
+
+# nginx web service
+FROM nginx:mainline-alpine
+MAINTAINER Spack Maintainers <maintainers@spack.io>
+COPY --from=build-env --chown=nginx:nginx /build/packages /build/packages.json /usr/share/nginx/html/api/
+COPY share/spack/docker/package-index/cors-header.conf /etc/nginx/conf.d/
+
+CMD ["nginx", "-g", "daemon off;"]

share/spack/docker/package-index/README.rst

@@ -0,0 +1,45 @@
+============================================
+The packages.spack.io Package Index REST API
+============================================
+
+This directory provides the docker recipe for the Spack package index on https://packages.spack.io
+
+On each merge to ``develop``, DockerHub builds a new image ``spack/packages.spack.io`` which is configured in:
+  https://cloud.docker.com/u/spack/repository/docker/spack/packages.spack.io/builds/edit
+
+------------
+The REST API
+------------
+
+The API is a simple, file-based JSON index.
+A specific package can be queried via the URI syntax:
+``https://packages.spack.io/api/:firstLetter/:packageName.json``
+which will return a HTTP status code ``200`` with a JSON file for all valid packages (content from ``spack list --format version_json``) and HTTP status code ``404`` for all other package names.
+
+Examples:
+
+- https://packages.spack.io/api/a/adios2.json
+- https://packages.spack.io/api/p/py-pandas.json
+
+There is also the full index available at once under https://packages.spack.io/api/packages.json
+
+Current down-stream dependencies are, e.g. the https://shields.io service:
+
+- https://shields.io/category/version
+- https://github.com/badges/shields/pull/3536
+
+--------------------
+Local Build and Test
+--------------------
+
+Execute in your local Spack source root directory:
+
+.. code-block:: bash
+
+   docker build -t spack/packages.spack.io:latest -f share/spack/docker/package-index/Dockerfile .
+
+Startup a local HTTP server on http://localhost:8080 via:
+
+.. code-block:: bash
+
+   docker run -p 8080:80 spack/packages.spack.io:latest

share/spack/packages/Dockerfile

@@ -1,12 +0,0 @@
-FROM ubuntu:18.04 AS build-env
-WORKDIR /build
-RUN apt-get update && apt-get install -y jq
-COPY packages.json ./
-COPY split.sh ./
-RUN /build/split.sh
-
-FROM nginx:mainline-alpine
-COPY --from=build-env --chown=nginx:nginx /build/packages /build/packages.json /usr/share/nginx/html/api/
-COPY cors-header.conf /etc/nginx/conf.d/
-
-CMD ["nginx", "-g", "daemon off;"]

share/spack/packages/build-image.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-script="$( basename "$0" )"
-cd "$( dirname "$0" )"
-
-export IMAGE="spack/packages.spack.io:latest"
-
-if [ "$script" '=' 'push-image.sh' ] ; then
-    docker push "${IMAGE}"
-else
-    docker build --no-cache --force-rm -t "${IMAGE}" .
-fi

share/spack/packages/push-image.sh

@@ -1 +0,0 @@
-build-image.sh

share/spack/qa/run-docker-tests

@@ -1,45 +0,0 @@
-#!/bin/bash -e
-#
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-#
-# Description:
-#     Runs Spack docker tests.  This builds a docker image for each of the
-#     configurations in share/spack/docker/config.
-#
-# Usage:
-#     run-docker-tests
-#
-
-__login_attempted=0
-__login_success=1
-ensure_docker_login() {
-    if [ "$__login_attempted" '!=' '0' ] ; then
-        return $__login_success
-    fi
-
-    echo "$DOCKER_PASSWORD" | \
-        docker login -u "$DOCKER_USERNAME" --password-stdin
-
-    if [ $? '=' '0' ] ; then
-          __login_success=0
-    fi
-
-    __login_attempted=1
-    return $__login_success
-}
-
-this_dir=$(cd $(dirname $0) && pwd)
-SPACK_BIN="${this_dir}/../../../bin/spack"
-
-# packages.spack.io service
-${SPACK_BIN} list --format version_json > ${this_dir}/../packages/packages.json
-./share/spack/packages/build-image.sh
-if [ "$TEST_SUITE" '=' "docker" -a \
-     "$TRAVIS_EVENT_TYPE" != "pull_request" ] && ensure_docker_login ; then
-    ./share/spack/packages/push-image.sh
-fi
-