spack/share/spack/gitlab/cloud_pipelines/configs/ci.yaml

ci:
  target: gitlab

  broken-tests-packages:
    - superlu-dist    # srun -n 4 hangs
    - papyrus

  pipeline-gen:
  - build-job:
      before_script-:
      - - cat /proc/loadavg || true
        - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
      - - touch ${SPACK_USER_CACHE_PATH}/cache/*/*  # bump mtime of cache so it is not invalidated
      - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
        - spack compiler list
      - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
        # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
      - - k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust $k
        # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
        - k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust $k
        - k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k
      script::
      - - if [ -n "$SPACK_EXTRA_MIRROR" ]; then spack mirror add local "${SPACK_EXTRA_MIRROR}/${SPACK_CI_STACK_NAME}"; fi
        - spack config blame mirrors
      - - spack --color=always --backtrace ci rebuild -j ${SPACK_BUILD_JOBS} --tests --timeout 300 > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
      after_script:
      - - cat /proc/loadavg || true
        - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
      - - time ./bin/spack python ${CI_PROJECT_DIR}/share/spack/gitlab/cloud_pipelines/scripts/common/aggregate_package_logs.spack.py
          --prefix /home/software/spack:${CI_PROJECT_DIR}/opt/spack
          --log install_times.json
          ${SPACK_ARTIFACTS_ROOT}/user_data/install_times.json || true
      variables:
        CI_JOB_SIZE: "default"
        SPACK_BUILD_JOBS: "4"
        KUBERNETES_CPU_REQUEST: "4000m"
        KUBERNETES_MEMORY_REQUEST: "16G"
        CI_GPG_KEY_ROOT: /mnt/key
        # SPACK_VERBOSE_SCRIPT: "1"
      id_tokens:
        GITLAB_OIDC_TOKEN:
          aud: "${OIDC_TOKEN_AUDIENCE}"

  - signing-job:
      image: { "name": "ghcr.io/spack/notary:0.0.1", "entrypoint": [""] }
      tags: ["aws"]
      script:
      - - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_BUILDCACHE_DESTINATION}/build_cache /tmp
        - /sign.sh
        - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_BUILDCACHE_DESTINATION}/build_cache
        - aws s3 cp /tmp/public_keys ${SPACK_BUILDCACHE_DESTINATION}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
      id_tokens:
        GITLAB_OIDC_TOKEN:
          aud: "${OIDC_TOKEN_AUDIENCE}"

  - copy-job:
      tags: ["service", "x86_64"]
      image: "ghcr.io/spack/python-aws-bash:0.0.1"
      before_script:
      - - if [[ $CI_COMMIT_TAG == "v"* ]]; then export SPACK_REPLACE_VERSION=$(echo "$CI_COMMIT_TAG" | sed 's/\(v[[:digit:]]\+\.[[:digit:]]\+\).*/releases\/\1/'); fi
        - if [[ $CI_COMMIT_TAG == "develop-"* ]]; then export SPACK_REPLACE_VERSION=develop; fi
        - export SPACK_COPY_ONLY_SOURCE=${SPACK_BUILDCACHE_SOURCE//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
      script:
      - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
        - echo Copying environment specs from ${SPACK_COPY_ONLY_SOURCE} to ${SPACK_COPY_ONLY_DESTINATION}
        - spack buildcache sync "${SPACK_COPY_ONLY_SOURCE}" "${SPACK_COPY_ONLY_DESTINATION}"
        - curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub
        - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_COPY_ONLY_DESTINATION}/build_cache/_pgp/spack-public-binary-key.pub"
        - spack buildcache update-index --keys "${SPACK_COPY_ONLY_DESTINATION}"
      when: "always"
      retry:
        max: 2
        when:
        - "runner_system_failure"
        - "stuck_or_timeout_failure"
        - "script_failure"
      interruptible: true
      variables:
        CI_JOB_SIZE: "medium"
        KUBERNETES_CPU_REQUEST: "4000m"
        KUBERNETES_MEMORY_REQUEST: "16G"
      id_tokens:
        GITLAB_OIDC_TOKEN:
          aud: "${OIDC_TOKEN_AUDIENCE}"

  - reindex-job:
      tags: ["service", "x86_64"]
      image: "ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01"
      variables:
        CI_JOB_SIZE: "medium"
        KUBERNETES_CPU_REQUEST: "4000m"
        KUBERNETES_MEMORY_REQUEST: "16G"
      id_tokens:
        GITLAB_OIDC_TOKEN:
          aud: "${OIDC_TOKEN_AUDIENCE}"

  # TODO: Remove this block in Spack 0.23
  - cleanup-job:
      tags: ["service"]
      variables:
        CI_JOB_SIZE: "small"
        KUBERNETES_CPU_REQUEST: "500m"
        KUBERNETES_MEMORY_REQUEST: "500M"
      id_tokens:
        GITLAB_OIDC_TOKEN:
          aud: "${OIDC_TOKEN_AUDIENCE}"

  - noop-job:
      tags: ["service"]
      image: busybox:latest
      variables:
        CI_OIDC_REQUIRED: 0
        GIT_STRATEGY: "none"
        CI_JOB_SIZE: "small"
        KUBERNETES_CPU_REQUEST: "100m"
        KUBERNETES_MEMORY_REQUEST: "5M"
      before_script:: []
      after_script:: []

  - any-job:
      tags: ["spack"]
      image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
      variables:
        # Disable local configs to avoid issues on shell runners
        SPACK_DISABLE_LOCAL_CONFIG: "1"
      before_script:
      - - export SPACK_USER_CACHE_PATH="${CI_PROJECT_DIR}/tmp/_user_cache/"
      - - uname -a || true
        - grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
        - nproc || true
      - - . "./share/spack/setup-env.sh"
        - spack --version
        - spack arch