stages: [ "generate", "build" ]

variables:
  SPACK_DISABLE_LOCAL_CONFIG: "1"
  SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/tmp/_user_cache/"
  # PR_MIRROR_FETCH_DOMAIN: "https://binaries-prs.spack.io"
  PR_MIRROR_FETCH_DOMAIN: "s3://spack-binaries-prs"
  PR_MIRROR_PUSH_DOMAIN: "s3://spack-binaries-prs"
  # PROTECTED_MIRROR_FETCH_DOMAIN: "https://binaries.spack.io"
  PROTECTED_MIRROR_FETCH_DOMAIN: "s3://spack-binaries"
  PROTECTED_MIRROR_PUSH_DOMAIN: "s3://spack-binaries"

default:
  image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }

# CI Platform-Arch
.cray_rhel_x86_64_v3:
  variables:
    SPACK_TARGET_PLATFORM: "cray-rhel"
    SPACK_TARGET_ARCH: "x86_64_v3"

.cray_sles_zen4:
  variables:
    SPACK_TARGET_PLATFORM: "cray-sles"
    SPACK_TARGET_ARCH: "zen4"

.darwin_x86_64:
  variables:
    SPACK_TARGET_PLATFORM: "darwin"
    SPACK_TARGET_ARCH: "x86_64"

.darwin_aarch64:
  variables:
    SPACK_TARGET_PLATFORM: "darwin"
    SPACK_TARGET_ARCH: "aarch64"

.linux_x86_64_v3:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
    SPACK_TARGET_ARCH: "x86_64_v3"

.linux_x86_64_v4:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
    SPACK_TARGET_ARCH: "x86_64_v4"

.linux_neoverse_v1:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
    SPACK_TARGET_ARCH: "neoverse_v1"

.linux_neoverse_v2:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
    SPACK_TARGET_ARCH: "neoverse_v2"

.linux_aarch64:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
    SPACK_TARGET_ARCH: "aarch64"

.linux_power:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
    SPACK_TARGET_ARCH: "ppc64le"

.win64-msvc2019:
  variables:
    SPACK_TARGET_PLATFORM: "win64"
    SPACK_TARGET_ARCH: "x86_64"

########################################
# Job templates
########################################
.base-job:
  variables:
    PIPELINE_MIRROR_TEMPLATE: "single-src-protected-mirrors.yaml.in"
    SPACK_CI_CONFIG_ROOT: "${CI_PROJECT_DIR}/share/spack/gitlab/cloud_pipelines/configs"
    SPACK_CI_SCRIPTS_ROOT: "${CI_PROJECT_DIR}/share/spack/gitlab/cloud_pipelines/scripts"

  rules:
    - if: $SPACK_CI_DISABLE_STACKS =~ /.+/ && $SPACK_CI_STACK_NAME =~ $SPACK_CI_DISABLE_STACKS
      when: never
    - if: $SPACK_CI_ENABLE_STACKS =~ /.+/ && $SPACK_CI_STACK_NAME !~ $SPACK_CI_ENABLE_STACKS
      when: never
    - if: $CI_COMMIT_REF_NAME == "develop" || $CI_COMMIT_REF_NAME =~ /^releases\/v.*/
      # Pipelines on develop/release branches only rebuild what is missing from the mirror
      when: always
      variables:
        SPACK_PIPELINE_TYPE: "spack_protected_branch"
        SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
        SPACK_REQUIRE_SIGNING: "True"
        OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
    - if: $CI_COMMIT_TAG =~ /^develop-[\d]{4}-[\d]{2}-[\d]{2}$/ || $CI_COMMIT_TAG =~ /^v.*/
      # Pipelines on tags (release or dev snapshots) only copy binaries from one mirror to another
      when: always
      variables:
        SPACK_PIPELINE_TYPE: "spack_copy_only"
        SPACK_COPY_BUILDCACHE: "${PROTECTED_MIRROR_PUSH_DOMAIN}/${CI_COMMIT_REF_NAME}"
        PIPELINE_MIRROR_TEMPLATE: "copy-only-protected-mirrors.yaml.in"
        OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
    - if: $CI_COMMIT_REF_NAME =~ /^pr[\d]+_.*$/
      # Pipelines on PR branches rebuild only what's missing, and do extra pruning
      when: always
      variables:
        SPACK_PIPELINE_TYPE: "spack_pull_request"
        SPACK_PRUNE_UNTOUCHED: "True"
        SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
        # TODO: Change sync script to include target in branch name.  Then we could
        # TODO: have multiple types of "PR" pipeline here.  It would be better if we could
        # TODO: keep just this one and use a regex to capture the target branch, but so
        # TODO: far gitlab doesn't support that.
        PR_TARGET_REF_NAME: "develop"
        PIPELINE_MIRROR_TEMPLATE: "multi-src-mirrors.yaml.in"
        OIDC_TOKEN_AUDIENCE: "pr_binary_mirror"

.generate-common:
  stage: generate
  script:
    - spack --version
    - spack env activate --without-view share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
    - spack
      --config-scope "${SPACK_CI_CONFIG_ROOT}"
      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
      ${CI_STACK_CONFIG_SCOPES}
      compiler find
    - spack
      --config-scope "${SPACK_CI_CONFIG_ROOT}"
      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
      audit configs
    # Command below needs to be `spack python` due to naming differences accross platforms
    - spack python ${SPACK_CI_SCRIPTS_ROOT}/common/expand_vars.py
      "${SPACK_CI_CONFIG_ROOT}/${PIPELINE_MIRROR_TEMPLATE}"
      "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
    - spack config add -f "${SPACK_CI_CONFIG_ROOT}/mirrors.yaml"
    - mkdir "${CI_PROJECT_DIR}/jobs_scratch_dir"
    - spack
      --config-scope "${SPACK_CI_CONFIG_ROOT}"
      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
      config blame > "${CI_PROJECT_DIR}/jobs_scratch_dir/spack.yaml.blame"
    - spack -v  --color=always
      --config-scope "${SPACK_CI_CONFIG_ROOT}"
      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
      ci generate --check-index-only -j ${SPACK_CONCRETIZE_JOBS}
      --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
  artifacts:
    paths:
      - "${CI_PROJECT_DIR}/jobs_scratch_dir"
      - "${CI_PROJECT_DIR}/tmp/_user_cache/cache/patches"
      - "${CI_PROJECT_DIR}/tmp/_user_cache/cache/providers"
      - "${CI_PROJECT_DIR}/tmp/_user_cache/cache/tags"
  variables:
    SPACK_CONCRETIZE_JOBS: 8
    KUBERNETES_CPU_REQUEST: 8000m
    KUBERNETES_MEMORY_REQUEST: 16G
    # avoid moving targets like branches and tags
    SPACK_CONCRETIZER_REQUIRE_CHECKSUM: 1
    SPACK_BACKTRACE: 1
  interruptible: true
  timeout: 60 minutes
  retry:
    max: 2
    when:
      - always

# Generate without tags for cases using external runners
.generate-base:
  extends: [ ".base-job", ".generate-common" ]
  before_script:
    - uname -a || true
    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
    - nproc || true
    - cat /proc/loadavg || true
    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
    - . "./share/spack/setup-env.sh"
    - spack arch
  after_script:
    - cat /proc/loadavg || true
    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true

.generate-x86_64:
  extends: [ ".generate-base" ]
  tags: ["spack", "public", "medium", "x86_64_v3"]

.generate-aarch64:
  extends: [ ".generate-base" ]
  tags: ["spack", "public", "medium", "aarch64"]

.generate-neoverse_v1:
  extends: [ ".generate-base" ]
  tags: ["spack", "public", "medium", "aarch64", "graviton3"]

.generate-neoverse-v2:
  extends: [ ".generate-base" ]
  tags: ["spack", "public", "medium", "neoverse_v2"]

.generate-win64:
  extends: [ ".base-job", ".generate-common" ]
  before_script:
    - $ErrorActionOld=$ErrorActionPreference
    - $ErrorActionPreference="SilentlyContinue"
    - python -c"import psutil;print(psutil.getloadavg())"
    - (Get-WmiObject Win32_PhysicalMemory | measure-object Capacity -sum).sum/1kb
    - $ErrorActionPreference=$ErrorActionOld
    - . .\share\spack\setup-env.ps1
  after_script:
    - $ErrorActionOld=$ErrorActionPreference
    - $ErrorActionPreference="SilentlyContinue"
    - python -c"import psutil;print(psutil.getloadavg())"
    - (Get-WmiObject Win32_PhysicalMemory | measure-object Capacity -sum).sum/1kb
    - $ErrorActionPreference=$ErrorActionOld

  tags: ["spack", "public", "medium", "x86_64-win"]
  image: "ghcr.io/johnwparent/windows-server21h2:sha-1c12b61"

.build:
  extends: [ ".base-job" ]
  stage: build

########################################
# TEMPLATE FOR ADDING ANOTHER PIPELINE
########################################
#
# First add a new spack.yml defining the pipeline to run in
# share/spack/gitlab/cloud_pipelines/stacks/my-super-cool-stack/spack.yaml
#
# Then add the following entries at the bottom of this file.
#
# Note that when extending other jobs, gitlab does not merge lists (nor
# does it merge dictionary values in the case of key conflicts).  So lists
# and duplicated dictionary keys are simply overridden.  For this reason,
# you should inlclude your custom definitions at the end of the of the
# extends list.
#
########################################
# My Super Cool Pipeline
########################################
# .my-super-cool-stack:
#   extends: [ ".linux_x86_64_v3" ]
#   variables:
#     SPACK_CI_STACK_NAME: my-super-cool-stack
#     tags: [ "all", "tags", "your", "job", "needs"]
#
# my-super-cool-stack-generate:
#   extends: [ ".generate", ".my-super-cool-stack" ]
#
# my-super-cool-stack-build:
#   extends: [ ".build", ".my-super-cool-stack" ]
#   trigger:
#     include:
#       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
#         job: my-super-cool-stack-generate
#     strategy: depend
#   needs:
#     - artifacts: True
#       job: my-super-cool-stack-generate

########################################
# E4S pipeline
########################################
.e4s:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: e4s

e4s-generate:
  extends: [ ".e4s", ".generate-x86_64"]
  image: ghcr.io/spack/spack/ubuntu22.04-runner-amd64-gcc-11.4:2024.03.01

e4s-build:
  extends: [ ".e4s", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: e4s-generate
    strategy: depend
  needs:
    - artifacts: True
      job: e4s-generate

########################################
# E4S Neoverse V2
########################################
.e4s-neoverse-v2:
  extends: [ ".linux_neoverse_v2" ]
  variables:
    SPACK_CI_STACK_NAME: e4s-neoverse-v2

e4s-neoverse-v2-generate:
  extends: [ ".e4s-neoverse-v2", ".generate-neoverse-v2" ]
  image: ghcr.io/spack/spack/ubuntu22.04-runner-arm64-gcc-11.4:2024.03.01

e4s-neoverse-v2-build:
  extends: [ ".e4s-neoverse-v2", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: e4s-neoverse-v2-generate
    strategy: depend
  needs:
    - artifacts: True
      job: e4s-neoverse-v2-generate

########################################
# E4S ROCm External pipeline
########################################
.e4s-rocm-external:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: e4s-rocm-external

e4s-rocm-external-generate:
  extends: [ ".e4s-rocm-external", ".generate-x86_64"]
  image: ghcr.io/spack/spack/ubuntu22.04-runner-amd64-gcc-11.4-rocm6.2.1:2024.10.08

e4s-rocm-external-build:
  extends: [ ".e4s-rocm-external", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: e4s-rocm-external-generate
    strategy: depend
  needs:
    - artifacts: True
      job: e4s-rocm-external-generate

########################################
# E4S OneAPI Pipeline
########################################
.e4s-oneapi:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: e4s-oneapi

e4s-oneapi-generate:
  extends: [ ".e4s-oneapi", ".generate-x86_64"]
  image: ghcr.io/spack/spack/ubuntu22.04-runner-amd64-oneapi-2024.2:2024.09.06

e4s-oneapi-build:
  extends: [ ".e4s-oneapi", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: e4s-oneapi-generate
    strategy: depend
  needs:
    - artifacts: True
      job: e4s-oneapi-generate

########################################
# E4S on Power
########################################
.e4s-power-generate-tags-and-image:
  image: { "name": "ghcr.io/spack/ubuntu20.04-runner-ppc64-gcc-11.4:2023.08.01", "entrypoint": [""] }
  tags: ["spack", "public", "large", "ppc64le"]

.e4s-power:
  extends: [".linux_power"]
  variables:
    SPACK_CI_STACK_NAME: e4s-power

e4s-power-generate:
  extends: [ ".e4s-power", ".generate-x86_64", ".e4s-power-generate-tags-and-image"]
  variables:
    # Override concretization pool for metal runners
    SPACK_CONCRETIZE_JOBS: 16

e4s-power-build:
  extends: [ ".e4s-power", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: e4s-power-generate
    strategy: depend
  needs:
    - artifacts: True
      job: e4s-power-generate

#########################################
# Build tests for different build-systems
#########################################
.build_systems:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: build_systems

build_systems-generate:
  extends: [ ".build_systems", ".generate-x86_64"]

build_systems-build:
  extends: [ ".build_systems", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: build_systems-generate
    strategy: depend
  needs:
    - artifacts: True
      job: build_systems-generate

###########################################
# Build tests for different developer tools
###########################################
.developer-tools-x86_64_v3-linux-gnu:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: developer-tools-x86_64_v3-linux-gnu

developer-tools-x86_64_v3-linux-gnu-generate:
  extends: [ ".developer-tools-x86_64_v3-linux-gnu", ".generate-x86_64"]
  image:  ghcr.io/spack/x86_64_v3-linux-gnu:2024-12-12

developer-tools-x86_64_v3-linux-gnu-build:
  extends: [ ".developer-tools-x86_64_v3-linux-gnu", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: developer-tools-x86_64_v3-linux-gnu-generate
    strategy: depend
  needs:
    - artifacts: True
      job: developer-tools-x86_64_v3-linux-gnu-generate

###########################################
# Build tests for different developer tools
# aarch64
###########################################
.developer-tools-aarch64-linux-gnu:
  extends: [ ".linux_aarch64" ]
  variables:
    SPACK_CI_STACK_NAME: developer-tools-aarch64-linux-gnu

developer-tools-aarch64-linux-gnu-generate:
  extends: [ ".developer-tools-aarch64-linux-gnu", ".generate-aarch64"]
  image:  ghcr.io/spack/aarch64-linux-gnu:v2024-12-18

developer-tools-aarch64-linux-gnu-build:
  extends: [ ".developer-tools-aarch64-linux-gnu", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: developer-tools-aarch64-linux-gnu-generate
    strategy: depend
  needs:
    - artifacts: True
      job: developer-tools-aarch64-linux-gnu-generate

###########################################
# Build tests for different developer tools
# darwin
###########################################
.developer-tools-darwin:
  extends: [ ".darwin_aarch64" ]
  variables:
    SPACK_CI_STACK_NAME: developer-tools-darwin

developer-tools-darwin-generate:
  tags: [ "macos-sonoma", "apple-clang-16", "aarch64-macos" ]
  extends: [ ".developer-tools-darwin", ".generate-base"]

developer-tools-darwin-build:
  extends: [ ".developer-tools-darwin", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: developer-tools-darwin-generate
    strategy: depend
  needs:
    - artifacts: True
      job: developer-tools-darwin-generate

#########################################
# RADIUSS
#########################################
.radiuss:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: radiuss

radiuss-generate:
  extends: [ ".radiuss", ".generate-x86_64" ]

radiuss-build:
  extends: [ ".radiuss", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: radiuss-generate
    strategy: depend
  needs:
    - artifacts: True
      job: radiuss-generate

########################################
# RADIUSS for AWS
########################################

.tags-x86_64_v4:
  tags: ["spack", "public", "aws", "medium", "x86_64_v4"]

# Include this AFTER .*-generate in "extends" list
.radiuss-aws-overrides:
  # This controls image for generate step; build step is controlled by spack.yaml
  # Note that generator emits OS info for build so these should be the same.
  image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2023-03-09", "entrypoint": [""] }

.radiuss-aws:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: radiuss-aws

radiuss-aws-generate:
  extends: [ ".radiuss-aws", ".generate-x86_64", ".radiuss-aws-overrides", ".tags-x86_64_v4" ]

radiuss-aws-build:
  extends: [ ".radiuss-aws", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: radiuss-aws-generate
    strategy: depend
  needs:
    - artifacts: True
      job: radiuss-aws-generate


# Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64)

.radiuss-aws-aarch64:
  extends: [ ".linux_aarch64" ]
  variables:
    SPACK_CI_STACK_NAME: radiuss-aws-aarch64

radiuss-aws-aarch64-generate:
  extends: [ ".radiuss-aws-aarch64", ".generate-aarch64", ".radiuss-aws-overrides" ]

radiuss-aws-aarch64-build:
  extends: [ ".radiuss-aws-aarch64", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: radiuss-aws-aarch64-generate
    strategy: depend
  needs:
    - artifacts: True
      job: radiuss-aws-aarch64-generate

########################################
# ECP Data & Vis SDK
########################################
.data-vis-sdk:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: data-vis-sdk

data-vis-sdk-generate:
  extends: [ ".data-vis-sdk", ".generate-x86_64"]
  image: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01

data-vis-sdk-build:
  extends: [ ".data-vis-sdk", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: data-vis-sdk-generate
    strategy: depend
  needs:
    - artifacts: True
      job: data-vis-sdk-generate

########################################
# AWS ISC Applications (x86_64)
########################################

# Call this AFTER .*-generate
.aws-isc-overrides:
  # This controls image for generate step; build step is controlled by spack.yaml
  # Note that generator emits OS info for build so these should be the same.
  image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2023-03-09", "entrypoint": [""] }

.aws-isc:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: aws-isc

aws-isc-generate:
  extends: [ ".aws-isc", ".generate-x86_64", ".aws-isc-overrides", ".tags-x86_64_v4" ]

aws-isc-build:
  extends: [ ".aws-isc", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: aws-isc-generate
    strategy: depend
  needs:
    - artifacts: True
      job: aws-isc-generate

# Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64)

.aws-isc-aarch64:
  extends: [ ".linux_aarch64" ]
  variables:
    SPACK_CI_STACK_NAME: aws-isc-aarch64

aws-isc-aarch64-generate:
  extends: [ ".aws-isc-aarch64", ".generate-aarch64", ".aws-isc-overrides" ]

aws-isc-aarch64-build:
  extends: [ ".aws-isc-aarch64", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: aws-isc-aarch64-generate
    strategy: depend
  needs:
    - artifacts: True
      job: aws-isc-aarch64-generate


########################################
# Spack Tutorial
########################################
.tutorial:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: tutorial

tutorial-generate:
  extends: [ ".tutorial", ".generate-x86_64"]
  image: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-10-30

tutorial-build:
  extends: [ ".tutorial", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: tutorial-generate
    strategy: depend
  needs:
    - artifacts: True
      job: tutorial-generate

#######################################
# Machine Learning - Linux x86_64 (CPU)
#######################################
.ml-linux-x86_64-cpu:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: ml-linux-x86_64-cpu

ml-linux-x86_64-cpu-generate:
  extends: [ ".generate-x86_64", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ]
  image: ghcr.io/spack/ubuntu-24.04:v2024-09-05-v2

ml-linux-x86_64-cpu-build:
  extends: [ ".build", ".ml-linux-x86_64-cpu" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: ml-linux-x86_64-cpu-generate
    strategy: depend
  needs:
    - artifacts: True
      job: ml-linux-x86_64-cpu-generate

########################################
# Machine Learning - Linux x86_64 (CUDA)
########################################
.ml-linux-x86_64-cuda:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: ml-linux-x86_64-cuda

ml-linux-x86_64-cuda-generate:
  extends: [ ".generate-x86_64", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ]
  image: ghcr.io/spack/ubuntu-24.04:v2024-09-05-v2

ml-linux-x86_64-cuda-build:
  extends: [ ".build", ".ml-linux-x86_64-cuda"  ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: ml-linux-x86_64-cuda-generate
    strategy: depend
  needs:
    - artifacts: True
      job: ml-linux-x86_64-cuda-generate

########################################
# Machine Learning - Linux x86_64 (ROCm)
########################################
.ml-linux-x86_64-rocm:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: ml-linux-x86_64-rocm

ml-linux-x86_64-rocm-generate:
  extends: [ ".generate-x86_64", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ]
  image: ghcr.io/spack/ubuntu-24.04:v2024-09-05-v2

ml-linux-x86_64-rocm-build:
  extends: [ ".build", ".ml-linux-x86_64-rocm" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: ml-linux-x86_64-rocm-generate
    strategy: depend
  needs:
    - artifacts: True
      job: ml-linux-x86_64-rocm-generate

########################################
# Machine Learning - Linux aarch64 (CPU)
########################################
.ml-linux-aarch64-cpu:
  extends: [ ".linux_aarch64" ]
  variables:
    SPACK_CI_STACK_NAME: ml-linux-aarch64-cpu

ml-linux-aarch64-cpu-generate:
  extends: [ ".generate-aarch64", .ml-linux-aarch64-cpu ]
  image: ghcr.io/spack/ubuntu-24.04:v2024-09-05-v2

ml-linux-aarch64-cpu-build:
  extends: [ ".build", ".ml-linux-aarch64-cpu" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: ml-linux-aarch64-cpu-generate
    strategy: depend
  needs:
    - artifacts: True
      job: ml-linux-aarch64-cpu-generate

#########################################
# Machine Learning - Linux aarch64 (CUDA)
#########################################
.ml-linux-aarch64-cuda:
  extends: [ ".linux_aarch64" ]
  variables:
    SPACK_CI_STACK_NAME: ml-linux-aarch64-cuda

ml-linux-aarch64-cuda-generate:
  extends: [ ".generate-aarch64", .ml-linux-aarch64-cuda ]
  image: ghcr.io/spack/ubuntu-24.04:v2024-09-05-v2

ml-linux-aarch64-cuda-build:
  extends: [ ".build", ".ml-linux-aarch64-cuda" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: ml-linux-aarch64-cuda-generate
    strategy: depend
  needs:
    - artifacts: True
      job: ml-linux-aarch64-cuda-generate

#########################################
# Machine Learning - Darwin aarch64 (MPS)
#########################################
.ml-darwin-aarch64-mps:
  extends: [".darwin_aarch64"]
  variables:
    SPACK_CI_STACK_NAME: ml-darwin-aarch64-mps

ml-darwin-aarch64-mps-generate:
  tags: [ "macos-sonoma", "apple-clang-16", "aarch64-macos" ]
  extends: [ ".ml-darwin-aarch64-mps", ".generate-base"]

ml-darwin-aarch64-mps-build:
  extends: [ ".ml-darwin-aarch64-mps", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: ml-darwin-aarch64-mps-generate
    strategy: depend
  needs:
    - artifacts: True
      job: ml-darwin-aarch64-mps-generate

########################################
# High Energy Physics (HEP) - Linux x86_64 (CPU)
########################################
.hep:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: hep

hep-generate:
  extends: [ ".hep", ".generate-x86_64"]
  image: ghcr.io/spack/spack/ubuntu22.04-runner-amd64-gcc-11.4:2024.03.01

hep-build:
  extends: [ ".hep", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: hep-generate
    strategy: depend
  needs:
    - artifacts: True
      job: hep-generate

########################################
# AWS ParallelCluster
########################################

.aws-pcluster-generate:
  image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07", "entrypoint": [""] }

# X86_64_V4 (one pipeline per target)
.aws-pcluster-x86_64_v4:
  variables:
    SPACK_CI_STACK_NAME: aws-pcluster-x86_64_v4

aws-pcluster-generate-x86_64_v4:
  extends: [ ".linux_x86_64_v4", ".aws-pcluster-x86_64_v4", ".generate-base", ".tags-x86_64_v4", ".aws-pcluster-generate"]
  before_script:
    - - . "./share/spack/setup-env.sh"
      # TODO: Move this to the container next time it is rebuilt
      - export PATH=/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-7.3.1/binutils-2.37-qvccg7zpskturysmr4bzbsfrx34kvazo/bin:$PATH

aws-pcluster-build-x86_64_v4:
  extends: [ ".linux_x86_64_v4", ".aws-pcluster-x86_64_v4", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: aws-pcluster-generate-x86_64_v4
    strategy: depend
  needs:
    - artifacts: True
      job: aws-pcluster-generate-x86_64_v4

# Neoverse_v1 (one pipeline per target)
.aws-pcluster-neoverse_v1:
  variables:
    SPACK_CI_STACK_NAME: aws-pcluster-neoverse_v1

aws-pcluster-generate-neoverse_v1:
  # TODO: Use updated runner tags: https://github.com/spack/spack-infrastructure/pull/694/files
  extends: [ ".linux_neoverse_v1", ".aws-pcluster-neoverse_v1", ".generate-neoverse_v1", ".aws-pcluster-generate"]
  before_script:
    - - . "./share/spack/setup-env.sh"
      # TODO: Move this to the container next time it is rebuilt
      - export PATH=/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-aarch64/gcc-7.3.1/binutils-2.37-2yxz3xsjfmesxujxtlrgcctxlyilynmp/bin:$PATH

aws-pcluster-build-neoverse_v1:
  extends: [  ".linux_neoverse_v1", ".aws-pcluster-neoverse_v1", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: aws-pcluster-generate-neoverse_v1
    strategy: depend
  needs:
    - artifacts: True
      job: aws-pcluster-generate-neoverse_v1

# Cray definitions
.generate-cray:
  extends: [ ".generate-common", ".base-job" ]
  variables:
    # Override concretization pool for metal runners
    SPACK_CONCRETIZE_JOBS: 16

  before_script:
    - echo $PATH
    - module avail
    - module list
    - uname -a || true
    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
    - nproc || true
    - cat /proc/loadavg || true
    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true
    - . "./share/spack/setup-env.sh"
  after_script:
    - cat /proc/loadavg || true
    - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true

.generate-cray-rhel:
  tags: [ "cray-rhel-x86_64_v3", "public" ]
  extends: [ ".generate-cray" ]

.generate-cray-sles:
  tags: [ "cray-sles-zen4", "public" ]
  extends: [ ".generate-cray" ]


#######################################
# E4S - Cray RHEL
#######################################
.e4s-cray-rhel:
  extends: [ ".cray_rhel_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: e4s-cray-rhel

e4s-cray-rhel-generate:
  extends: [ ".generate-cray-rhel", ".e4s-cray-rhel" ]

e4s-cray-rhel-build:
  extends: [ ".build", ".e4s-cray-rhel" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: e4s-cray-rhel-generate
    strategy: depend
  needs:
    - artifacts: True
      job: e4s-cray-rhel-generate

#######################################
# E4S - Cray SLES
#######################################
.e4s-cray-sles:
  extends: [ ".cray_sles_zen4" ]
  variables:
    SPACK_CI_STACK_NAME: e4s-cray-sles

.e4s-cray-sles-generate:
  extends: [ ".generate-cray-sles", ".e4s-cray-sles" ]

.e4s-cray-sles-build:
  allow_failure: true  # libsci_cray.so broken, misses DT_NEEDED for libdl.so
  extends: [ ".build", ".e4s-cray-sles" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: e4s-cray-sles-generate
    strategy: depend
  needs:
    - artifacts: True
      job: e4s-cray-sles-generate

#######################################
# Windows Visualization Tools
#######################################
.windows-vis:
  extends: [".win64-msvc2019"]
  variables:
    SPACK_CI_STACK_NAME: windows-vis
    # TODO: windows runners are currently m5.2xlarge with a max 8 vCPU and 32GiB MEM
    # setting request to half of the available allocation to allow room for overhead.
    KUBERNETES_CPU_REQUEST: 4000m
    KUBERNETES_MEMORY_REQUEST: 16G

windows-vis-generate:
  extends: [ ".generate-win64", ".windows-vis" ]

windows-vis-build:
  extends: [ ".build", ".windows-vis"]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: windows-vis-generate
    strategy: depend
  needs:
    - artifacts: True
      job: windows-vis-generate

#######################################
# Bootstrap x86_64-linux-gnu
#######################################
.bootstrap-x86_64-linux-gnu:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: bootstrap-x86_64-linux-gnu

bootstrap-x86_64-linux-gnu-generate:
  extends: [ .generate-x86_64, .bootstrap-x86_64-linux-gnu ]
  image: ghcr.io/spack/ubuntu-24.04:v2024-09-05-v2

bootstrap-x86_64-linux-gnu-build:
  extends: [ .build, .bootstrap-x86_64-linux-gnu ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: bootstrap-x86_64-linux-gnu-generate
    strategy: depend
  needs:
    - artifacts: True
      job: bootstrap-x86_64-linux-gnu-generate

#######################################
# Bootstrap aarch64-darwin
#######################################
.bootstrap-aarch64-darwin:
  extends: [.darwin_aarch64]
  variables:
    SPACK_CI_STACK_NAME: bootstrap-aarch64-darwin

bootstrap-aarch64-darwin-generate:
  tags: [ "macos-sonoma", "apple-clang-16", "aarch64-macos" ]
  extends: [.bootstrap-aarch64-darwin, .generate-base]

bootstrap-aarch64-darwin-build:
  extends: [.bootstrap-aarch64-darwin, .build]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: bootstrap-aarch64-darwin-generate
    strategy: depend
  needs:
    - artifacts: true
      job: bootstrap-aarch64-darwin-generate