zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CI: remove redundant sections (#38514)

Browse Source 
Refactor gitlab ci configs so that mac and cray jobs can reuse as much higher level
configuration as possible.

* CI: remove redundant sections
* CI: Include base linux CI configs in cray stacks

Relocation and runner mapping is consistent between cray and linux runners.

* Export user cache path in before script
* CI: add GPG root for mac runners
* Disable user configs

Metal runners share a ~ directory

* Disable user config and add configs in activate env
This commit is contained in:
kwryankrattiger 2023-08-07 11:23:16 -05:00 committed by GitHub
parent ea9c488897
commit b54d286b4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 70 additions and 716 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

124
share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
View File

@ -19,6 +19,11 @@ default:
SPACK_TARGET_PLATFORM: "darwin" SPACK_TARGET_PLATFORM: "darwin"
SPACK_TARGET_ARCH: "x86_64" SPACK_TARGET_ARCH: "x86_64"
.darwin_aarch64:
variables:
SPACK_TARGET_PLATFORM: "darwin"
SPACK_TARGET_ARCH: "aarch64"
.linux_x86_64_v3: .linux_x86_64_v3:
variables: variables:
SPACK_TARGET_PLATFORM: "linux" SPACK_TARGET_PLATFORM: "linux"
@ -60,6 +65,8 @@ default:
.base-job: .base-job:
variables: variables:
SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}" SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
SPACK_DISABLE_LOCAL_CONFIG: "1"
SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/tmp/_user_cache/"
rules: rules:
- if: $CI_COMMIT_REF_NAME == "develop" - if: $CI_COMMIT_REF_NAME == "develop"
# Pipelines on develop only rebuild what is missing from the mirror # Pipelines on develop only rebuild what is missing from the mirror
@ -101,7 +108,7 @@ default:
AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID} AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY} AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
.generate-base: .generate-common:
stage: generate stage: generate
script: script:
- uname -a || true - uname -a || true
@ -136,41 +143,17 @@ default:
when: when:
- always - always
.generate: # Generate without tags for cases using external runners
extends: [ ".base-job", ".generate-base" ] .generate-base:
extends: [ ".base-job", ".generate-common" ]
.generate-x86_64:
extends: [ ".generate-base" ]
tags: ["spack", "public", "medium", "x86_64"] tags: ["spack", "public", "medium", "x86_64"]
.darwin-generate-base: .generate-aarch64:
stage: generate extends: [ ".generate-base" ]
script: tags: ["spack", "public", "medium", "aarch64"]
- export SPACK_DISABLE_LOCAL_CONFIG=1
- export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache
- uname -a || true
- grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
- nproc || true
- . "./share/spack/setup-env.sh"
- spack --version
- cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
- spack env activate --without-view .
- spack -d ci generate --check-index-only
--buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
--artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
--output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
after_script:
- cat /proc/loadavg || true
artifacts:
paths:
- "${CI_PROJECT_DIR}/jobs_scratch_dir"
interruptible: true
timeout: 60 minutes
retry:
max: 2
when:
- always
.darwin-generate:
extends: [ ".base-job", ".darwin-generate-base" ]
.generate-deprecated: .generate-deprecated:
extends: [ ".base-job" ] extends: [ ".base-job" ]
@ -204,10 +187,6 @@ default:
- always - always
tags: ["spack", "public", "medium", "x86_64"] tags: ["spack", "public", "medium", "x86_64"]
.generate-aarch64:
extends: [ ".base-job", ".generate" ]
tags: ["spack", "public", "medium", "aarch64"]
.build: .build:
extends: [ ".base-job" ] extends: [ ".base-job" ]
stage: build stage: build
@ -294,7 +273,7 @@ protected-publish:
SPACK_CI_STACK_NAME: e4s SPACK_CI_STACK_NAME: e4s
e4s-generate: e4s-generate:
extends: [ ".e4s", ".generate"] extends: [ ".e4s", ".generate-x86_64"]
image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
e4s-build: e4s-build:
@ -317,7 +296,7 @@ e4s-build:
SPACK_CI_STACK_NAME: gpu-tests SPACK_CI_STACK_NAME: gpu-tests
gpu-tests-generate: gpu-tests-generate:
extends: [ ".gpu-tests", ".generate"] extends: [ ".gpu-tests", ".generate-x86_64"]
image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
gpu-tests-build: gpu-tests-build:
@ -340,7 +319,7 @@ gpu-tests-build:
SPACK_CI_STACK_NAME: e4s-oneapi SPACK_CI_STACK_NAME: e4s-oneapi
e4s-oneapi-generate: e4s-oneapi-generate:
extends: [ ".e4s-oneapi", ".generate"] extends: [ ".e4s-oneapi", ".generate-x86_64"]
image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023.07.21 image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023.07.21
e4s-oneapi-build: e4s-oneapi-build:
@ -367,7 +346,7 @@ e4s-oneapi-build:
SPACK_CI_STACK_NAME: e4s-power SPACK_CI_STACK_NAME: e4s-power
e4s-power-generate: e4s-power-generate:
extends: [ ".e4s-power", ".generate", ".e4s-power-generate-tags-and-image"] extends: [ ".e4s-power", ".generate-x86_64", ".e4s-power-generate-tags-and-image"]
e4s-power-build: e4s-power-build:
extends: [ ".e4s-power", ".build" ] extends: [ ".e4s-power", ".build" ]
@ -389,7 +368,7 @@ e4s-power-build:
SPACK_CI_STACK_NAME: build_systems SPACK_CI_STACK_NAME: build_systems
build_systems-generate: build_systems-generate:
extends: [ ".build_systems", ".generate"] extends: [ ".build_systems", ".generate-x86_64"]
build_systems-build: build_systems-build:
extends: [ ".build_systems", ".build" ] extends: [ ".build_systems", ".build" ]
@ -411,7 +390,7 @@ build_systems-build:
SPACK_CI_STACK_NAME: radiuss SPACK_CI_STACK_NAME: radiuss
radiuss-generate: radiuss-generate:
extends: [ ".radiuss", ".generate" ] extends: [ ".radiuss", ".generate-x86_64" ]
radiuss-build: radiuss-build:
extends: [ ".radiuss", ".build" ] extends: [ ".radiuss", ".build" ]
@ -443,7 +422,7 @@ radiuss-build:
SPACK_CI_STACK_NAME: radiuss-aws SPACK_CI_STACK_NAME: radiuss-aws
radiuss-aws-generate: radiuss-aws-generate:
extends: [ ".radiuss-aws", ".generate", ".radiuss-aws-overrides", ".tags-x86_64_v4" ] extends: [ ".radiuss-aws", ".generate-x86_64", ".radiuss-aws-overrides", ".tags-x86_64_v4" ]
radiuss-aws-build: radiuss-aws-build:
extends: [ ".radiuss-aws", ".build" ] extends: [ ".radiuss-aws", ".build" ]
@ -487,7 +466,7 @@ radiuss-aws-aarch64-build:
SPACK_CI_STACK_NAME: data-vis-sdk SPACK_CI_STACK_NAME: data-vis-sdk
data-vis-sdk-generate: data-vis-sdk-generate:
extends: [ ".data-vis-sdk", ".generate"] extends: [ ".data-vis-sdk", ".generate-x86_64"]
image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01
data-vis-sdk-build: data-vis-sdk-build:
@ -517,7 +496,7 @@ data-vis-sdk-build:
SPACK_CI_STACK_NAME: aws-isc SPACK_CI_STACK_NAME: aws-isc
aws-isc-generate: aws-isc-generate:
extends: [ ".aws-isc", ".generate", ".aws-isc-overrides", ".tags-x86_64_v4" ] extends: [ ".aws-isc", ".generate-x86_64", ".aws-isc-overrides", ".tags-x86_64_v4" ]
aws-isc-build: aws-isc-build:
extends: [ ".aws-isc", ".build" ] extends: [ ".aws-isc", ".build" ]
@ -561,7 +540,7 @@ aws-isc-aarch64-build:
SPACK_CI_STACK_NAME: tutorial SPACK_CI_STACK_NAME: tutorial
tutorial-generate: tutorial-generate:
extends: [ ".tutorial", ".generate"] extends: [ ".tutorial", ".generate-x86_64"]
image: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-05-07 image: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-05-07
tutorial-build: tutorial-build:
@ -584,7 +563,7 @@ tutorial-build:
SPACK_CI_STACK_NAME: ml-linux-x86_64-cpu SPACK_CI_STACK_NAME: ml-linux-x86_64-cpu
ml-linux-x86_64-cpu-generate: ml-linux-x86_64-cpu-generate:
extends: [ ".generate", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ] extends: [ ".generate-x86_64", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ]
image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
ml-linux-x86_64-cpu-build: ml-linux-x86_64-cpu-build:
@ -607,7 +586,7 @@ ml-linux-x86_64-cpu-build:
SPACK_CI_STACK_NAME: ml-linux-x86_64-cuda SPACK_CI_STACK_NAME: ml-linux-x86_64-cuda
ml-linux-x86_64-cuda-generate: ml-linux-x86_64-cuda-generate:
extends: [ ".generate", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ] extends: [ ".generate-x86_64", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ]
image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
ml-linux-x86_64-cuda-build: ml-linux-x86_64-cuda-build:
@ -630,7 +609,7 @@ ml-linux-x86_64-cuda-build:
SPACK_CI_STACK_NAME: ml-linux-x86_64-rocm SPACK_CI_STACK_NAME: ml-linux-x86_64-rocm
ml-linux-x86_64-rocm-generate: ml-linux-x86_64-rocm-generate:
extends: [ ".generate", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ] extends: [ ".generate-x86_64", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ]
image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
ml-linux-x86_64-rocm-build: ml-linux-x86_64-rocm-build:
@ -648,12 +627,13 @@ ml-linux-x86_64-rocm-build:
# Machine Learning - Darwin aarch64 (MPS) # Machine Learning - Darwin aarch64 (MPS)
######################################## ########################################
.ml-darwin-aarch64-mps: .ml-darwin-aarch64-mps:
extends: [".darwin_aarch64"]
variables: variables:
SPACK_CI_STACK_NAME: ml-darwin-aarch64-mps SPACK_CI_STACK_NAME: ml-darwin-aarch64-mps
ml-darwin-aarch64-mps-generate: ml-darwin-aarch64-mps-generate:
tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ] tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ]
extends: [ ".ml-darwin-aarch64-mps", ".darwin-generate"] extends: [ ".ml-darwin-aarch64-mps", ".generate-base"]
ml-darwin-aarch64-mps-build: ml-darwin-aarch64-mps-build:
extends: [ ".ml-darwin-aarch64-mps", ".build" ] extends: [ ".ml-darwin-aarch64-mps", ".build" ]
@ -717,7 +697,7 @@ deprecated-ci-build:
SPACK_CI_STACK_NAME: aws-pcluster-icelake SPACK_CI_STACK_NAME: aws-pcluster-icelake
# aws-pcluster-generate-icelake: # aws-pcluster-generate-icelake:
# extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".generate", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ] # extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".generate-x86_64", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
# aws-pcluster-build-icelake: # aws-pcluster-build-icelake:
# extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".build" ] # extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".build" ]
@ -736,7 +716,7 @@ deprecated-ci-build:
SPACK_CI_STACK_NAME: aws-pcluster-skylake SPACK_CI_STACK_NAME: aws-pcluster-skylake
# aws-pcluster-generate-skylake: # aws-pcluster-generate-skylake:
# extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".generate", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ] # extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".generate-x86_64", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
# aws-pcluster-build-skylake: # aws-pcluster-build-skylake:
# extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".build" ] # extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".build" ]
@ -810,42 +790,12 @@ deprecated-ci-build:
SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1" SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
.generate-cray: .generate-cray:
extends: [ ".base-cray-job" ] extends: [ ".generate-common", ".base-cray-job" ]
stage: generate stage: generate
script: before_script:
- echo $PATH - echo $PATH
- module avail - module avail
- module list - module list
- export SPACK_DISABLE_LOCAL_CONFIG=1
- export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache
- uname -a || true
- grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
- nproc || true
- . "./share/spack/setup-env.sh"
- spack --version
- cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
- spack env activate --without-view .
- export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
- spack
--config-scope "${SPACK_CI_CONFIG_ROOT}"
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
${CI_STACK_CONFIG_SCOPES}
ci generate --check-index-only
--buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
--artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
--output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
after_script:
- cat /proc/loadavg || true
artifacts:
paths:
- "${CI_PROJECT_DIR}/jobs_scratch_dir"
interruptible: true
timeout: 60 minutes
retry:
max: 2
when:
- always
.generate-cray-rhel: .generate-cray-rhel:
tags: [ "cray-rhel-zen4", "public" ] tags: [ "cray-rhel-zen4", "public" ]
@ -903,4 +853,4 @@ e4s-cray-sles-build:
strategy: depend strategy: depend
needs: needs:
- artifacts: True - artifacts: True
job: e4s-cray-sles-generate job: e4s-cray-sles-generate

25
share/spack/gitlab/cloud_pipelines/configs/ci.yaml
View File

@ -10,19 +10,17 @@ ci:
- build-job: - build-job:
before_script-: before_script-:
- - spack list --count # ensure that spack's cache is populated - - spack list --count # ensure that spack's cache is populated
script:: - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
- - spack compiler find - spack compiler find
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi - if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi
- spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
- mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
# AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
- k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust $k - - k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust $k
# UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
- k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust $k - k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust $k
- k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k - k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k
- spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) script::
- spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
after_script: after_script:
- - cat /proc/loadavg || true - - cat /proc/loadavg || true
variables: variables:
@ -47,8 +45,7 @@ ci:
- if [[ $CI_COMMIT_TAG == "develop-"* ]]; then export SPACK_REPLACE_VERSION=develop; fi - if [[ $CI_COMMIT_TAG == "develop-"* ]]; then export SPACK_REPLACE_VERSION=develop; fi
- export SPACK_BUILDCACHE_SOURCE=${SPACK_SOURCE_MIRROR//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}} - export SPACK_BUILDCACHE_SOURCE=${SPACK_SOURCE_MIRROR//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}}
script: script:
- - cd ${SPACK_CONCRETE_ENV_DIR} - - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- echo Copying environment specs from ${SRC_MIRROR} to ${SPACK_BUILDCACHE_DESTINATION} - echo Copying environment specs from ${SRC_MIRROR} to ${SPACK_BUILDCACHE_DESTINATION}
- spack buildcache sync "${SPACK_BUILDCACHE_SOURCE}" "${SPACK_BUILDCACHE_DESTINATION}" - spack buildcache sync "${SPACK_BUILDCACHE_SOURCE}" "${SPACK_BUILDCACHE_DESTINATION}"
- curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub - curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub
@ -89,12 +86,16 @@ ci:
KUBERNETES_MEMORY_REQUEST: "500M" KUBERNETES_MEMORY_REQUEST: "500M"
- any-job: - any-job:
image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
tags: ["spack"] tags: ["spack"]
image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
variables:
# Disable local configs to avoid issues on shell runners
SPACK_DISABLE_LOCAL_CONFIG: "1"
before_script: before_script:
- - export SPACK_USER_CACHE_PATH="${CI_PROJECT_DIR}/_user_cache/"
- - uname -a || true - - uname -a || true
- grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true - grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
- nproc - nproc || true
- - . "./share/spack/setup-env.sh" - - . "./share/spack/setup-env.sh"
- spack --version - spack --version
- spack arch - spack arch

284
share/spack/gitlab/cloud_pipelines/configs/cray-rhel/ci.yaml
View File

@ -5,287 +5,3 @@ ci:
- build-job: - build-job:
variables: variables:
CI_GPG_KEY_ROOT: /etc/protected-runner CI_GPG_KEY_ROOT: /etc/protected-runner
- match_behavior: first
submapping:
- match:
- hipblas
- llvm
- llvm-amdgpu
- pango
- paraview
- py-tensorflow
- py-torch
- qt
- rocblas
- visit
build-job:
tags: [ "spack", "huge" ]
variables:
CI_JOB_SIZE: huge
SPACK_BUILD_JOBS: "12"
- match:
- ascent
- atk
- axom
- cistem
- cmake
- ctffind
- cuda
- dealii
- dray
- dyninst
- ecp-data-vis-sdk
- gcc
- ginkgo
- hdf5
- hpx
- kokkos-kernels
- kokkos-nvcc-wrapper
- lbann
- magma
- mesa
- mfem
- mpich
- netlib-lapack
- nvhpc
- oce
- openblas
- openfoam
- openturns
- parallelio
- plumed
- precice
#- py-tensorflow
#- qt
- raja
- relion
#- rocblas
- rocfft
- rocsolver
- rocsparse
- rust
- slate
- strumpack
- sundials
- trilinos
- umpire
#- visit
- vtk
- vtk-h
- vtk-m
- warpx
- wrf
- wxwidgets
build-job:
tags: [ "spack", "large" ]
variables:
CI_JOB_SIZE: large
SPACK_BUILD_JOBS: "8"
- match:
- adios2
- amrex
- archer
- ascent
- autoconf-archive
- axom
- binutils
- blaspp
- blt
- boost
- butterflypack
- cabana
- caliper
- camp
- chai
- conduit
- curl
- datatransferkit
- double-conversion
- dray
- eigen
- faodel
- ffmpeg
- fftw
- fortrilinos
- gettext
- gperftools
- gptune
- hdf5
- heffte
- hpctoolkit
- hwloc
- hydrogen
- hypre
- kokkos
- lammps
- lapackpp
- legion
- libtool
- libxml2
- libzmq
- llvm-openmp-ompt
- mbedtls
- mfem
- mpich
- mvapich2
- nasm
- netlib-scalapack
- omega-h
- openblas
- openjpeg
- openmpi
- openpmd-api
- pagmo2
- papyrus
- parsec
- pdt
- pegtl
- petsc
- pumi
- py-beniget
- py-cinemasci
- pygmo
- py-ipython-genutils
- py-packaging
- py-petsc4py
- py-scipy
- py-statsmodels
- py-warlock
- py-warpx
- raja
- samrai
- slepc
- slurm
- sqlite
- strumpack
- sundials
- superlu-dist
- tasmanian
- tau
- upcxx
- vtk
- vtk-h
- vtk-m
- zfp
build-job:
tags: [ "spack", "medium" ]
variables:
CI_JOB_SIZE: "medium"
SPACK_BUILD_JOBS: "2"
- match:
- alsa-lib
- ant
- antlr
- argobots
- autoconf-archive
- automake
- berkeley-db
- bison
- blt
- bzip2
- camp
- cmake
- curl
- czmq
- darshan-util
- diffutils
- docbook-xml
- exmcutils
- expat
- findutils
- flit
- freetype
- gawk
- gdbm
- gettext
- glib
- gmake
- gotcha
- hpcviewer
- hwloc
- jansson
- json-c
- libbsd
- libedit
- libevent
- libfabric
- libffi
- libgcrypt
- libiconv
- libidn2
- libjpeg-turbo
- libmd
- libnrm
- libpciaccess
- libpng
- libsigsegv
- libsodium
- libunistring
- libunwind
- libxml2
- libyaml
- libzmq
- lua
- lua-luaposix
- lz4
- m4
- meson
- metis
- mpfr
- ncurses
- ninja
- numactl
- openblas
- openjdk
- openssh
- openssl
- papi
- parallel-netcdf
- pcre
- pcre2
- pdsh
- perl
- perl-data-dumper
- pkgconf
- py-alembic
- py-cffi
- py-cycler
- py-decorator
- py-idna
- py-jsonschema
- py-kiwisolver
- py-mistune
- py-pycparser
- py-setuptools
- py-setuptools-scm
- py-six
- py-testpath
- py-wheel
- qhull
- readline
- sed
- slurm
- snappy
- sqlite
- superlu
- swig
- tar
- tcl
- texinfo
- tut
- unzip
- util-linux-uuid
- util-macros
- xz
- yaml-cpp
- zfp
- zlib
- zstd
build-job:
tags: [ "spack", "small" ]
variables:
CI_JOB_SIZE: "small"
SPACK_BUILD_JOBS: "1"

284
share/spack/gitlab/cloud_pipelines/configs/cray-sles/ci.yaml
View File

@ -5,287 +5,3 @@ ci:
- build-job: - build-job:
variables: variables:
CI_GPG_KEY_ROOT: /etc/protected-runner CI_GPG_KEY_ROOT: /etc/protected-runner
- match_behavior: first
submapping:
- match:
- hipblas
- llvm
- llvm-amdgpu
- pango
- paraview
- py-tensorflow
- py-torch
- qt
- rocblas
- visit
build-job:
tags: [ "spack", "huge" ]
variables:
CI_JOB_SIZE: huge
SPACK_BUILD_JOBS: "12"
- match:
- ascent
- atk
- axom
- cistem
- cmake
- ctffind
- cuda
- dealii
- dray
- dyninst
- ecp-data-vis-sdk
- gcc
- ginkgo
- hdf5
- hpx
- kokkos-kernels
- kokkos-nvcc-wrapper
- lbann
- magma
- mesa
- mfem
- mpich
- netlib-lapack
- nvhpc
- oce
- openblas
- openfoam
- openturns
- parallelio
- plumed
- precice
#- py-tensorflow
#- qt
- raja
- relion
#- rocblas
- rocfft
- rocsolver
- rocsparse
- rust
- slate
- strumpack
- sundials
- trilinos
- umpire
#- visit
- vtk
- vtk-h
- vtk-m
- warpx
- wrf
- wxwidgets
build-job:
tags: [ "spack", "large" ]
variables:
CI_JOB_SIZE: large
SPACK_BUILD_JOBS: "8"
- match:
- adios2
- amrex
- archer
- ascent
- autoconf-archive
- axom
- binutils
- blaspp
- blt
- boost
- butterflypack
- cabana
- caliper
- camp
- chai
- conduit
- curl
- datatransferkit
- double-conversion
- dray
- eigen
- faodel
- ffmpeg
- fftw
- fortrilinos
- gettext
- gperftools
- gptune
- hdf5
- heffte
- hpctoolkit
- hwloc
- hydrogen
- hypre
- kokkos
- lammps
- lapackpp
- legion
- libtool
- libxml2
- libzmq
- llvm-openmp-ompt
- mbedtls
- mfem
- mpich
- mvapich2
- nasm
- netlib-scalapack
- omega-h
- openblas
- openjpeg
- openmpi
- openpmd-api
- pagmo2
- papyrus
- parsec
- pdt
- pegtl
- petsc
- pumi
- py-beniget
- py-cinemasci
- pygmo
- py-ipython-genutils
- py-packaging
- py-petsc4py
- py-scipy
- py-statsmodels
- py-warlock
- py-warpx
- raja
- samrai
- slepc
- slurm
- sqlite
- strumpack
- sundials
- superlu-dist
- tasmanian
- tau
- upcxx
- vtk
- vtk-h
- vtk-m
- zfp
build-job:
tags: [ "spack", "medium" ]
variables:
CI_JOB_SIZE: "medium"
SPACK_BUILD_JOBS: "2"
- match:
- alsa-lib
- ant
- antlr
- argobots
- autoconf-archive
- automake
- berkeley-db
- bison
- blt
- bzip2
- camp
- cmake
- curl
- czmq
- darshan-util
- diffutils
- docbook-xml
- exmcutils
- expat
- findutils
- flit
- freetype
- gawk
- gdbm
- gettext
- glib
- gmake
- gotcha
- hpcviewer
- hwloc
- jansson
- json-c
- libbsd
- libedit
- libevent
- libfabric
- libffi
- libgcrypt
- libiconv
- libidn2
- libjpeg-turbo
- libmd
- libnrm
- libpciaccess
- libpng
- libsigsegv
- libsodium
- libunistring
- libunwind
- libxml2
- libyaml
- libzmq
- lua
- lua-luaposix
- lz4
- m4
- meson
- metis
- mpfr
- ncurses
- ninja
- numactl
- openblas
- openjdk
- openssh
- openssl
- papi
- parallel-netcdf
- pcre
- pcre2
- pdsh
- perl
- perl-data-dumper
- pkgconf
- py-alembic
- py-cffi
- py-cycler
- py-decorator
- py-idna
- py-jsonschema
- py-kiwisolver
- py-mistune
- py-pycparser
- py-setuptools
- py-setuptools-scm
- py-six
- py-testpath
- py-wheel
- qhull
- readline
- sed
- slurm
- snappy
- sqlite
- superlu
- swig
- tar
- tcl
- texinfo
- tut
- unzip
- util-linux-uuid
- util-macros
- xz
- yaml-cpp
- zfp
- zlib
- zstd
build-job:
tags: [ "spack", "small" ]
variables:
CI_JOB_SIZE: "small"
SPACK_BUILD_JOBS: "1"

4
share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml Normal file
View File

@ -0,0 +1,4 @@
ci:
pipeline-gen:
- build-job-remove:
image: macos-run-on-metal

5
share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
View File

@ -1,5 +1,10 @@
ci: ci:
pipeline-gen: pipeline-gen:
- build-job:
before_script-:
# Test package relocation on linux using a modified prefix
# This is not well supported on MacOS (https://github.com/spack/spack/issues/37162)
- - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
- match_behavior: first - match_behavior: first
submapping: submapping:
- match: - match:

5
share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml
View File

@ -1,4 +1,7 @@
spack: spack:
include:
- $spack/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
view: false view: false
concretizer: concretizer:
@ -62,4 +65,4 @@ spack:
mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray-rhel" } mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray-rhel" }
cdash: cdash:
build-group: E4S Cray build-group: E4S Cray

5
share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-sles/spack.yaml
View File

@ -1,4 +1,7 @@
spack: spack:
include:
- $spack/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
view: false view: false
concretizer: concretizer:
@ -45,4 +48,4 @@ spack:
mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray-sles" } mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray-sles" }
cdash: cdash:
build-group: E4S Cray SLES build-group: E4S Cray SLES

50
share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
View File

@ -1,19 +1,6 @@
spack: spack:
view: false view: false
concretizer:
unify: false
reuse: false
config:
concretizer: clingo
db_lock_timeout: 120
install_tree:
root: $spack/opt/spack
padded_length: 256
projections:
all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
packages: packages:
all: all:
require: target=aarch64 require: target=aarch64
@ -97,42 +84,11 @@ spack:
ci: ci:
pipeline-gen: pipeline-gen:
- build-job-remove: - build-job-remove:
image: no-image tags: [ spack, public ]
tags: [spack, public]
- build-job: - build-job:
variables:
CI_GPG_KEY_ROOT: /etc/protected-runner
tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ] tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ]
script::
- - spack compiler find
- cd ${SPACK_CONCRETE_ENV_DIR}
- spack env activate --without-view .
- if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; fi
- mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
# AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
- if [[ -r /etc/protected-runner/e4s.gpg ]]; then spack gpg trust /etc/protected-runner/e4s.gpg; fi
# UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
- if [[ -r /etc/protected-runner/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /etc/protected-runner/intermediate_ci_signing_key.gpg; fi
- if [[ -r /etc/protected-runner/spack_public_key.gpg ]]; then spack gpg trust /etc/protected-runner/spack_public_key.gpg; fi
- spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
after_script:
- - cat /proc/loadavg || true
- signing-job:
image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] }
tags: ["aws"]
script:
- - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp
- /sign.sh
- aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
- aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp --recursive --exclude "*" --include "*.pub"
- any-job:
image: "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18"
tags: ["spack"]
before_script:
- - uname -a || true
- grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
- nproc || true
- - . "./share/spack/setup-env.sh"
- spack --version
- spack arch
cdash: cdash:
build-group: Machine Learning MPS build-group: Machine Learning MPS