zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

E4S Cray CI Stack (#37837)

Browse Source 
* e4s cray ci stack

* e4s ci: add cray

* add zen4 tag

* WIP: new defintions just for cray

* updates

* remove ci signing job overrride, not necessary

* echo $PATH and show modules loaded

* add mirror

* add external def for cray-libsci

* comment out quantum-espresso

* use /etc/protected-runner as key path

* cray ci stack: do not remove tags: [spack, public]

* make cray stack composable

* generate job should run on public tagged runner, override default config:install_tree:root

* CI: Use relative path in default script

* CI: Use relative includes paths for shell runners

* Use concrete_env_dir for relpath

* ml-darwin-aarch64-mps: jax has bazel codesign issue

---------

Co-authored-by: Scott Wittenburg <scott.wittenburg@kitware.com>
Co-authored-by: Ryan Krattiger <ryan.krattiger@kitware.com>
This commit is contained in:
eugeneswalker 2023-06-06 20:41:32 -05:00 committed by GitHub
parent fa9fb60df3
commit 5f1bc15e80
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 503 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/spack/spack/ci.py
View File

@ -946,7 +946,7 @@ def generate_gitlab_ci_yaml(
# Add config scopes to environment # Add config scopes to environment
env_includes = env_yaml_root["spack"].get("include", []) env_includes = env_yaml_root["spack"].get("include", [])
cli_scopes = [ cli_scopes = [
os.path.abspath(s.path) os.path.relpath(s.path, concrete_env_dir)
for s in cfg.scopes().values() for s in cfg.scopes().values()
if type(s) == cfg.ImmutableConfigScope if type(s) == cfg.ImmutableConfigScope
and s.path not in env_includes and s.path not in env_includes

91
share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
View File

@ -4,6 +4,11 @@ default:
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
# CI Platform-Arch # CI Platform-Arch
.cray_zen4:
variables:
SPACK_TARGET_PLATFORM: "cray"
SPACK_TARGET_ARCH: "zen4"
.darwin_x86_64: .darwin_x86_64:
variables: variables:
SPACK_TARGET_PLATFORM: "darwin" SPACK_TARGET_PLATFORM: "darwin"
@ -815,3 +820,89 @@ aws-pcluster-build-neoverse_v1:
needs: needs:
- artifacts: True - artifacts: True
job: aws-pcluster-generate-neoverse_v1 job: aws-pcluster-generate-neoverse_v1
# Cray definitions
.base-cray-job:
variables:
SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-cray/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
AWS_ACCESS_KEY_ID: ${CRAY_MIRRORS_AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${CRAY_MIRRORS_AWS_SECRET_ACCESS_KEY}
rules:
- if: $CI_COMMIT_REF_NAME == "develop"
# Pipelines on develop only rebuild what is missing from the mirror
when: always
variables:
SPACK_PIPELINE_TYPE: "spack_protected_branch"
- if: $CI_COMMIT_REF_NAME =~ /^pr[\d]+_.*$/
# Pipelines on PR branches rebuild only what's missing, and do extra pruning
when: always
variables:
SPACK_PIPELINE_TYPE: "spack_pull_request"
SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-cray/prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
SPACK_PRUNE_UNTOUCHED: "True"
SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
.generate-cray:
tags: [ "cce@15.0.1", "cray-zen4", "public" ]
extends: [ ".base-cray-job" ]
stage: generate
script:
- echo $PATH
- module avail
- module list
- export SPACK_DISABLE_LOCAL_CONFIG=1
- export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache
- uname -a || true
- grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
- nproc || true
- . "./share/spack/setup-env.sh"
- spack --version
- cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
- spack env activate --without-view .
- export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
- spack
--config-scope "${SPACK_CI_CONFIG_ROOT}"
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
${CI_STACK_CONFIG_SCOPES}
ci generate --check-index-only
--buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
--artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
--output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
after_script:
- cat /proc/loadavg || true
artifacts:
paths:
- "${CI_PROJECT_DIR}/jobs_scratch_dir"
interruptible: true
timeout: 60 minutes
retry:
max: 2
when:
- always
.build-cray:
extends: [ ".base-cray-job" ]
stage: build
#######################################
# E4S - Cray
#######################################
.e4s-cray:
extends: [ ".cray_zen4" ]
variables:
SPACK_CI_STACK_NAME: e4s-cray
e4s-cray-generate:
extends: [ ".generate-cray", ".e4s-cray" ]
e4s-cray-build:
extends: [ ".build-cray", ".e4s-cray" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: e4s-cray-generate
strategy: depend
needs:
- artifacts: True
job: e4s-cray-generate

295
share/spack/gitlab/cloud_pipelines/configs/cray/ci.yaml Normal file
View File

@ -0,0 +1,295 @@
ci:
pipeline-gen:
- build-job-remove:
image: no-image
- build-job:
script+:
# AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
- if [[ -r /etc/protected-runner/e4s.gpg ]]; then spack gpg trust /etc/protected-runner/e4s.gpg; fi
# UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
- if [[ -r /etc/protected-runner/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /etc/protected-runner/intermediate_ci_signing_key.gpg; fi
- if [[ -r /etc/protected-runner/spack_public_key.gpg ]]; then spack gpg trust /etc/protected-runner/spack_public_key.gpg; fi
- match_behavior: first
submapping:
- match:
- hipblas
- llvm
- llvm-amdgpu
- pango
- paraview
- py-tensorflow
- py-torch
- qt
- rocblas
- visit
build-job:
tags: [ "spack", "huge" ]
variables:
CI_JOB_SIZE: huge
SPACK_BUILD_JOBS: "12"
- match:
- ascent
- atk
- axom
- cistem
- cmake
- ctffind
- cuda
- dealii
- dray
- dyninst
- ecp-data-vis-sdk
- gcc
- ginkgo
- hdf5
- hpx
- kokkos-kernels
- kokkos-nvcc-wrapper
- lbann
- magma
- mesa
- mfem
- mpich
- netlib-lapack
- nvhpc
- oce
- openblas
- openfoam
- openturns
- parallelio
- plumed
- precice
#- py-tensorflow
#- qt
- raja
- relion
#- rocblas
- rocfft
- rocsolver
- rocsparse
- rust
- slate
- strumpack
- sundials
- trilinos
- umpire
#- visit
- vtk
- vtk-h
- vtk-m
- warpx
- wrf
- wxwidgets
build-job:
tags: [ "spack", "large" ]
variables:
CI_JOB_SIZE: large
SPACK_BUILD_JOBS: "8"
- match:
- adios2
- amrex
- archer
- ascent
- autoconf-archive
- axom
- binutils
- blaspp
- blt
- boost
- butterflypack
- cabana
- caliper
- camp
- chai
- conduit
- curl
- datatransferkit
- double-conversion
- dray
- eigen
- faodel
- ffmpeg
- fftw
- fortrilinos
- gettext
- gperftools
- gptune
- hdf5
- heffte
- hpctoolkit
- hwloc
- hydrogen
- hypre
- kokkos
- lammps
- lapackpp
- legion
- libtool
- libxml2
- libzmq
- llvm-openmp-ompt
- mbedtls
- mfem
- mpich
- mvapich2
- nasm
- netlib-scalapack
- omega-h
- openblas
- openjpeg
- openmpi
- openpmd-api
- pagmo2
- papyrus
- parsec
- pdt
- pegtl
- petsc
- pumi
- py-beniget
- py-cinemasci
- pygmo
- py-ipython-genutils
- py-packaging
- py-petsc4py
- py-scipy
- py-statsmodels
- py-warlock
- py-warpx
- raja
- samrai
- slepc
- slurm
- sqlite
- strumpack
- sundials
- superlu-dist
- tasmanian
- tau
- upcxx
- vtk
- vtk-h
- vtk-m
- zfp
build-job:
tags: [ "spack", "medium" ]
variables:
CI_JOB_SIZE: "medium"
SPACK_BUILD_JOBS: "2"
- match:
- alsa-lib
- ant
- antlr
- argobots
- autoconf-archive
- automake
- berkeley-db
- bison
- blt
- bzip2
- camp
- cmake
- curl
- czmq
- darshan-util
- diffutils
- docbook-xml
- exmcutils
- expat
- findutils
- flit
- freetype
- gawk
- gdbm
- gettext
- glib
- gmake
- gotcha
- hpcviewer
- hwloc
- jansson
- json-c
- libbsd
- libedit
- libevent
- libfabric
- libffi
- libgcrypt
- libiconv
- libidn2
- libjpeg-turbo
- libmd
- libnrm
- libpciaccess
- libpng
- libsigsegv
- libsodium
- libunistring
- libunwind
- libxml2
- libyaml
- libzmq
- lua
- lua-luaposix
- lz4
- m4
- meson
- metis
- mpfr
- ncurses
- ninja
- numactl
- openblas
- openjdk
- openssh
- openssl
- papi
- parallel-netcdf
- pcre
- pcre2
- pdsh
- perl
- perl-data-dumper
- pkgconf
- py-alembic
- py-cffi
- py-cycler
- py-decorator
- py-idna
- py-jsonschema
- py-kiwisolver
- py-mistune
- py-pycparser
- py-setuptools
- py-setuptools-scm
- py-six
- py-testpath
- py-wheel
- qhull
- readline
- sed
- slurm
- snappy
- sqlite
- superlu
- swig
- tar
- tcl
- texinfo
- tut
- unzip
- util-linux-uuid
- util-macros
- xz
- yaml-cpp
- zfp
- zlib
- zstd
build-job:
tags: [ "spack", "small" ]
variables:
CI_JOB_SIZE: "small"
SPACK_BUILD_JOBS: "1"

9
share/spack/gitlab/cloud_pipelines/configs/cray/config.yaml Normal file
View File

@ -0,0 +1,9 @@
config:
concretizer: clingo
db_lock_timeout: 120
install_tree:
root: $spack/opt/spack
padded_length: 256
projections:
all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'

4
share/spack/gitlab/cloud_pipelines/configs/cray/zen4/ci.yaml Normal file
View File

@ -0,0 +1,4 @@
ci:
pipeline-gen:
- build-job:
tags: ["cce@15.0.1", "cray-zen4"]

102
share/spack/gitlab/cloud_pipelines/stacks/e4s-cray/spack.yaml Normal file
View File

@ -0,0 +1,102 @@
spack:
view: false
concretizer:
reuse: false
unify: false
compilers:
- compiler:
spec: cce@15.0.1
paths:
cc: cc
cxx: CC
f77: ftn
fc: ftn
flags: {}
operating_system: rhel8
target: any
modules:
- PrgEnv-cray/8.3.3
- cce/15.0.1
environment:
set:
MACHTYPE: x86_64
- compiler:
spec: gcc@11.2.0
paths:
cc: gcc
cxx: g++
f77: gfortran
fc: gfortran
flags: {}
operating_system: rhel8
target: any
modules:
- PrgEnv-gnu
- gcc/11.2.0
environment: {}
packages:
all:
require: '%cce@15.0.1'
compiler: [cce@15.0.1]
providers:
blas: [cray-libsci]
lapack: [cray-libsci]
mpi: [cray-mpich]
tbb: [intel-tbb]
scalapack: [netlib-scalapack]
target: [zen4]
variants: +mpi
binutils:
variants: +ld +gold +headers +libiberty ~nls
hdf5:
variants: +fortran +hl +shared
libunwind:
variants: +pic +xz
ncurses:
require: '@6.3 +termlib'
openblas:
require: '@0.3.20'
variants: threads=openmp
python:
require: '@3.7.15'
xz:
variants: +pic
elfutils:
variants: +bzip2 ~nls +xz
require: '%gcc'
# EXTERNALS
cray-mpich:
buildable: false
externals:
- spec: cray-mpich@8.1.25 %cce@15.0.1
prefix: /opt/cray/pe/mpich/8.1.25/ofi/cray/10.0
modules:
- cray-mpich/8.1.25
cray-libsci:
buildable: false
externals:
- spec: cray-libsci@23.02.1.1 %cce@15.0.1
prefix: /opt/cray/pe/libsci/23.02.1.1/CRAY/9.0/x86_64/
modules:
- cray-libsci/23.02.1.1
specs:
- butterflypack
- hypre
- kokkos
- kokkos-kernels
- petsc
- raja
- slepc
- superlu-dist
- tau
mirrors: { "mirror": "s3://spack-binaries-cray/develop/e4s-cray" }
cdash:
build-group: E4S Cray

2
share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml
View File

@ -26,7 +26,7 @@ spack:
- py-transformers - py-transformers
# JAX # JAX
- py-jax # - py-jax # bazel codesign
# - py-jaxlib # bazel codesign # - py-jaxlib # bazel codesign
# Keras # Keras