zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pipelines: simplify and lint aws-pcluster-* (#47989)

Browse Source
This commit is contained in:
Massimiliano Culpo 2024-12-10 12:16:51 +01:00 committed by GitHub
parent 30c00353d4
commit b50dbb8604
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 59 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/spack/spack/util/spack_yaml.py
View File

@ -416,10 +416,12 @@ def dump_config(data, stream, *, default_flow_style=False, blame=False):
if blame:
handler = ConfigYAML(yaml_type=YAMLType.ANNOTATED_SPACK_CONFIG_FILE)
handler.yaml.default_flow_style = default_flow_style
handler.yaml.width = maxint
return _dump_annotated(handler, data, stream)
handler = ConfigYAML(yaml_type=YAMLType.SPACK_CONFIG_FILE)
handler.yaml.default_flow_style = default_flow_style
handler.yaml.width = maxint
return handler.dump(data, stream)

81
share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
View File

@ -1,6 +1,5 @@
spack:
view: false
definitions:
- apps:
- gromacs
@ -9,20 +8,18 @@ spack:
- openfoam
- quantum-espresso
- wrf
- targets:
- 'target=neoverse_v1'
- 'target=neoverse_n1'
- target=neoverse_v1
- target=neoverse_n1
specs:
- matrix:
- [$apps]
- [$targets]
- [$apps]
- [$targets]
ci:
pipeline-gen:
- build-job:
image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07", "entrypoint": [""] }
tags: ["aarch64"]
image: {name: ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07, entrypoint: ['']}
tags: [aarch64]
before_script:
- - . "./share/spack/setup-env.sh"
- . /etc/profile.d/modules.sh
@ -31,27 +28,12 @@ spack:
- export PATH=/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-aarch64/gcc-7.3.1/binutils-2.37-2yxz3xsjfmesxujxtlrgcctxlyilynmp/bin:$PATH
- signing-job:
before_script:
# Do not distribute Intel & ARM binaries
- - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
- for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
# Do not distribute Intel & ARM binaries
- - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
- for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
cdash:
build-group: AWS Packages
compilers:
- compiler:
environment: {}
extra_rpaths: []
flags: {}
modules: []
operating_system: amzn2
paths:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
f77: /usr/bin/gfortran
fc: /usr/bin/gfortran
spec: gcc@=7.3.1
target: aarch64
- compiler:
environment: {}
extra_rpaths: []
@ -65,41 +47,34 @@ spack:
fc: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-aarch64/gcc-7.3.1/gcc-12.4.0-v6wxye6ijzrxnzxftcwnpu3psohsjl2b/bin/gfortran
spec: gcc@=12.4.0
target: aarch64
packages:
acfl:
require:
- one_of: ["%gcc target=aarch64"]
message: "Clang based compilers need GCC libraries and they should be made available for the wide range of CPUs they actually support.
Edit $SPACK_ROOT/etc/spack/packages.yaml to change this default."
- '%gcc target=aarch64'
gromacs:
require:
- one_of:
- "gromacs@2024.3 %arm ^fftw ^openmpi"
- "gromacs@2024.3 %gcc ^armpl-gcc ^openmpi"
- gromacs@2024.3 %gcc ^armpl-gcc ^openmpi
libfabric:
buildable: true
externals:
- prefix: /opt/amazon/efa/
spec: libfabric@1.17.0
require: ['fabrics=shm,efa']
- prefix: /opt/amazon/efa/
spec: libfabric@1.17.0
require:
- fabrics=shm,efa
llvm:
variants: ~lldb
mpas-model:
require:
- one_of:
- "precision=single make_target=llvm %arm ^parallelio+pnetcdf"
- "precision=single %gcc ^parallelio+pnetcdf"
- precision=single %gcc ^parallelio+pnetcdf
mpich:
require: "mpich pmi=pmi2 device=ch4 netmod=ofi +slurm"
require:
- mpich pmi=pmi2 device=ch4 netmod=ofi +slurm
nvhpc:
require:
- one_of:
- "nvhpc %gcc target=aarch64"
message: "NVHPC should be built with GCC and should be made available for the wide range of CPUs they actually support.
Edit $SPACK_ROOT/etc/spack/packages.yaml to change this default."
- nvhpc %gcc target=aarch64
openfoam:
require: "openfoam %gcc ^scotch@6.0.9"
require:
- openfoam ^scotch@6.0.9
openmpi:
variants: ~atomics ~cuda ~cxx ~cxx_exceptions ~internal-hwloc ~java +legacylaunchers ~lustre ~memchecker +pmi +romio ~singularity +vt +wrapper-rpath fabrics=ofi schedulers=slurm
require: '@4:'
@ -108,19 +83,15 @@ spack:
# require:
# - one_of: ["palace cxxflags=\"-include cstdint\" ^fmt@9.1.0"]
pmix:
require: "pmix@3:"
require: 'pmix@3:'
quantum-espresso:
require: "quantum-espresso@6.6 %gcc ^armpl-gcc"
require:
- quantum-espresso@6.6 %gcc ^armpl-gcc
slurm:
buildable: false
externals:
- prefix: /opt/slurm
spec: slurm@22.05.8 +pmix
wrf:
require:
- one_of:
- "wrf%arm"
- "wrf%gcc"
- prefix: /opt/slurm
spec: slurm@22.05.8 +pmix
all:
compiler: [gcc, arm, nvhpc, clang]
providers:

139
share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/spack.yaml
View File

@ -1,31 +1,28 @@
spack:
view: false
definitions:
- apps:
- gromacs %oneapi
- lammps %oneapi
# earliest oneapi version with fix does not run on AmazonLinux2, see https://github.com/spack/spack/pull/46457
# - mpas-model %oneapi
# earliest oneapi version with fix does not run on AmazonLinux2, see https://github.com/spack/spack/pull/46457
# - mpas-model %oneapi
- openfoam %gcc
- palace %oneapi ^superlu-dist%oneapi # hack: force fortran-rt provider through superlu-dist
# TODO: Find out how to make +ipo cmake flag work.
# - quantum-espresso %oneapi
- palace %oneapi ^superlu-dist%oneapi # hack: force fortran-rt provider through superlu-dist
# TODO: Find out how to make +ipo cmake flag work.
# - quantum-espresso %oneapi
- openmpi %oneapi
- wrf %oneapi
- targets:
- 'target=x86_64_v4'
- 'target=x86_64_v3'
- target=x86_64_v4
- target=x86_64_v3
specs:
- matrix:
- [$apps]
- [$targets]
- [$apps]
- [$targets]
ci:
pipeline-gen:
- build-job:
image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07", "entrypoint": [""] }
image: {name: ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07, entrypoint: ['']}
before_script:
- - . "./share/spack/setup-env.sh"
- . /etc/profile.d/modules.sh
@ -34,13 +31,11 @@ spack:
- export PATH=/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-7.3.1/binutils-2.37-qvccg7zpskturysmr4bzbsfrx34kvazo/bin:$PATH
- signing-job:
before_script:
# Do not distribute Intel & ARM binaries
- - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
- for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
# Do not distribute Intel & ARM binaries
- - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
- for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
cdash:
build-group: AWS Packages
compilers:
- compiler:
environment: {}
@ -69,131 +64,59 @@ spack:
fc: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-12.4.0/intel-oneapi-compilers-2024.1.0-f5u3psfhbwscasajkn324igtupn3blop/compiler/2024.1/bin/ifx
spec: oneapi@=2024.1.0
target: x86_64
packages:
cpio:
require:
- one_of:
- "cflags=-std=c18 target=x86_64_v4"
- "cflags=-std=c18 target=x86_64_v3"
when: "%intel"
gettext:
# Newer gettext cannot build with gcc@12 and old AL2 glibc headers
# Older gettext versions do not build correctly with oneapi.
require:
- one_of:
- '@:0.20'
- '%oneapi'
- one_of: ['@:0.20 %gcc', '%oneapi']
gromacs:
require:
- one_of:
- "+intel_provided_gcc ^intel-oneapi-mkl target=x86_64_v4"
- "+intel_provided_gcc ^intel-oneapi-mkl target=x86_64_v3"
when: "%intel"
- one_of:
- "+intel_provided_gcc target=x86_64_v4 ^intel-oneapi-mkl"
- "+intel_provided_gcc target=x86_64_v3 ^intel-oneapi-mkl"
when: "%oneapi"
intel-oneapi-compilers:
require: "intel-oneapi-compilers %gcc target=x86_64_v3"
- +intel_provided_gcc ^intel-oneapi-mkl
intel-oneapi-mpi:
variants: +external-libfabric generic-names=True
lammps:
require:
- one_of:
- "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package +intel fft=mkl ^intel-oneapi-mkl target=x86_64_v4"
- "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package fft=mkl ^intel-oneapi-mkl target=x86_64_v3"
when: "%intel"
- one_of:
- "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package +intel fft=mkl ^intel-oneapi-mkl target=x86_64_v4"
- "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package fft=mkl ^intel-oneapi-mkl target=x86_64_v3"
when: "%oneapi"
libidn2:
require:
- one_of:
- "cflags=-std=c18 target=x86_64_v4"
- "cflags=-std=c18 target=x86_64_v3"
when: "%intel"
- lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package fft=mkl ^intel-oneapi-mkl
- one_of: [+intel target=x86_64_v4, target=x86_64_v3]
libfabric:
buildable: true
externals:
- prefix: /opt/amazon/efa/
spec: libfabric@1.17.0
require: ['fabrics=shm,efa']
libunistring:
- prefix: /opt/amazon/efa/
spec: libfabric@1.17.0
require:
- one_of:
- "cflags=-std=c18 target=x86_64_v4"
- "cflags=-std=c18 target=x86_64_v3"
when: "%intel"
- fabrics=shm,efa
mpas-model:
require:
- one_of:
- "precision=single ^parallelio+pnetcdf target=x86_64_v4"
- "precision=single ^parallelio+pnetcdf target=x86_64_v3"
when: "%intel"
- one_of:
- "precision=single ^parallelio+pnetcdf target=x86_64_v4"
- "precision=single ^parallelio+pnetcdf target=x86_64_v3"
when: "%oneapi"
- spec: precision=single ^parallelio+pnetcdf
when: '%oneapi'
mpich:
require:
- one_of:
- "mpich pmi=pmi2 device=ch4 netmod=ofi +slurm target=x86_64_v4"
- "mpich pmi=pmi2 device=ch4 netmod=ofi +slurm target=x86_64_v3"
- mpich pmi=pmi2 device=ch4 netmod=ofi +slurm
openfoam:
require:
- one_of:
- "openfoam %gcc ^scotch@6.0.9 target=x86_64_v4"
- "openfoam %gcc ^scotch@6.0.9 target=x86_64_v3"
- openfoam %gcc ^scotch@6.0.9
openmpi:
variants: ~atomics ~cuda ~cxx ~cxx_exceptions ~internal-hwloc ~java +legacylaunchers ~lustre ~memchecker +pmi +romio ~singularity +vt +wrapper-rpath fabrics=ofi schedulers=slurm
require:
- one_of:
- "openmpi @4: target=x86_64_v4"
- "openmpi @4: target=x86_64_v3"
- 'openmpi @4:'
palace:
require:
- one_of:
- "palace ^fmt@9.1.0 target=x86_64_v4"
- "palace ^fmt@9.1.0 target=x86_64_v3"
when: "%oneapi"
- one_of:
- "palace ^fmt@9.1.0"
when: "%gcc"
- palace ^fmt@9.1.0
pmix:
require:
- one_of:
- "pmix@3: target=x86_64_v4"
- "pmix@3: target=x86_64_v3"
- 'pmix@3:'
quantum-espresso:
require:
- one_of:
- "quantum-espresso@6.6 ^intel-oneapi-mkl+cluster target=x86_64_v4"
- "quantum-espresso@6.6 ^intel-oneapi-mkl+cluster target=x86_64_v3"
when: "%intel"
- one_of:
- "quantum-espresso@6.6 ^intel-oneapi-mkl+cluster target=x86_64_v4"
- "quantum-espresso@6.6 ^intel-oneapi-mkl+cluster target=x86_64_v3"
when: "%oneapi"
- quantum-espresso@6.6 ^intel-oneapi-mkl+cluster
slurm:
buildable: false
externals:
- prefix: /opt/slurm
spec: slurm@22.05.8 +pmix
- prefix: /opt/slurm
spec: slurm@22.05.8 +pmix
wrf:
require:
- one_of:
- "wrf@4 build_type=dm+sm target=x86_64_v4"
- "wrf@4 build_type=dm+sm target=x86_64_v3"
- "wrf@4.2.2 +netcdf_classic fflags=\"-fp-model fast=2 -no-heap-arrays -no-prec-div -no-prec-sqrt -fno-common\" build_type=dm+sm target=x86_64_v3"
when: "%intel"
- one_of:
- "wrf@4 build_type=dm+sm target=x86_64_v4"
- "wrf@4 build_type=dm+sm target=x86_64_v3"
- "wrf@4.2.2 +netcdf_classic fflags=\"-fp-model fast=2 -no-heap-arrays -no-prec-div -no-prec-sqrt -fno-common\" build_type=dm+sm target=x86_64_v3"
when: "%oneapi"
- wrf@4 build_type=dm+sm
all:
compiler: [oneapi, gcc]
permissions: