pipelines: simplify and lint aws-pcluster-* (#47989)

lib/spack/spack/util/spack_yaml.py

@@ -416,10 +416,12 @@ def dump_config(data, stream, *, default_flow_style=False, blame=False):
     if blame:
         handler = ConfigYAML(yaml_type=YAMLType.ANNOTATED_SPACK_CONFIG_FILE)
         handler.yaml.default_flow_style = default_flow_style
+        handler.yaml.width = maxint
         return _dump_annotated(handler, data, stream)
 
     handler = ConfigYAML(yaml_type=YAMLType.SPACK_CONFIG_FILE)
     handler.yaml.default_flow_style = default_flow_style
+    handler.yaml.width = maxint
     return handler.dump(data, stream)
 
 

share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml

@@ -1,6 +1,5 @@
 spack:
   view: false
-
   definitions:
   - apps:
     - gromacs
@@ -9,11 +8,9 @@ spack:
     - openfoam
     - quantum-espresso
     - wrf
-
   - targets:
-      - 'target=neoverse_v1'
+    - target=neoverse_v1
-      - 'target=neoverse_n1'
+    - target=neoverse_n1
-
   specs:
   - matrix:
     - [$apps]
@@ -21,8 +18,8 @@ spack:
   ci:
     pipeline-gen:
     - build-job:
-        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07", "entrypoint": [""] }
+        image: {name: ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07, entrypoint: ['']}
-        tags: ["aarch64"]
+        tags: [aarch64]
         before_script:
         - - . "./share/spack/setup-env.sh"
           - . /etc/profile.d/modules.sh
@@ -34,24 +31,9 @@ spack:
             # Do not distribute Intel & ARM binaries
         - - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
           - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
-
   cdash:
     build-group: AWS Packages
-
   compilers:
-  - compiler:
-      environment: {}
-      extra_rpaths: []
-      flags: {}
-      modules: []
-      operating_system: amzn2
-      paths:
-        cc: /usr/bin/gcc
-        cxx: /usr/bin/g++
-        f77: /usr/bin/gfortran
-        fc: /usr/bin/gfortran
-      spec: gcc@=7.3.1
-      target: aarch64
   - compiler:
       environment: {}
       extra_rpaths: []
@@ -65,41 +47,34 @@ spack:
         fc: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-aarch64/gcc-7.3.1/gcc-12.4.0-v6wxye6ijzrxnzxftcwnpu3psohsjl2b/bin/gfortran
       spec: gcc@=12.4.0
       target: aarch64
-
   packages:
     acfl:
       require:
-        - one_of: ["%gcc target=aarch64"]
+      - '%gcc target=aarch64'
-          message: "Clang based compilers need GCC libraries and they should be made available for the wide range of CPUs they actually support.
-                  Edit $SPACK_ROOT/etc/spack/packages.yaml to change this default."
     gromacs:
       require:
-        - one_of:
+      - gromacs@2024.3 %gcc ^armpl-gcc ^openmpi
-            - "gromacs@2024.3 %arm ^fftw ^openmpi"
-            - "gromacs@2024.3 %gcc ^armpl-gcc ^openmpi"
     libfabric:
       buildable: true
       externals:
       - prefix: /opt/amazon/efa/
         spec: libfabric@1.17.0
-      require: ['fabrics=shm,efa']
+      require:
+      - fabrics=shm,efa
     llvm:
       variants: ~lldb
     mpas-model:
       require:
-        - one_of:
+      - precision=single %gcc ^parallelio+pnetcdf
-            - "precision=single make_target=llvm %arm ^parallelio+pnetcdf"
-            - "precision=single %gcc ^parallelio+pnetcdf"
     mpich:
-      require: "mpich pmi=pmi2 device=ch4 netmod=ofi +slurm"
+      require:
+      - mpich pmi=pmi2 device=ch4 netmod=ofi +slurm
     nvhpc:
       require:
-        - one_of:
+      - nvhpc %gcc target=aarch64
-            - "nvhpc %gcc target=aarch64"
-          message: "NVHPC should be built with GCC and should be made available for the wide range of CPUs they actually support.
-                  Edit $SPACK_ROOT/etc/spack/packages.yaml to change this default."
     openfoam:
-      require: "openfoam %gcc ^scotch@6.0.9"
+      require:
+      - openfoam ^scotch@6.0.9
     openmpi:
       variants: ~atomics ~cuda ~cxx ~cxx_exceptions ~internal-hwloc ~java +legacylaunchers ~lustre ~memchecker +pmi +romio ~singularity +vt +wrapper-rpath fabrics=ofi schedulers=slurm
       require: '@4:'
@@ -108,19 +83,15 @@ spack:
     #   require:
     #     - one_of: ["palace cxxflags=\"-include cstdint\" ^fmt@9.1.0"]
     pmix:
-      require: "pmix@3:"
+      require: 'pmix@3:'
     quantum-espresso:
-      require: "quantum-espresso@6.6 %gcc ^armpl-gcc"
+      require:
+      - quantum-espresso@6.6 %gcc ^armpl-gcc
     slurm:
       buildable: false
       externals:
       - prefix: /opt/slurm
         spec: slurm@22.05.8 +pmix
-    wrf:
-      require:
-        - one_of:
-            - "wrf%arm"
-            - "wrf%gcc"
     all:
       compiler: [gcc, arm, nvhpc, clang]
       providers:

share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/spack.yaml

@@ -1,6 +1,5 @@
 spack:
   view: false
-
   definitions:
   - apps:
     - gromacs %oneapi
@@ -13,11 +12,9 @@ spack:
         # - quantum-espresso %oneapi
     - openmpi %oneapi
     - wrf %oneapi
-
   - targets:
-      - 'target=x86_64_v4'
+    - target=x86_64_v4
-      - 'target=x86_64_v3'
+    - target=x86_64_v3
-
   specs:
   - matrix:
     - [$apps]
@@ -25,7 +22,7 @@ spack:
   ci:
     pipeline-gen:
     - build-job:
-        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07", "entrypoint": [""] }
+        image: {name: ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07, entrypoint: ['']}
         before_script:
         - - . "./share/spack/setup-env.sh"
           - . /etc/profile.d/modules.sh
@@ -37,10 +34,8 @@ spack:
             # Do not distribute Intel & ARM binaries
         - - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
           - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
-
   cdash:
     build-group: AWS Packages
-
   compilers:
   - compiler:
       environment: {}
@@ -69,113 +64,51 @@ spack:
         fc: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-12.4.0/intel-oneapi-compilers-2024.1.0-f5u3psfhbwscasajkn324igtupn3blop/compiler/2024.1/bin/ifx
       spec: oneapi@=2024.1.0
       target: x86_64
-
   packages:
-    cpio:
-      require:
-        - one_of:
-            - "cflags=-std=c18 target=x86_64_v4"
-            - "cflags=-std=c18 target=x86_64_v3"
-          when: "%intel"
     gettext:
       # Newer gettext cannot build with gcc@12 and old AL2 glibc headers
       # Older gettext versions do not build correctly with oneapi.
       require:
-        - one_of:
+      - one_of: ['@:0.20 %gcc', '%oneapi']
-            - '@:0.20'
-            - '%oneapi'
     gromacs:
       require:
-        - one_of:
+      - +intel_provided_gcc ^intel-oneapi-mkl
-            - "+intel_provided_gcc ^intel-oneapi-mkl target=x86_64_v4"
-            - "+intel_provided_gcc ^intel-oneapi-mkl target=x86_64_v3"
-          when: "%intel"
-        - one_of:
-            - "+intel_provided_gcc target=x86_64_v4 ^intel-oneapi-mkl"
-            - "+intel_provided_gcc target=x86_64_v3 ^intel-oneapi-mkl"
-          when: "%oneapi"
-    intel-oneapi-compilers:
-      require: "intel-oneapi-compilers %gcc target=x86_64_v3"
     intel-oneapi-mpi:
       variants: +external-libfabric generic-names=True
     lammps:
       require:
-        - one_of:
+      - lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package fft=mkl ^intel-oneapi-mkl
-            - "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package +intel fft=mkl ^intel-oneapi-mkl target=x86_64_v4"
+      - one_of: [+intel target=x86_64_v4, target=x86_64_v3]
-            - "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package fft=mkl ^intel-oneapi-mkl target=x86_64_v3"
-          when: "%intel"
-        - one_of:
-            - "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package +intel fft=mkl ^intel-oneapi-mkl target=x86_64_v4"
-            - "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package fft=mkl ^intel-oneapi-mkl target=x86_64_v3"
-          when: "%oneapi"
-    libidn2:
-      require:
-        - one_of:
-            - "cflags=-std=c18 target=x86_64_v4"
-            - "cflags=-std=c18 target=x86_64_v3"
-          when: "%intel"
     libfabric:
       buildable: true
       externals:
       - prefix: /opt/amazon/efa/
         spec: libfabric@1.17.0
-      require: ['fabrics=shm,efa']
-    libunistring:
       require:
-        - one_of:
+      - fabrics=shm,efa
-            - "cflags=-std=c18 target=x86_64_v4"
-            - "cflags=-std=c18 target=x86_64_v3"
-          when: "%intel"
     mpas-model:
       require:
-        - one_of:
+      - spec: precision=single ^parallelio+pnetcdf
-            - "precision=single ^parallelio+pnetcdf target=x86_64_v4"
+        when: '%oneapi'
-            - "precision=single ^parallelio+pnetcdf target=x86_64_v3"
-          when: "%intel"
-        - one_of:
-            - "precision=single ^parallelio+pnetcdf target=x86_64_v4"
-            - "precision=single ^parallelio+pnetcdf target=x86_64_v3"
-          when: "%oneapi"
     mpich:
       require:
-        - one_of:
+      - mpich pmi=pmi2 device=ch4 netmod=ofi +slurm
-            - "mpich pmi=pmi2 device=ch4 netmod=ofi +slurm target=x86_64_v4"
-            - "mpich pmi=pmi2 device=ch4 netmod=ofi +slurm target=x86_64_v3"
     openfoam:
       require:
-        - one_of:
+      - openfoam %gcc ^scotch@6.0.9
-            - "openfoam %gcc ^scotch@6.0.9 target=x86_64_v4"
-            - "openfoam %gcc ^scotch@6.0.9 target=x86_64_v3"
     openmpi:
       variants: ~atomics ~cuda ~cxx ~cxx_exceptions ~internal-hwloc ~java +legacylaunchers ~lustre ~memchecker +pmi +romio ~singularity +vt +wrapper-rpath fabrics=ofi schedulers=slurm
       require:
-        - one_of:
+      - 'openmpi @4:'
-            - "openmpi @4: target=x86_64_v4"
-            - "openmpi @4: target=x86_64_v3"
     palace:
       require:
-        - one_of:
+      - palace ^fmt@9.1.0
-            - "palace ^fmt@9.1.0 target=x86_64_v4"
-            - "palace ^fmt@9.1.0 target=x86_64_v3"
-          when: "%oneapi"
-        - one_of:
-            - "palace ^fmt@9.1.0"
-          when: "%gcc"
     pmix:
       require:
-        - one_of:
+      - 'pmix@3:'
-            - "pmix@3: target=x86_64_v4"
-            - "pmix@3: target=x86_64_v3"
     quantum-espresso:
       require:
-        - one_of:
+      - quantum-espresso@6.6 ^intel-oneapi-mkl+cluster
-            - "quantum-espresso@6.6 ^intel-oneapi-mkl+cluster target=x86_64_v4"
-            - "quantum-espresso@6.6 ^intel-oneapi-mkl+cluster target=x86_64_v3"
-          when: "%intel"
-        - one_of:
-            - "quantum-espresso@6.6 ^intel-oneapi-mkl+cluster target=x86_64_v4"
-            - "quantum-espresso@6.6 ^intel-oneapi-mkl+cluster target=x86_64_v3"
-          when: "%oneapi"
     slurm:
       buildable: false
       externals:
@@ -183,17 +116,7 @@ spack:
         spec: slurm@22.05.8 +pmix
     wrf:
       require:
-        - one_of:
+      - wrf@4 build_type=dm+sm
-            - "wrf@4 build_type=dm+sm target=x86_64_v4"
-            - "wrf@4 build_type=dm+sm target=x86_64_v3"
-            - "wrf@4.2.2 +netcdf_classic fflags=\"-fp-model fast=2 -no-heap-arrays -no-prec-div -no-prec-sqrt -fno-common\" build_type=dm+sm target=x86_64_v3"
-          when: "%intel"
-        - one_of:
-            - "wrf@4 build_type=dm+sm target=x86_64_v4"
-            - "wrf@4 build_type=dm+sm target=x86_64_v3"
-            - "wrf@4.2.2 +netcdf_classic fflags=\"-fp-model fast=2 -no-heap-arrays -no-prec-div -no-prec-sqrt -fno-common\" build_type=dm+sm target=x86_64_v3"
-          when: "%oneapi"
-
     all:
       compiler: [oneapi, gcc]
       permissions: