Update pipeline configurations

Signed-off-by: Massimiliano Culpo <massimiliano.culpo@gmail.com>

share/spack/gitlab/cloud_pipelines/configs/config.yaml

@@ -4,5 +4,5 @@ config:
     root: /home/software/spack
     padded_length: 256
     projections:
-      all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
+      all: "{architecture.platform}/{architecture.target}/{name}-{version}-{hash}"
 

share/spack/gitlab/cloud_pipelines/scripts/pcluster/setup-pcluster.sh

@@ -74,7 +74,7 @@ install_compilers() {
     if [ -f "/bootstrap-compilers/spack/etc/spack/compilers.yaml" ]; then
         # Running inside a gitlab CI container
         # Intel and gcc@12 compiler are pre-installed and their location is known in
-        cp /bootstrap-compilers/spack/etc/spack/compilers.yaml "${SPACK_ROOT}"/etc/spack/
+        spack compiler list
     else
         spack compiler add --scope site
         # We need to treat compilers as essentially external, i.e. their installation location

share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml

@@ -57,34 +57,22 @@ spack:
 
   - compiler_specs:
     - gcc@11.2
-    # Licensing OK?
-    # - intel-oneapi-compilers@2022.1
-    # - nvhpc
 
   - app_specs:
     - bwa
 # Depends on simde which requires newer compiler?
     #- bowtie2
-    # Requires x86_64 specific ASM
-    #- cistem
-    - cromwell
-    - fastqc
     - flux-sched
     - flux-core
     - flux-pmix
-    - gatk
     - gromacs
     - lammps
     - wrf build_type=dm+sm
     - mfem
     - mpas-model ^parallelio+pnetcdf
-    - nextflow
     - octave
     - openfoam
     - osu-micro-benchmarks
-    - parallel
-    # - paraview
-    - picard
     - quantum-espresso
     - raja
 # Depends on bowtie2 -> simde which requires newer compiler?
@@ -94,12 +82,9 @@ spack:
     - salmon
     - samtools
     - seqtk
-    - snakemake
     - star
     # Requires gcc@9:
     #- ufs-weather-model
-# requires LLVM which fails without constraint
-    #- visit
 
   - lib_specs:
     - openmpi fabrics=ofi
@@ -115,7 +100,6 @@ spack:
     - target=aarch64
     - target=neoverse_n1
 
-
   specs:
 
   - matrix:
@@ -131,7 +115,10 @@ spack:
   - matrix:
     - - $compiler_specs
     - - $compiler
-    - - $target
+
+  - matrix:
+    - [cromwell, fastqc, gatk, nextflow, parallel, picard, snakemake]
+    - [$target]
 
   ci:
     pipeline-gen:

share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml

@@ -2,6 +2,8 @@ spack:
   view: false
   packages:
     all:
+      require:
+      - target=x86_64_v3
       providers:
         blas:
         - openblas
@@ -12,7 +14,8 @@ spack:
         - mpich
       variants: +mpi
     tbb:
-      require: intel-tbb
+      require:
+      - intel-tbb
     binutils:
       variants: +ld +gold +headers +libiberty ~nls
       version:
@@ -43,8 +46,6 @@ spack:
       - '1.18'
     relion:
       variants: ~mklfft
-#    texlive:
-#      version: [20210325]
     trilinos:
       variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
         +ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
@@ -57,56 +58,35 @@ spack:
 
   - compiler_specs:
     - gcc@11.2
-    # Licensing OK?
-    # - intel-oneapi-compilers@2022.1
-    # - nvhpc
 
   - cuda_specs:
-# Disabled for consistency with aarch64
-    #- relion +cuda cuda_arch=70
     - raja +cuda cuda_arch=70
     - mfem +cuda cuda_arch=70
 
   - app_specs:
     - bwa
-# Disabled for consistency with aarch64
-    #- bowtie2
-# Disabled for consistency with aarch64
-    #- cistem
-    - cromwell
-    - fastqc
     - flux-sched
     - flux-core
     - flux-pmix
-    - gatk
     - gromacs
     - lammps
     - wrf build_type=dm+sm
     - mfem
     - mpas-model ^parallelio+pnetcdf
-    - nextflow
     - octave
     - openfoam
     - osu-micro-benchmarks
-    - parallel
-    # - paraview
-    - picard
     - quantum-espresso
 # Build broken for gcc@7.3.1 x86_64_v4 (error: '_mm512_loadu_epi32' was not declared in this scope)
     #- raja
-# Disabled for consistency with aarch64
-    #- rsem
     # Errors on texlive
     #- rstudio
     - salmon
     - samtools
     - seqtk
-    - snakemake
     - star
     # Requires gcc@9:
     #- ufs-weather-model
-# Disabled for consistency with aarch64
-    #- visit
 
   - lib_specs:
     - openmpi fabrics=ofi
@@ -118,31 +98,32 @@ spack:
   - compiler:
     - '%gcc@7.3.1'
 
-  - target:
-    - target=x86_64_v3
-
-
   specs:
 
   - matrix:
     - - $cuda_specs
     - - $compiler
-    - - $target
 
   - matrix:
     - - $app_specs
     - - $compiler
-    - - $target
 
   - matrix:
     - - $lib_specs
     - - $compiler
-    - - $target
 
   - matrix:
     - - $compiler_specs
     - - $compiler
-    - - $target
+
+  - cromwell
+  - fastqc
+  - gatk
+  - parallel
+  - nextflow
+  - picard
+  - snakemake
+
 
   ci:
     pipeline-gen:

share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/packages.yaml

@@ -55,9 +55,11 @@ packages:
           - "wrf%arm"
           - "wrf%gcc"
   all:
-    compiler: [gcc, arm, nvhpc, clang]
     providers:
       blas: [armpl-gcc, openblas]
+      c: [gcc, arm, nvhpc, llvm]
+      cxx: [gcc, arm, nvhpc, llvm]
+      fortran: [gcc, arm, nvhpc, llvm]
       fftw-api: [armpl-gcc, fftw]
       lapack: [armpl-gcc, openblas]
       mpi: [openmpi, mpich]

share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/packages.yaml

@@ -23,8 +23,6 @@ packages:
           - "+intel_provided_gcc target=x86_64_v4 ^intel-oneapi-mkl"
           - "+intel_provided_gcc target=x86_64_v3 ^intel-oneapi-mkl"
         when: "%oneapi"
-  intel-oneapi-compilers:
-    require: "intel-oneapi-compilers %gcc target=x86_64_v3"
   intel-oneapi-mpi:
     variants: +external-libfabric generic-names=True
   lammps:
@@ -124,14 +122,16 @@ packages:
         when: "%oneapi"
 
   all:
-    compiler: [oneapi, gcc]
     permissions:
       read: world
       write: user
     providers:
       blas: [intel-oneapi-mkl]
+      c: [intel-oneapi-compilers, gcc]
+      cxx: [intel-oneapi-compilers, gcc]
       daal: [intel-oneapi-dal]
       fftw-api: [intel-oneapi-mkl]
+      fortran: [intel-oneapi-compilers, gcc]
       ipp: [intel-oneapi-ipp]
       lapack: [intel-oneapi-mkl]
       mkl: [intel-oneapi-mkl]
@@ -139,5 +139,22 @@ packages:
       tbb: [intel-oneapi-tbb, intel-tbb]
       scalapack: [intel-oneapi-mkl]
 
-
+  # compilers
-
+  gcc:
+    externals:
+    - spec: gcc@12.4.0 languages='c,c++,fortran'
+      prefix: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-7.3.1/gcc-12.4.0-pttzchh7o54nhmycj4wgzw5mic6rk2nb
+      extra_attributes:
+        compilers:
+          c: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-7.3.1/gcc-12.4.0-pttzchh7o54nhmycj4wgzw5mic6rk2nb/bin/gcc
+          cxx: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-7.3.1/gcc-12.4.0-pttzchh7o54nhmycj4wgzw5mic6rk2nb/bin/g++
+          fortran: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-7.3.1/gcc-12.4.0-pttzchh7o54nhmycj4wgzw5mic6rk2nb/bin/gfortran
+  intel-oneapi-compilers:
+    externals:
+      - spec: intel-oneapi-compilers@2024.1.0 target=x86_64_v3
+        prefix: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-12.4.0/intel-oneapi-compilers-2024.1.0-f5u3psfhbwscasajkn324igtupn3blop/compiler/2024.1
+        extra_attributes:
+          compilers:
+            c: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-12.4.0/intel-oneapi-compilers-2024.1.0-f5u3psfhbwscasajkn324igtupn3blop/compiler/2024.1/bin/icx
+            cxx: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-12.4.0/intel-oneapi-compilers-2024.1.0-f5u3psfhbwscasajkn324igtupn3blop/compiler/2024.1/bin/icpx
+            fortran: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-12.4.0/intel-oneapi-compilers-2024.1.0-f5u3psfhbwscasajkn324igtupn3blop/compiler/2024.1/bin/ifx

share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml

@@ -2,9 +2,13 @@ spack:
   view: false
   packages:
     all:
-      require: target=x86_64_v3
+      require:
-  definitions:
+      - target=x86_64_v3
-    - default_specs:
+    c:
+      require:
+      - gcc
+
+  specs:
   - 'uncrustify build_system=autotools'
   - 'uncrustify build_system=cmake'
   - lz4  # MakefilePackage
@@ -13,13 +17,6 @@ spack:
   - openjpeg  # CMakePackage
   - r-rcpp  # RPackage
   - ruby-rake  # RubyPackage
-    - arch:
-      - '%gcc'
-
-  specs:
-    - matrix:
-        - - $default_specs
-        - - $arch
 
   cdash:
     build-group: Build Systems

share/spack/gitlab/cloud_pipelines/stacks/developer-tools-manylinux2014/spack.yaml

@@ -1,17 +1,32 @@
 spack:
   view: false
+
   packages:
     all:
-      require: target=x86_64_v3
+      require:
+      - target=x86_64_v3
+      prefer:
+      - '%gcc'
+    gcc:
+      externals:
+        - spec: gcc@=10.2.1 languages=c,c++,fortran
+          prefix: /opt/rh/devtoolset-10/root/usr
+          extra_attributes:
+            compilers:
+              c: /opt/rh/devtoolset-10/root/usr/bin/gcc
+              cxx: /opt/rh/devtoolset-10/root/usr/bin/g++
+              fortran: /opt/rh/devtoolset-10/root/usr/bin/gfortran
+
   concretizer:
     unify: true
     reuse: false
-  definitions:
+
-  - default_specs:
+  specs:
     # editors
   - neovim~no_luajit
   - py-pynvim
-    - emacs+json+native+treesitter   # note, pulls in gcc
+  # FIXME (compiler as nodes): recover dependency on gcc as a library when +native
+  - emacs+json~native+treesitter   # note, pulls in gcc
     # - tree-sitter is a dep, should also have cli but no package
   - nano   # just in case
     # tags and scope search helpers
@@ -68,29 +83,6 @@ spack:
   - doxygen
   - meson
 
-  - arch:
-    - '%gcc target=x86_64_v3'
-
-  specs:
-  - matrix:
-    - - $default_specs
-    - - $arch
-
-  compilers:
-  - compiler:
-      spec: gcc@=10.2.1
-      paths:
-        cc: /opt/rh/devtoolset-10/root/usr/bin/gcc
-        cxx: /opt/rh/devtoolset-10/root/usr/bin/g++
-        f77: /opt/rh/devtoolset-10/root/usr/bin/gfortran
-        fc: /opt/rh/devtoolset-10/root/usr/bin/gfortran
-      flags: {}
-      operating_system: centos7
-      target: x86_64
-      modules: []
-      environment: {}
-      extra_rpaths: []
-
   ci:
     pipeline-gen:
     - build-job:

share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml

@@ -12,9 +12,11 @@ spack:
     all:
       prefer:
       - "%cce"
-      compiler: [cce]
       providers:
         blas: [cray-libsci]
+        c: [cce]
+        cxx: [cce]
+        fortran: [cce]
         lapack: [cray-libsci]
         mpi: [cray-mpich]
         tbb: [intel-tbb]

share/spack/gitlab/cloud_pipelines/stacks/e4s-power/spack.yaml

@@ -7,8 +7,9 @@ spack:
 
   packages:
     all:
-      require: "%gcc@9.4.0 target=ppc64le"
+      prefer:
-      compiler: [gcc@9.4.0]
+      - "%gcc@9.4.0"
+      - target=ppc64le
       providers:
         blas: [openblas]
         mpi: [mpich]

share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml

@@ -3,7 +3,6 @@ spack:
 
   packages:
     all:
-      compiler: [gcc@11.1.0]
       providers:
         blas: [openblas]
         mpi: [mpich]

share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml

@@ -2,17 +2,14 @@ spack:
   view: false
   packages:
     all:
-      require: target=x86_64_v3
+      require:
+      - target=x86_64_v3
+      - '%gcc@7.5.0'
 
       providers:
         mpi: [mvapich2]
 
-  definitions:
+  specs:
-    #- compilers: ['%gcc@8.3.1', '%clang@10.0.0']
-    - compilers: ['%gcc@7.5.0']
-
-    # Note skipping spot since no spack package for it
-    - radiuss:
   - ascent  # ^conduit@0.6.0
   - axom
   - blt
@@ -21,7 +18,7 @@ spack:
   - chai  # ~examples
   - conduit  # ^hdf5+shared
   - flux-core
-      #- flux-sched
+  # - flux-sched
   - hypre
   - lbann
   - lvarray ~tests  # per Spack issue #23192  # ~examples
@@ -36,14 +33,9 @@ spack:
   - scr
   - sundials
   - umpire # ~openmp
-      #- visit   # ^mesa-glu@9.0.0
+  # - visit   # ^mesa-glu@9.0.0
   - xbraid
   - zfp
 
-  specs:
-  - matrix:
-    - [$radiuss]
-    - [$compilers]
-
   cdash:
     build-group: RADIUSS