feat: add HEP cloud_pipeline stack (#40932)

* dd4hep: depends_on root +root7 in some cases

* dd4hep: avoid self-referential whens -> conflicts

* feat: add HEP cloud_pipeline stack

* [hep] add hep jobs to .gitlab-ci.yml

* [hep] rm hep-generate image override

* [hep] fix root variants; add some more packages

* [hep] disable hep-build for now

* [hep] hepmc3 +interfaces

* [hep] hep-generate on gcc-11.4 runner image

* [hep] sherpa -blackhat

* [hep] align package requirements to e4s for more cache hits

* [hep] lcio -examples due to conflicts

* [hep] marks geant4-data as external

* [hep] enable build

* [hep] elfutils now always has bzip2 xz dependency

* [hep] avoid gaudi ^gdb@8.1 with explicit version gdb@14.1

* [hep] geant4 -vtk for now

* [hep] root ~oracle

* [hep] external geant4-data@11.2.0

* [hep] sherap ~python

* [hep] gaudi ^gdb +python to avoid failing gdb@8.1

* [hep] pythia8 ~madgraph5amc

* [hep] remove garfieldpp which fails to download

* [hep] sherpa ~recola because recola needs pic variant

* [hep] sherpa ~rivet since conflicts("cxxstd=11", when="~rivet")

* [hep] unify: when_possible

* [hep] thepeg ~rivet

* [hep] root ~shadow

* [hep] lcio ~jar

* [hep] root ~postgres

* [hep] herwig3 ~evtgen

* [hep] herwig3 has no evtgen variant

* [hep] no image override in hep-generate

* [hep] re-add ubuntu-22.04 image for newer gcc than 7

* [hep] remove herwig3

* [hep] remove package configs

* [hep] avoid non-existent variants in latest versions

* [hep] allow for sherpa@3:

* [hep] acts cxxstd=20

* [hep] gaudi ^gdb +python

* [hep] gaudi ^gdb +debuginfod

* [hep] add some packages to large and huge kubernetes groups

* [hep] root +rpath

* [hep] add qt-declarative to large 8core 8GB CI group

* [hep] add sherpa to 8 core, 3 GB CI medium group

* [hep] add root to 12 core, 19 GB huge CI group

* [hep] gaudi: touch pytest.ini

* [hep] ~ instead of -

Co-authored-by: Seth R. Johnson <johnsonsr@ornl.gov>

---------

Co-authored-by: Seth R. Johnson <johnsonsr@ornl.gov>

share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml

@@ -785,6 +785,29 @@ ml-darwin-aarch64-mps-build:
     - artifacts: True
       job: ml-darwin-aarch64-mps-generate
 
+########################################
+# High Energy Physics (HEP) - Linux x86_64 (CPU)
+########################################
+.hep:
+  extends: [ ".linux_x86_64_v3" ]
+  variables:
+    SPACK_CI_STACK_NAME: hep
+
+hep-generate:
+  extends: [ ".hep", ".generate-x86_64"]
+  image: ghcr.io/spack/spack/ubuntu22.04-runner-amd64-gcc-11.4:2024.03.01
+
+hep-build:
+  extends: [ ".hep", ".build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: hep-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: hep-generate
+
 ########################################
 # AWS ParallelCluster
 ########################################

share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml

@@ -72,7 +72,9 @@ ci:
 
     - match:
       - dealii
+      - geant4
       - rocblas
+      - root
       build-job:
         tags: [ "spack", "huge" ]
         variables:
@@ -82,6 +84,7 @@ ci:
           KUBERNETES_MEMORY_REQUEST: 19G
 
     - match:
+      - acts
       - ecp-data-vis-sdk
       - intel-tbb
       - llvm-amdgpu
@@ -102,6 +105,7 @@ ci:
       - cuda
       - dray
       - ecp-data-vis-sdk
+      - gaudi
       - gcc
       - ginkgo
       - hdf5
@@ -144,7 +148,11 @@ ci:
           KUBERNETES_MEMORY_REQUEST: "9G"
 
     - match:
+      - dd4hep
       - hipblas
+      - qt-base
+      - qt-declarative
+      - rivet
       - rocfft
       - umpire
       build-job:
@@ -193,6 +201,7 @@ ci:
 
     - match:
       - parallelio
+      - sherpa
       build-job:
         tags: [ "spack", "medium" ]
         variables:

share/spack/gitlab/cloud_pipelines/stacks/hep/spack.yaml

@@ -0,0 +1,64 @@
+spack:
+  view: false
+
+  concretizer:
+    reuse: false
+    unify: when_possible
+
+  packages:
+    all:
+      require: '%gcc target=x86_64_v3'
+      providers:
+        blas: [openblas]
+        mpi: [mpich]
+        tbb: [intel-tbb]
+      variants: +mpi
+
+    # Mark geant4 data as external to prevent wasting bandwidth on GB-scale files
+    geant4-data:
+      externals:
+        - spec: geant4-data@11.3.0
+          prefix: /usr
+        - spec: geant4-data@11.2.2
+          prefix: /usr
+        - spec: geant4-data@11.2.0
+          prefix: /usr
+        - spec: geant4-data@11.1.0
+          prefix: /usr
+        - spec: geant4-data@11.0.0
+          prefix: /usr
+
+  specs:
+  # CPU
+  - acts +analysis +dd4hep +edm4hep +examples +fatras +geant4 +hepmc3 +podio +pythia8 +python +tgeo cxxstd=20
+  - dd4hep +ddalign +ddcad +ddcond +dddetectors +dddigi +ddeve +ddg4 +ddrec +edm4hep +hepmc3 +lcio +utilityapps +xercesc
+  - delphes +pythia8
+  - edm4hep
+  - fastjet
+  - fjcontrib
+  - gaudi +aida +examples +heppdt +xercesc ^gdb +debuginfod +python
+  - geant4 +opengl +qt +threads ~vtk ^[virtuals=qmake] qt
+  - hepmc
+  - hepmc3 +interfaces +protobuf +python +rootio
+  #- herwig3 +njet +vbfnlo  # Note: herwig3 fails to find evtgen
+  - lcio -examples ~jar +rootdict  # Note: lcio +examples ^ncurses -termlib, which leads to conflicts
+  - lhapdf +python
+  - madgraph5amc
+  - opendatadetector
+  - podio +rntuple +sio
+  - py-uproot +lz4 +xrootd +zstd
+  - pythia8 +evtgen +fastjet +hdf5 +hepmc +hepmc3 +lhapdf ~madgraph5amc +python +rivet ~root # pythia8 and root circularly depend
+  - rivet hepmc=3
+  - root +davix +dcache +examples +fftw +fits +fortran +gdml +graphviz +gsl +http +math +minuit +mlp +mysql +opengl ~postgres +pythia8 +python +r +roofit +root7 +rpath ~shadow +spectrum +sqlite +ssl +tbb +threads +tmva +unuran +vc +vdt +veccore +webgui +x +xml +xrootd
+  - sherpa +analysis ~blackhat +gzip +hepmc3 +hepmc3root +lhapdf +lhole +openloops +pythia ~python ~recola ~rivet +root +ufo
+  - thepeg ~rivet
+  - vecgeom +gdml +geant4 +root
+  - xrootd +davix +http +krb5 +python +readline +scitokens-cpp
+
+  ci:
+    pipeline-gen:
+    - build-job:
+        image: "ghcr.io/spack/spack/ubuntu22.04-runner-amd64-gcc-11.4:2024.03.01"
+
+  cdash:
+    build-group: HEP

var/spack/repos/builtin/packages/gaudi/package.py

@@ -138,6 +138,11 @@ class Gaudi(CMakePackage):
     # The Intel VTune dependency is taken aside because it requires a license
     depends_on("intel-parallel-studio -mpi +vtune", when="+vtune")
 
+    def patch(self):
+        # ensure an empty pytest.ini is present to prevent finding one
+        # accidentally in a higher directory than the stage directory
+        touch("pytest.ini")
+
     def cmake_args(self):
         args = [
             # Note: gaudi only builds examples when testing enabled