hep stack: build also with cuda and rocm where possible (#48528)

share/spack/gitlab/cloud_pipelines/stacks/hep/spack.yaml

@@ -13,29 +13,41 @@ spack:
         mpi: [mpich]
         tbb: [intel-tbb]
       variants: +mpi
+    acts:
+      require: +analysis +dd4hep +edm4hep +examples +fatras +geant4 +hepmc3 +podio +pythia8 +python ~svg +tgeo cxxstd=20
+    celeritas:
+      require: +geant4 +hepmc3 +root +shared cxxstd=20
+    hip:
+      require: '@5.7.1 +rocm'
+    root:
+      require: +davix +dcache +examples +fftw +fits +fortran +gdml +graphviz +gsl +http +math +minuit +mlp +mysql +opengl +postgres +pythia8 +python +r +roofit +root7 +rpath ~shadow +spectrum +sqlite +ssl +tbb +threads +tmva +tmva-cpu +unuran +vc +vdt +veccore +webgui +x +xml +xrootd # cxxstd=20
+      # note: root cxxstd=20 not concretizable within sherpa
+    vecgeom:
+      require: +gdml +geant4 +root +shared cxxstd=20
 
     # Mark geant4 data as external to prevent wasting bandwidth on GB-scale files
     geant4-data:
+      buildable: false
       externals:
-        - spec: geant4-data@11.3.0
+      - spec: geant4-data@11.3.0
-          prefix: /usr
+        prefix: /usr
-        - spec: geant4-data@11.2.2
+      - spec: geant4-data@11.2.2
-          prefix: /usr
+        prefix: /usr
-        - spec: geant4-data@11.2.0
+      - spec: geant4-data@11.2.0
-          prefix: /usr
+        prefix: /usr
-        - spec: geant4-data@11.1.0
+      - spec: geant4-data@11.1.0
-          prefix: /usr
+        prefix: /usr
-        - spec: geant4-data@11.0.0
+      - spec: geant4-data@11.0.0
-          prefix: /usr
+        prefix: /usr
 
   specs:
   # CPU
-  - acts +analysis +dd4hep +edm4hep +examples +fatras +geant4 +hepmc3 +podio +pythia8 +python +tgeo cxxstd=20
+  - acts ~cuda
   #- agile  # fails on c++>11 compiler
   - alpgen
   - ampt
   - apfel +lhapdf +python
-  - celeritas +geant4 +hepmc3 +openmp +root +shared +vecgeom cxxstd=20
+  - celeritas ~cuda +openmp ~rocm +vecgeom
   - cepgen
   - cernlib +shared
   - collier
@@ -82,19 +94,28 @@ spack:
   - py-vector
   - pythia8 +evtgen +fastjet +hdf5 +hepmc +hepmc3 +lhapdf ~madgraph5amc +python +rivet ~root # pythia8 and root circularly depend
   - rivet hepmc=3
-  - root +davix +dcache +examples +fftw +fits +fortran +gdml +graphviz +gsl +http +math +minuit +mlp +mysql +opengl +postgres +pythia8 +python +r +roofit +root7 +rpath ~shadow +spectrum +sqlite +ssl +tbb +threads +tmva +unuran +vc +vdt +veccore +webgui +x +xml +xrootd
+  - root ~cuda
   - sherpa +analysis ~blackhat +gzip +hepmc3 +hepmc3root +lhapdf +lhole +openloops +pythia ~python ~recola ~rivet +root +ufo cxxstd=20
   - tauola +hepmc3 +lhapdf cxxstd=20
   - thepeg hepmc=3 ~rivet
-  - vecgeom +gdml +geant4 +root
+  - vecgeom ~cuda
   - whizard +fastjet +gosam hepmc=3 +lcio +lhapdf +openloops +openmp +pythia8
   - xrootd +davix +http +krb5 +python +readline +scitokens-cpp
   - yoda +root
 
+  # CUDA
+  #- acts +cuda +traccc cuda_arch=80
+  #- celeritas +cuda ~openmp +vecgeom cuda_arch=80
+  - root +cuda +cudnn +tmva-gpu
+  - vecgeom +cuda cuda_arch=80
+
+  # ROCm
+  - celeritas +rocm amdgpu_target=gfx90a ~openmp ~vecgeom  # only available with ORANGE
+
   ci:
     pipeline-gen:
     - build-job:
-        image: "ghcr.io/spack/spack/ubuntu22.04-runner-amd64-gcc-11.4:2024.03.01"
+        image: ghcr.io/spack/spack/ubuntu22.04-runner-amd64-gcc-11.4:2024.03.01
 
   cdash:
     build-group: HEP

var/spack/repos/builtin/packages/celeritas/package.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+from spack.build_systems.cmake import CMakeBuilder
 from spack.package import *
 
 
@@ -81,13 +82,22 @@ class Celeritas(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("py-breathe", type="build", when="+doc")
     depends_on("py-sphinx", type="build", when="+doc")
 
+    with when("+cuda"):
+        depends_on("thrust")
+    with when("+rocm"):
+        depends_on("hiprand")
+        depends_on("rocprim")
+        depends_on("rocrand")
+        depends_on("rocthrust")
+
     for _std in _cxxstd_values:
         depends_on("geant4 cxxstd=" + _std, when="+geant4 cxxstd=" + _std)
         depends_on("root cxxstd=" + _std, when="+root cxxstd=" + _std)
         depends_on("vecgeom cxxstd=" + _std, when="+vecgeom cxxstd=" + _std)
 
+    depends_on("vecgeom +cuda cuda_arch=none", when="+vecgeom +cuda cuda_arch=none")
     for _arch in CudaPackage.cuda_arch_values:
-        depends_on("vecgeom+cuda cuda_arch=" + _arch, when="+vecgeom +cuda cuda_arch=" + _arch)
+        depends_on(f"vecgeom +cuda cuda_arch={_arch}", when=f"+vecgeom +cuda cuda_arch={_arch}")
 
     conflicts("+rocm", when="+cuda", msg="AMD and NVIDIA accelerators are incompatible")
     conflicts("+rocm", when="+vecgeom", msg="HIP support is only available with ORANGE")
@@ -111,7 +121,7 @@ def cmake_args(self):
             from_variant("CELERITAS_BUILD_DOCS", "doc"),
             define("CELERITAS_BUILD_DEMOS", False),
             define("CELERITAS_BUILD_TESTS", False),
-            from_variant("Celeritas_USE_HIP", "rocm"),
+            from_variant("CELERITAS_USE_HIP", "rocm"),
             define("CELERITAS_USE_MPI", False),
             define("CELERITAS_USE_Python", True),
         ]
@@ -119,6 +129,22 @@ def cmake_args(self):
         for pkg in ["CUDA", "Geant4", "HepMC3", "OpenMP", "ROOT", "SWIG", "VecGeom"]:
             args.append(from_variant("CELERITAS_USE_" + pkg, pkg.lower()))
 
+        if self.spec.satisfies("+cuda"):
+            args.append(CMakeBuilder.define_cuda_architectures(self))
+        if self.spec.satisfies("+rocm"):
+            args.append(CMakeBuilder.define_hip_architectures(self))
+            args.append(
+                define(
+                    "CMAKE_HIP_FLAGS",
+                    " ".join(
+                        [
+                            f"-I{self.spec[p].prefix.include}"
+                            for p in ["hiprand", "rocprim", "rocrand", "rocthrust"]
+                        ]
+                    ),
+                )
+            )
+
         if self.version < Version("0.5"):
             # JSON is required for 0.5 and later
             args.append(define("CELERITAS_USE_JSON", True))