diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index 1dc5e213b20..8f8628030af 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -718,7 +718,7 @@ tutorial-build:
 
 ml-linux-x86_64-cpu-generate:
   extends: [ ".generate-x86_64", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ]
-  image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:v2024-01-29
+  image: ghcr.io/spack/ubuntu-22.04:v2024-05-07
 
 ml-linux-x86_64-cpu-build:
   extends: [ ".build", ".ml-linux-x86_64-cpu" ]
@@ -741,7 +741,7 @@ ml-linux-x86_64-cpu-build:
 
 ml-linux-x86_64-cuda-generate:
   extends: [ ".generate-x86_64", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ]
-  image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:v2024-01-29
+  image: ghcr.io/spack/ubuntu-22.04:v2024-05-07
 
 ml-linux-x86_64-cuda-build:
   extends: [ ".build", ".ml-linux-x86_64-cuda"  ]
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
index e4bd85591fc..5656455adf5 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
@@ -79,7 +79,7 @@ spack:
     pipeline-gen:
     - build-job:
         image:
-          name: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:v2024-01-29
+          name: ghcr.io/spack/ubuntu-22.04:v2024-05-07
           entrypoint: ['']
 
   cdash:
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
index 68bfa16d0cb..ef7234fd55e 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
@@ -83,7 +83,7 @@ spack:
     pipeline-gen:
     - build-job:
         image:
-          name: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:v2024-01-29
+          name: ghcr.io/spack/ubuntu-22.04:v2024-05-07
           entrypoint: ['']
 
   cdash:
diff --git a/var/spack/repos/builtin/packages/amr-wind/package.py b/var/spack/repos/builtin/packages/amr-wind/package.py
index 19fbf7ffd97..807d917cec8 100644
--- a/var/spack/repos/builtin/packages/amr-wind/package.py
+++ b/var/spack/repos/builtin/packages/amr-wind/package.py
@@ -21,6 +21,12 @@ class AmrWind(CMakePackage, CudaPackage, ROCmPackage):
     license("BSD-3-Clause")
 
     version("main", branch="main", submodules=True)
+    version(
+        "2.1.0", tag="v2.1.0", commit="bc787f21deca9239928182e27400133934c62658", submodules=True
+    )
+    version(
+        "2.0.0", tag="v2.0.0", commit="ea448365033fc6bc9ee0febeb369b377f4fd8240", submodules=True
+    )
     version(
         "1.4.0", tag="v1.4.0", commit="bdddf133e41a9b7b4c8ce28f1ea1bebec47678f5", submodules=True
     )
@@ -91,7 +97,7 @@ class AmrWind(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("openfast@3.5:", when="@2: +openfast")
     depends_on("helics@:3.3.2", when="+helics")
     depends_on("helics@:3.3.2+mpi", when="+helics+mpi")
-    depends_on("fftw", when="@2: +waves2amr")
+    depends_on("fftw", when="@2.1: +waves2amr")
 
     for arch in CudaPackage.cuda_arch_values:
         depends_on("hypre+cuda cuda_arch=%s" % arch, when="+cuda+hypre cuda_arch=%s" % arch)
@@ -104,7 +110,7 @@ class AmrWind(CMakePackage, CudaPackage, ROCmPackage):
 
     conflicts("+openmp", when="+cuda")
     conflicts("+shared", when="+cuda")
-    conflicts("@:1.4.0", when="+waves2amr")
+    conflicts("@:2.0", when="+waves2amr")
 
     def setup_build_environment(self, env):
         # Avoid compile errors with Intel interprocedural optimization
diff --git a/var/spack/repos/builtin/packages/bash/package.py b/var/spack/repos/builtin/packages/bash/package.py
index e7b4c1c3b1b..ae322c17857 100644
--- a/var/spack/repos/builtin/packages/bash/package.py
+++ b/var/spack/repos/builtin/packages/bash/package.py
@@ -44,6 +44,17 @@ class Bash(AutotoolsPackage, GNUMirrorPackage):
         ("5.2", "013", "094b4fd81bc488a26febba5d799689b64d52a5505b63e8ee854f48d356bc7ce6"),
         ("5.2", "014", "3ef9246f2906ef1e487a0a3f4c647ae1c289cbd8459caa7db5ce118ef136e624"),
         ("5.2", "015", "ef73905169db67399a728e238a9413e0d689462cb9b72ab17a05dba51221358a"),
+        ("5.2", "016", "155853bc5bd10e40a9bea369fb6f50a203a7d0358e9e32321be0d9fa21585915"),
+        ("5.2", "017", "1c48cecbc9b7b4217990580203b7e1de19c4979d0bd2c0e310167df748df2c89"),
+        ("5.2", "018", "4641dd49dd923b454dd0a346277907090410f5d60a29a2de3b82c98e49aaaa80"),
+        ("5.2", "019", "325c26860ad4bba8558356c4ab914ac57e7b415dac6f5aae86b9b05ccb7ed282"),
+        ("5.2", "020", "b6fc252aeb95ce67c9b017d29d81e8a5e285db4bf20d4ec8cdca35892be5c01d"),
+        ("5.2", "021", "8334b88117ad047598f23581aeb0c66c0248cdd77abc3b4e259133aa307650cd"),
+        ("5.2", "022", "78b5230a49594ec30811e72dcd0f56d1089710ec7828621022d08507aa57e470"),
+        ("5.2", "023", "af905502e2106c8510ba2085aa2b56e64830fc0fdf6ee67ebb459ac11696dcd3"),
+        ("5.2", "024", "971534490117eb05d97d7fd81f5f9d8daf927b4d581231844ffae485651b02c3"),
+        ("5.2", "025", "5138f487e7cf71a6323dc81d22419906f1535b89835cc2ff68847e1a35613075"),
+        ("5.2", "026", "96ee1f549aa0b530521e36bdc0ba7661602cfaee409f7023cac744dd42852eac"),
         ("5.1", "001", "ebb07b3dbadd98598f078125d0ae0d699295978a5cdaef6282fe19adef45b5fa"),
         ("5.1", "002", "15ea6121a801e48e658ceee712ea9b88d4ded022046a6147550790caf04f5dbe"),
         ("5.1", "003", "22f2cc262f056b22966281babf4b0a2f84cb7dd2223422e5dcd013c3dcbab6b1"),
diff --git a/var/spack/repos/builtin/packages/exawind/package.py b/var/spack/repos/builtin/packages/exawind/package.py
index 8ea3720414b..2db941c6648 100644
--- a/var/spack/repos/builtin/packages/exawind/package.py
+++ b/var/spack/repos/builtin/packages/exawind/package.py
@@ -16,59 +16,84 @@ class Exawind(CMakePackage, CudaPackage, ROCmPackage):
 
     tags = ["ecp", "ecp-apps"]
 
-    # Testing is currently always enabled, but should be optional in the future
-    # to avoid cloning the mesh submodule
-    version("master", branch="main", submodules=True)
-    version("1.0.0", tag="v1.0.0", submodules=True)
     license("Apache-2.0")
 
-    variant("openfast", default=False, description="Enable OpenFAST integration")
-    variant("hypre", default=True, description="Enable hypre solver")
-    variant("stk_simd", default=False, description="Enable SIMD in STK")
-    variant("umpire", default=False, description="Enable Umpire")
-    variant("tiny_profile", default=False, description="Turn on AMR-wind with tiny profile")
+    version("master", branch="main", submodules=True, preferred=True)
+    version("1.0.0", tag="v1.0.0", submodules=True)
+
+    variant("amr_wind_gpu", default=False, description="Enable AMR-Wind on the GPU")
+    variant("nalu_wind_gpu", default=False, description="Enable Nalu-Wind on the GPU")
     variant("sycl", default=False, description="Enable SYCL backend for AMR-Wind")
     variant("gpu-aware-mpi", default=False, description="gpu-aware-mpi")
 
-    conflicts("amr-wind+hypre", when="+sycl")
-
     for arch in CudaPackage.cuda_arch_values:
-        depends_on("amr-wind+cuda cuda_arch=%s" % arch, when="+cuda cuda_arch=%s" % arch)
-        depends_on("nalu-wind+cuda cuda_arch=%s" % arch, when="+cuda cuda_arch=%s" % arch)
+        depends_on(
+            "amr-wind+cuda cuda_arch=%s" % arch, when="+amr_wind_gpu+cuda cuda_arch=%s" % arch
+        )
+        depends_on(
+            "nalu-wind+cuda cuda_arch=%s" % arch, when="+nalu_wind_gpu+cuda cuda_arch=%s" % arch
+        )
+        depends_on(
+            "trilinos+cuda cuda_arch=%s" % arch, when="+nalu_wind_gpu+cuda cuda_arch=%s" % arch
+        )
 
     for arch in ROCmPackage.amdgpu_targets:
-        depends_on("amr-wind+rocm amdgpu_target=%s" % arch, when="+rocm amdgpu_target=%s" % arch)
-        depends_on("nalu-wind+rocm amdgpu_target=%s" % arch, when="+rocm amdgpu_target=%s" % arch)
+        depends_on(
+            "amr-wind+rocm amdgpu_target=%s" % arch,
+            when="+amr_wind_gpu+rocm amdgpu_target=%s" % arch,
+        )
+        depends_on(
+            "nalu-wind+rocm amdgpu_target=%s" % arch,
+            when="+nalu_wind_gpu+rocm amdgpu_target=%s" % arch,
+        )
+        depends_on(
+            "trilinos+rocm amdgpu_target=%s" % arch,
+            when="+nalu_wind_gpu+rocm amdgpu_target=%s" % arch,
+        )
 
-    depends_on("nalu-wind+tioga")
-    depends_on("amr-wind+netcdf+mpi")
-    depends_on("tioga~nodegid")
+    depends_on("nalu-wind+hypre+fsi+openfast+tioga")
+    depends_on("amr-wind+netcdf+mpi+tiny_profile")
+    depends_on("trilinos")
     depends_on("yaml-cpp@0.6:")
-    depends_on("nalu-wind+openfast", when="+openfast")
-    depends_on("amr-wind+hypre", when="+hypre~sycl")
-    depends_on("amr-wind~hypre", when="~hypre")
-    depends_on("nalu-wind+hypre", when="+hypre")
-    depends_on("nalu-wind~hypre", when="~hypre")
-    depends_on("amr-wind+sycl", when="+sycl")
-    depends_on("nalu-wind+umpire", when="+umpire")
-    depends_on("amr-wind+umpire", when="+umpire")
-    depends_on("amr-wind+tiny_profile", when="+tiny_profile")
+    depends_on("tioga~nodegid")
+    depends_on("openfast+cxx@2.6.0:")
+    depends_on("amr-wind+sycl", when="+amr_wind_gpu+sycl")
+    depends_on("kokkos-nvcc-wrapper", type="build", when="+cuda")
+    depends_on("mpi")
     depends_on("nalu-wind+gpu-aware-mpi", when="+gpu-aware-mpi")
     depends_on("amr-wind+gpu-aware-mpi", when="+gpu-aware-mpi")
     depends_on("nalu-wind@2.0.0:", when="@1.0.0:")
     depends_on("amr-wind@0.9.0:", when="@1.0.0:")
     depends_on("tioga@1.0.0:", when="@1.0.0:")
 
+    with when("~amr_wind_gpu~nalu_wind_gpu"):
+        conflicts("+cuda")
+        conflicts("+rocm")
+        conflicts("+sycl")
+    with when("~nalu_wind_gpu"):
+        conflicts("^nalu-wind+cuda")
+        conflicts("^nalu-wind+rocm")
+    with when("~amr_wind_gpu"):
+        conflicts("^amr-wind+cuda")
+        conflicts("^amr-wind+rocm")
+        conflicts("^amr-wind+sycl")
+    conflicts("+amr_wind_gpu", when="~cuda~rocm~sycl")
+    conflicts("+nalu_wind_gpu", when="~cuda~rocm")
+    conflicts("+nalu_wind_gpu", when="+sycl")
+    conflicts("^amr-wind+hypre", when="~amr_wind_gpu+nalu_wind_gpu")
+    conflicts("^amr-wind+hypre", when="+amr_wind_gpu~nalu_wind_gpu")
+    conflicts("+sycl", when="+cuda")
+    conflicts("+rocm", when="+cuda")
+    conflicts("+sycl", when="+rocm")
+
     def cmake_args(self):
         spec = self.spec
 
         args = [self.define("MPI_HOME", spec["mpi"].prefix)]
 
-        if "+umpire" in self.spec:
-            args.append(self.define_from_variant("EXAWIND_ENABLE_UMPIRE", "umpire"))
-            args.append(self.define("UMPIRE_DIR", self.spec["umpire"].prefix))
-
         if spec.satisfies("+cuda"):
+            args.append(self.define("CMAKE_CXX_COMPILER", spec["mpi"].mpicxx))
+            args.append(self.define("CMAKE_C_COMPILER", spec["mpi"].mpicc))
             args.append(self.define("EXAWIND_ENABLE_CUDA", True))
             args.append(self.define("CUDAToolkit_ROOT", self.spec["cuda"].prefix))
             args.append(self.define("EXAWIND_CUDA_ARCH", self.spec.variants["cuda_arch"].value))
@@ -77,6 +102,7 @@ def cmake_args(self):
             targets = self.spec.variants["amdgpu_target"].value
             args.append(self.define("EXAWIND_ENABLE_ROCM", True))
             args.append(self.define("CMAKE_CXX_COMPILER", self.spec["hip"].hipcc))
+            # Optimization to only build one specific target architecture:
             args.append(self.define("CMAKE_HIP_ARCHITECTURES", ";".join(str(x) for x in targets)))
             args.append(self.define("AMDGPU_TARGETS", ";".join(str(x) for x in targets)))
             args.append(self.define("GPU_TARGETS", ";".join(str(x) for x in targets)))
@@ -84,14 +110,17 @@ def cmake_args(self):
         if spec.satisfies("^amr-wind+hdf5"):
             args.append(self.define("H5Z_ZFP_USE_STATIC_LIBS", True))
 
-        if spec.satisfies("^amr-wind+ascent"):
-            args.append(self.define("CMAKE_EXE_LINKER_FLAGS", self.compiler.openmp_flag))
-
         return args
 
     def setup_build_environment(self, env):
-        if "~stk_simd" in self.spec:
-            env.append_flags("CXXFLAGS", "-DUSE_STK_SIMD_NONE")
+        env.append_flags("CXXFLAGS", "-DUSE_STK_SIMD_NONE")
+        if "+rocm+amr_wind_gpu~nalu_wind_gpu" in self.spec:
+            # Manually turn off device self.defines to solve Kokkos issues in Nalu-Wind headers
+            env.append_flags("CXXFLAGS", "-U__HIP_DEVICE_COMPILE__ -DDESUL_HIP_RDC")
+        if "+cuda" in self.spec:
+            env.set("OMPI_CXX", self.spec["kokkos-nvcc-wrapper"].kokkos_cxx)
+            env.set("MPICH_CXX", self.spec["kokkos-nvcc-wrapper"].kokkos_cxx)
+            env.set("MPICXX_CXX", self.spec["kokkos-nvcc-wrapper"].kokkos_cxx)
         if "+rocm" in self.spec:
             env.set("OMPI_CXX", self.spec["hip"].hipcc)
             env.set("MPICH_CXX", self.spec["hip"].hipcc)
diff --git a/var/spack/repos/builtin/packages/file/package.py b/var/spack/repos/builtin/packages/file/package.py
index d2bc55cb710..3d8029ab49c 100644
--- a/var/spack/repos/builtin/packages/file/package.py
+++ b/var/spack/repos/builtin/packages/file/package.py
@@ -19,6 +19,7 @@ class File(AutotoolsPackage):
 
     license("BSD-2-Clause")
 
+    version("5.45", sha256="fc97f51029bb0e2c9f4e3bffefdaf678f0e039ee872b9de5c002a6d09c784d82")
     version("5.44", sha256="3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b")
     version("5.43", sha256="8c8015e91ae0e8d0321d94c78239892ef9dbc70c4ade0008c0e95894abfb1991")
     version("5.42", sha256="c076fb4d029c74073f15c43361ef572cfb868407d347190ba834af3b1639b0e4")
diff --git a/var/spack/repos/builtin/packages/fish/package.py b/var/spack/repos/builtin/packages/fish/package.py
index cca14b84bda..a2126dbaa13 100644
--- a/var/spack/repos/builtin/packages/fish/package.py
+++ b/var/spack/repos/builtin/packages/fish/package.py
@@ -22,6 +22,11 @@ class Fish(CMakePackage):
     license("GPL-2.0-only")
 
     version("master", branch="master")
+    version("3.7.1", sha256="614c9f5643cd0799df391395fa6bbc3649427bb839722ce3b114d3bbc1a3b250")
+    version("3.7.0", sha256="df1b7378b714f0690b285ed9e4e58afe270ac98dbc9ca5839589c1afcca33ab1")
+    version("3.6.4", sha256="0f3f610e580de092fbe882c8aa76623ecf91bb16fdf0543241e6e90d5d4bc393")
+    version("3.6.3", sha256="55520128c8ef515908a3821423b430db9258527a6c6acb61c7cb95626b5a48d5")
+    version("3.6.2", sha256="a21a6c986f1f80273895ba7e905fa80ad7e1a262ddb3d979efa443367eaf4863")
     version("3.6.1", sha256="55402bb47ca6739d8aba25e41780905b5ce1bce0a5e0dd17dca908b5bc0b49b2")
     version("3.6.0", sha256="97044d57773ee7ca15634f693d917ed1c3dc0fa7fde1017f1626d60b83ea6181")
     version("3.5.1", sha256="a6d45b3dc5a45dd31772e7f8dfdfecabc063986e8f67d60bd7ca60cc81db6928")
diff --git a/var/spack/repos/builtin/packages/gromacs/package.py b/var/spack/repos/builtin/packages/gromacs/package.py
index e95e88c211f..0c1f000a7c3 100644
--- a/var/spack/repos/builtin/packages/gromacs/package.py
+++ b/var/spack/repos/builtin/packages/gromacs/package.py
@@ -33,8 +33,10 @@ class Gromacs(CMakePackage, CudaPackage):
 
     version("main", branch="main")
     version("master", branch="main", deprecated=True)
+    version("2024.2", sha256="802a7e335f2e895770f57b159e4ec368ebb0ff2ce6daccf706c6e8025c36852b")
     version("2024.1", sha256="937d8f12a36fffbf2af7add71adbb5aa5c5537892d46c9a76afbecab1aa0aac7")
     version("2024", sha256="04d226d52066a8bc3a42e00d6213de737b4ec292e26703065924ff01956801e2")
+    version("2023.5", sha256="9cc491d3601a5fe0ec0de727e4432c34877f596fe8a463d4cf0f0f53fb34d08b")
     version("2023.4", sha256="e5d6c4d9e7ccacfaccb0888619bd21b5ea8911f82b410e68d6db5d40f695f231")
     version("2023.3", sha256="4ec8f8d0c7af76b13f8fd16db8e2c120e749de439ae9554d9f653f812d78d1cb")
     version("2023.2", sha256="bce1480727e4b2bb900413b75d99a3266f3507877da4f5b2d491df798f9fcdae")
@@ -161,7 +163,7 @@ class Gromacs(CMakePackage, CudaPackage):
         "sve",
         default=True,
         description="Enable SVE on aarch64 if available",
-        when="target=neoverse_v1",
+        when="target=neoverse_v1:,neoverse_v2:",
     )
     variant(
         "sve", default=True, description="Enable SVE on aarch64 if available", when="target=a64fx"
diff --git a/var/spack/repos/builtin/packages/hipcc/0014-remove-compiler-rt-linkage-for-host.6.0.patch b/var/spack/repos/builtin/packages/hipcc/0014-remove-compiler-rt-linkage-for-host.6.0.patch
index e49f0552041..74b90fafe03 100644
--- a/var/spack/repos/builtin/packages/hipcc/0014-remove-compiler-rt-linkage-for-host.6.0.patch
+++ b/var/spack/repos/builtin/packages/hipcc/0014-remove-compiler-rt-linkage-for-host.6.0.patch
@@ -1,25 +1,24 @@
 diff --git a/bin/hipcc.pl b/bin/hipcc.pl
-index 513a427..cd2d6ac 100755
+index 513a427..780dc5c 100755
 --- a/bin/hipcc.pl
 +++ b/bin/hipcc.pl
-@@ -160,11 +160,14 @@ if ($HIP_PLATFORM eq "amd") {
+@@ -160,11 +160,13 @@ if ($HIP_PLATFORM eq "amd") {
      if($isWindows) {
          $execExtension = ".exe";
      }
 -    $HIPCC=get_normalized_path("$HIP_CLANG_PATH/clang++" . $execExtension);
 +    # llvm_path is set inside the hip recipe
-+    $LLVM_PATH= $ENV{'LLVM_PATH'};
-+    $HIPCC="${LLVM_PATH}/bin/clang++" . $execExtension;
++    $HIP_CLANG_PATH= $ENV{'HIP_CLANG_PATH'};
++    $HIPCC="${HIP_CLANG_PATH}/clang++" . $execExtension;
 
      # If $HIPCC clang++ is not compiled, use clang instead
      if ( ! -e $HIPCC ) {
 -        $HIPCC=get_normalized_path("$HIP_CLANG_PATH/clang" . $execExtension);
-+        $LLVM_PATH= $ENV{'LLVM_PATH'};
-+        $HIPCC="${LLVM_PATH}/bin/clang" . $execExtension;
++        $HIPCC="${HIP_CLANG_PATH}/clang" . $execExtension;
          $HIPLDFLAGS = "--driver-mode=g++";
      }
      # to avoid using dk linker or MSVC linker
-@@ -484,7 +487,8 @@ if($HIP_PLATFORM eq "amd"){
+@@ -484,7 +486,8 @@ if($HIP_PLATFORM eq "amd"){
              $targetsStr = $ENV{HCC_AMDGPU_TARGET};
          } elsif (not $isWindows) {
              # Else try using rocm_agent_enumerator
diff --git a/var/spack/repos/builtin/packages/hipcc/0014-remove-compiler-rt-linkage-for-host.6.1.patch b/var/spack/repos/builtin/packages/hipcc/0014-remove-compiler-rt-linkage-for-host.6.1.patch
index 3b9c3aa2284..5bd44f33b88 100644
--- a/var/spack/repos/builtin/packages/hipcc/0014-remove-compiler-rt-linkage-for-host.6.1.patch
+++ b/var/spack/repos/builtin/packages/hipcc/0014-remove-compiler-rt-linkage-for-host.6.1.patch
@@ -1,26 +1,24 @@
 diff --git a/amd/hipcc/bin/hipcc.pl b/amd/hipcc/bin/hipcc.pl
-index 513a427..ffeafaa 100755
+index 513a427..780dc5c 100755
 --- a/amd/hipcc/bin/hipcc.pl
 +++ b/amd/hipcc/bin/hipcc.pl
-@@ -160,11 +160,15 @@ if ($HIP_PLATFORM eq "amd") {
+@@ -160,11 +160,13 @@ if ($HIP_PLATFORM eq "amd") {
      if($isWindows) {
          $execExtension = ".exe";
      }
 -    $HIPCC=get_normalized_path("$HIP_CLANG_PATH/clang++" . $execExtension);
-+    # llvm_path is set inside the hip recipe
-+    $LLVM_PATH= $ENV{'LLVM_PATH'};
-+    $HIPCC="${LLVM_PATH}/bin/clang++" . $execExtension;
-+
- 
++    # hip_clang_path is set inside the hip recipe
++    $HIP_CLANG_PATH= $ENV{'HIP_CLANG_PATH'};
++    $HIPCC="${HIP_CLANG_PATH}/clang++" . $execExtension;
+
      # If $HIPCC clang++ is not compiled, use clang instead
      if ( ! -e $HIPCC ) {
 -        $HIPCC=get_normalized_path("$HIP_CLANG_PATH/clang" . $execExtension);
-+        $LLVM_PATH= $ENV{'LLVM_PATH'};
-+        $HIPCC="${LLVM_PATH}/bin/clang" . $execExtension;
++        $HIPCC="${HIP_CLANG_PATH}/clang" . $execExtension;
          $HIPLDFLAGS = "--driver-mode=g++";
      }
      # to avoid using dk linker or MSVC linker
-@@ -484,7 +488,8 @@ if($HIP_PLATFORM eq "amd"){
+@@ -484,7 +486,8 @@ if($HIP_PLATFORM eq "amd"){
              $targetsStr = $ENV{HCC_AMDGPU_TARGET};
          } elsif (not $isWindows) {
              # Else try using rocm_agent_enumerator
diff --git a/var/spack/repos/builtin/packages/man-db/package.py b/var/spack/repos/builtin/packages/man-db/package.py
index 836c0c2a27d..83c3df7eef7 100644
--- a/var/spack/repos/builtin/packages/man-db/package.py
+++ b/var/spack/repos/builtin/packages/man-db/package.py
@@ -18,6 +18,8 @@ class ManDb(AutotoolsPackage):
 
     license("GPL-2.0-or-later")
 
+    version("2.12.1", sha256="ddee249daeb78cf92bab794ccd069cc8b575992265ea20e239e887156e880265")
+    version("2.12.0", sha256="415a6284a22764ad22ff0f66710d853be7790dd451cd71436e3d25c74d996a95")
     version("2.11.2", sha256="cffa1ee4e974be78646c46508e6dd2f37e7c589aaab2938cc1064f058fef9f8d")
     version("2.10.2", sha256="ee97954d492a13731903c9d0727b9b01e5089edbd695f0cdb58d405a5af5514d")
     version("2.10.1", sha256="2ffd8f2e80122fe72e60c740c851e6a3e15c9a7921185eb4752c1c672824bed6")
diff --git a/var/spack/repos/builtin/packages/nalu-wind/package.py b/var/spack/repos/builtin/packages/nalu-wind/package.py
index 18efa6d6fe9..2d85e9aac91 100644
--- a/var/spack/repos/builtin/packages/nalu-wind/package.py
+++ b/var/spack/repos/builtin/packages/nalu-wind/package.py
@@ -118,12 +118,18 @@ class NaluWind(CMakePackage, CudaPackage, ROCmPackage):
     conflicts("^trilinos+cuda", when="~cuda")
     conflicts("^trilinos+rocm", when="~rocm")
 
+    def setup_dependent_run_environment(self, env, dependent_spec):
+        spec = self.spec
+        if spec.satisfies("+cuda") or spec.satisfies("+rocm"):
+            env.set("CUDA_LAUNCH_BLOCKING", "1")
+            env.set("CUDA_MANAGED_FORCE_DEVICE_ALLOC", "1")
+            env.set("HIP_LAUNCH_BLOCKING", "1")
+            env.set("HIP_MANAGED_FORCE_DEVICE_ALLOC", "1")
+
     def setup_build_environment(self, env):
         spec = self.spec
         env.append_flags("CXXFLAGS", "-DUSE_STK_SIMD_NONE")
         if spec.satisfies("+cuda"):
-            env.set("CUDA_LAUNCH_BLOCKING", "1")
-            env.set("CUDA_MANAGED_FORCE_DEVICE_ALLOC", "1")
             env.set("OMPI_CXX", self.spec["kokkos-nvcc-wrapper"].kokkos_cxx)
             env.set("MPICH_CXX", self.spec["kokkos-nvcc-wrapper"].kokkos_cxx)
             env.set("MPICXX_CXX", self.spec["kokkos-nvcc-wrapper"].kokkos_cxx)
@@ -134,15 +140,15 @@ def cmake_args(self):
         spec = self.spec
 
         args = [
-            self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
             self.define("CMAKE_CXX_COMPILER", spec["mpi"].mpicxx),
             self.define("CMAKE_Fortran_COMPILER", spec["mpi"].mpifc),
             self.define("Trilinos_DIR", spec["trilinos"].prefix),
             self.define("YAML_DIR", spec["yaml-cpp"].prefix),
+            self.define("CMAKE_CXX_STANDARD", "17"),
+            self.define_from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
             self.define_from_variant("ENABLE_CUDA", "cuda"),
             self.define_from_variant("ENABLE_WIND_UTILS", "wind-utils"),
             self.define_from_variant("ENABLE_BOOST", "boost"),
-            self.define_from_variant("CMAKE_CXX_STANDARD", "17"),
             self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
             self.define_from_variant("ENABLE_OPENFAST", "openfast"),
             self.define_from_variant("ENABLE_TIOGA", "tioga"),
diff --git a/var/spack/repos/builtin/packages/nccl/package.py b/var/spack/repos/builtin/packages/nccl/package.py
index 4517124f661..44418411126 100644
--- a/var/spack/repos/builtin/packages/nccl/package.py
+++ b/var/spack/repos/builtin/packages/nccl/package.py
@@ -17,6 +17,7 @@ class Nccl(MakefilePackage, CudaPackage):
     maintainers("adamjstewart")
     libraries = ["libnccl.so"]
 
+    version("2.21.5-1", sha256="1923596984d85e310b5b6c52b2c72a1b93da57218f2bc5a5c7ac3d59297a3303")
     version("2.20.3-1", sha256="19456bd63ca7d23a8319cbbdbaaf6c25949dd51161a9f8809f6b7453282983dd")
     version("2.19.3-1", sha256="1c5474553afedb88e878c772f13d6f90b9226b3f2971dfa6f873adb9443100c2")
     version("2.18.5-1", sha256="16ac98f3e926c024ce48e10ab220e19ce734adc48c423cfd55ad6f509bd1179f")
diff --git a/var/spack/repos/builtin/packages/pkg-config/package.py b/var/spack/repos/builtin/packages/pkg-config/package.py
index 82cd66f78ec..8888ffe559d 100644
--- a/var/spack/repos/builtin/packages/pkg-config/package.py
+++ b/var/spack/repos/builtin/packages/pkg-config/package.py
@@ -53,17 +53,18 @@ def setup_dependent_build_environment(self, env, dependent_spec):
         env.append_path("ACLOCAL_PATH", self.prefix.share.aclocal)
 
     def configure_args(self):
+        spec = self.spec
         config_args = ["--enable-shared"]
 
-        if "+internal_glib" in self.spec:
+        if spec.satisfies("+internal_glib"):
             # There's a bootstrapping problem here;
             # glib uses pkg-config as well, so break
             # the cycle by using the internal glib.
             config_args.append("--with-internal-glib")
 
-        c_name = self.spec.compiler.name
-        if "oneapi" in c_name or "cce" in c_name:
-            # Don't treat int-conversion warning as error with oneapi and cce.
-            config_args.append("CFLAGS=-Wno-error=int-conversion")
+        for strict_compiler in ("%oneapi", "%cce", "%apple-clang@15:", "%clang@15:"):
+            if spec.satisfies(strict_compiler):
+                config_args.append("CFLAGS=-Wno-error=int-conversion")
+                break
 
         return config_args
diff --git a/var/spack/repos/builtin/packages/py-tensorflow/Find_ROCm_Components_Individiually.2.16.patch b/var/spack/repos/builtin/packages/py-tensorflow/Find_ROCm_Components_Individiually.2.16.patch
new file mode 100644
index 00000000000..aab9b09c531
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-tensorflow/Find_ROCm_Components_Individiually.2.16.patch
@@ -0,0 +1,364 @@
+diff --git a/third_party/gpus/find_rocm_config.py b/third_party/gpus/find_rocm_config.py
+index cd64efe6..e38495e2 100644
+--- a/third_party/gpus/find_rocm_config.py
++++ b/third_party/gpus/find_rocm_config.py
+@@ -70,9 +70,10 @@ def _get_header_version(path, name):
+ def _find_rocm_config(rocm_install_path):
+
+   def rocm_version_numbers(path):
++    rocm_core_path = os.environ.get('ROCM_CORE_PATH')
+     possible_version_files = [
+-        "include/rocm-core/rocm_version.h",  # ROCm 5.2
+-        "include/rocm_version.h",  # ROCm 5.1 and prior
++        rocm_core_path + "/include/rocm-core/rocm_version.h",
++        rocm_core_path + "/include/rocm_version.h",  # ROCm 5.1 and prior
+     ]
+     version_file = None
+     for f in possible_version_files:
+@@ -101,9 +102,10 @@ def _find_rocm_config(rocm_install_path):
+ def _find_hipruntime_config(rocm_install_path):
+
+   def hipruntime_version_number(path):
++    hip_path = os.environ.get('HIP_PATH')
+     possible_version_files = [
+-        "include/hip/hip_version.h",  # ROCm 5.2
+-        "hip/include/hip/hip_version.h",  # ROCm 5.1 and prior
++        hip_path + "/include/hip/hip_version.h",
++        hip_path + "/hip/include/hip/hip_version.h",  # ROCm 5.1 and prior
+     ]
+     version_file = None
+     for f in possible_version_files:
+@@ -133,7 +135,9 @@ def _find_hipruntime_config(rocm_install_path):
+ def _find_miopen_config(rocm_install_path):
+
+   def miopen_version_numbers(path):
++    miopen_path = os.environ.get('MIOPEN_PATH')
+     possible_version_files = [
++        miopen_path + "/include/miopen/version.h",
+         "include/miopen/version.h",  # ROCm 5.2 and prior
+         "miopen/include/miopen/version.h",  # ROCm 5.1 and prior
+     ]
+@@ -164,7 +168,9 @@ def _find_miopen_config(rocm_install_path):
+ def _find_rocblas_config(rocm_install_path):
+
+   def rocblas_version_numbers(path):
++    rocblas_path = os.environ.get('ROCBLAS_PATH')
+     possible_version_files = [
++        rocblas_path + "/include/rocblas/internal/rocblas-version.h",
+         "include/rocblas/internal/rocblas-version.h",  # ROCm 5.2
+         "rocblas/include/internal/rocblas-version.h",  # ROCm 5.1 and prior
+     ]
+@@ -196,7 +202,9 @@ def _find_rocblas_config(rocm_install_path):
+ def _find_rocrand_config(rocm_install_path):
+
+   def rocrand_version_number(path):
++    rocrand_path = os.environ.get('ROCRAND_PATH')
+     possible_version_files = [
++        rocrand_path + "/include/rocrand/rocrand_version.h",
+         "include/rocrand/rocrand_version.h",  # ROCm 5.1
+         "rocrand/include/rocrand_version.h",  # ROCm 5.0 and prior
+     ]
+@@ -222,7 +230,9 @@ def _find_rocrand_config(rocm_install_path):
+ def _find_rocfft_config(rocm_install_path):
+
+   def rocfft_version_numbers(path):
++    rocfft_path = os.environ.get('ROCFFT_PATH')
+     possible_version_files = [
++        rocfft_path + "/include/rocfft/rocfft-version.h",
+         "include/rocfft/rocfft-version.h",  # ROCm 5.2
+         "rocfft/include/rocfft-version.h",  # ROCm 5.1 and prior
+     ]
+@@ -253,7 +263,9 @@ def _find_rocfft_config(rocm_install_path):
+ def _find_hipfft_config(rocm_install_path):
+
+   def hipfft_version_numbers(path):
++    hipfft_path = os.environ.get('HIPFFT_PATH')
+     possible_version_files = [
++        hipfft_path + "/include/hipfft/hipfft-version.h",
+         "include/hipfft/hipfft-version.h",  # ROCm 5.2
+         "hipfft/include/hipfft-version.h",  # ROCm 5.1 and prior
+     ]
+@@ -284,7 +296,9 @@ def _find_hipfft_config(rocm_install_path):
+ def _find_roctracer_config(rocm_install_path):
+
+   def roctracer_version_numbers(path):
++    roctracer_path = os.environ.get('ROCTRACER_PATH')
+     possible_version_files = [
++        roctracer_path + "/include/roctracer/roctracer.h",
+         "include/roctracer/roctracer.h",  # ROCm 5.2
+         "roctracer/include/roctracer.h",  # ROCm 5.1 and prior
+     ]
+@@ -316,7 +330,9 @@ def _find_roctracer_config(rocm_install_path):
+ def _find_hipsparse_config(rocm_install_path):
+
+   def hipsparse_version_numbers(path):
++    hipsparse_path = os.environ.get('HIPSPARSE_PATH')
+     possible_version_files = [
++        hipsparse_path + "/include/hipsparse/hipsparse-version.h",
+         "include/hipsparse/hipsparse-version.h",  # ROCm 5.2
+         "hipsparse/include/hipsparse-version.h",  # ROCm 5.1 and prior
+     ]
+@@ -346,7 +362,9 @@ def _find_hipsparse_config(rocm_install_path):
+ def _find_hipsolver_config(rocm_install_path):
+
+   def hipsolver_version_numbers(path):
++    hipsolver_path = os.environ.get('HIPSOLVER_PATH')
+     possible_version_files = [
++        hipsolver_path + "/include/hipsolver/internal/hipsolver-version.h",
+         "include/hipsolver/internal/hipsolver-version.h",  # ROCm 5.2
+         "hipsolver/include/internal/hipsolver-version.h",  # ROCm 5.1
+         "hipsolver/include/hipsolver-version.h",  # ROCm 5.0 and prior
+@@ -378,7 +396,9 @@ def _find_hipsolver_config(rocm_install_path):
+ def _find_rocsolver_config(rocm_install_path):
+
+   def rocsolver_version_numbers(path):
++    rocsolver_path = os.environ.get('ROCSOLVER_PATH')
+     possible_version_files = [
++        rocsolver_path + "/include/rocsolver/rocsolver-version.h",
+         "include/rocsolver/rocsolver-version.h",  # ROCm 5.2
+         "rocsolver/include/rocsolver-version.h",  # ROCm 5.1 and prior
+     ]
+diff --git a/third_party/gpus/rocm_configure.bzl b/third_party/gpus/rocm_configure.bzl
+index 6eb17486..3e9a68d3 100644
+--- a/third_party/gpus/rocm_configure.bzl
++++ b/third_party/gpus/rocm_configure.bzl
+@@ -30,7 +30,6 @@ load(
+     "which",
+ )
+
+-
+ _GCC_HOST_COMPILER_PATH = "GCC_HOST_COMPILER_PATH"
+ _GCC_HOST_COMPILER_PREFIX = "GCC_HOST_COMPILER_PREFIX"
+ _ROCM_TOOLKIT_PATH = "ROCM_PATH"
+@@ -184,17 +183,23 @@ def _rocm_include_path(repository_ctx, rocm_config, bash_bin):
+     """
+     inc_dirs = []
+
++    hsa_path = get_host_environ(repository_ctx, "HSA_ROCR_DEV_PATH")
++    hip_path = get_host_environ(repository_ctx, "HIP_PATH")
++    rocprim_path = get_host_environ(repository_ctx, "ROCPRIM_PATH")
++    rocsolver_path = get_host_environ(repository_ctx, "ROCSOLVER_PATH")
++    rocblas_path = get_host_environ(repository_ctx, "ROCBLAS_PATH")
++    llvm_path = get_host_environ(repository_ctx, "LLVM_PATH")
+     # Add HSA headers (needs to match $HSA_PATH)
+-    inc_dirs.append(rocm_config.rocm_toolkit_path + "/hsa/include")
++    inc_dirs.append(hsa_path + "/include/hsa")
+
+     # Add HIP headers (needs to match $HIP_PATH)
+     inc_dirs.append(rocm_config.rocm_toolkit_path + "/hip/include")
+     if int(rocm_config.rocm_version_number) >= 50200:
+         inc_dirs.append(rocm_config.rocm_toolkit_path + "/include")
+-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/hip")
+-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/rocprim")
+-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/rocsolver")
+-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/rocblas")
++        inc_dirs.append(hip_path + "/include/hip")
++        inc_dirs.append(rocprim_path + "/include/rocprim")
++        inc_dirs.append(rocsolver_path + "/include/rocsolver")
++        inc_dirs.append(rocblas_path + "/include/rocblas")
+
+     # Add HIP-Clang headers (realpath relative to compiler binary)
+     rocm_toolkit_path = realpath(repository_ctx, rocm_config.rocm_toolkit_path, bash_bin)
+@@ -203,7 +208,7 @@ def _rocm_include_path(repository_ctx, rocm_config, bash_bin):
+     inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/14.0.0/include")
+     inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/15.0.0/include")
+     inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/16.0.0/include")
+-    inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/17.0.0/include")
++    inc_dirs.append(llvm_path + "/lib/clang/17.0.0/include/")
+     inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/17/include")
+     inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/18/include")
+     inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/19/include")
+@@ -230,7 +235,8 @@ def _amdgpu_targets(repository_ctx, rocm_toolkit_path, bash_bin):
+     """Returns a list of strings representing AMDGPU targets."""
+     amdgpu_targets_str = get_host_environ(repository_ctx, _TF_ROCM_AMDGPU_TARGETS)
+     if not amdgpu_targets_str:
+-        cmd = "%s/bin/rocm_agent_enumerator" % rocm_toolkit_path
++        rocminfo_path = get_host_environ(repository_ctx, "ROCMINFO_PATH")
++        cmd = rocminfo_path + "/bin/rocm_agent_enumerator"
+         result = execute(repository_ctx, [bash_bin, "-c", cmd])
+         targets = [target for target in result.stdout.strip().split("\n") if target != "gfx000"]
+         targets = {x: None for x in targets}
+@@ -351,23 +357,33 @@ def _find_libs(repository_ctx, rocm_config, hipfft_or_rocfft, miopen_path, rccl_
+     Returns:
+       Map of library names to structs of filename and path
+     """
++    hip_path = get_host_environ(repository_ctx, "HIP_PATH")
++    rocblas_path = get_host_environ(repository_ctx, "ROCBLAS_PATH")
++    hipfft_path = get_host_environ(repository_ctx, "HIPFFT_PATH")
++    hiprand_path = get_host_environ(repository_ctx, "HIPRAND_PATH")
++    hipsparse_path = get_host_environ(repository_ctx, "HIPSPARSE_PATH")
++    roctracer_path = get_host_environ(repository_ctx, "ROCTRACER_PATH")
++    rocsolver_path = get_host_environ(repository_ctx, "ROCSOLVER_PATH")
++    hipsolver_path = get_host_environ(repository_ctx, "HIPSOLVER_PATH")
++    hipblas_path = get_host_environ(repository_ctx, "HIPBLAS_PATH")
++
+     libs_paths = [
+         (name, _rocm_lib_paths(repository_ctx, name, path))
+         for name, path in [
+-            ("amdhip64", rocm_config.rocm_toolkit_path),
+-            ("rocblas", rocm_config.rocm_toolkit_path),
+-            (hipfft_or_rocfft, rocm_config.rocm_toolkit_path),
+-            ("hiprand", rocm_config.rocm_toolkit_path),
+-            ("MIOpen", miopen_path),
++            ("amdhip64", hip_path + "/lib"),
++            ("rocblas", rocblas_path),
++            (hipfft_or_rocfft, hipfft_path),
++            ("hiprand", hiprand_path),
++           ("MIOpen", miopen_path),
+             ("rccl", rccl_path),
+-            ("hipsparse", rocm_config.rocm_toolkit_path),
+-            ("roctracer64", rocm_config.rocm_toolkit_path),
+-            ("rocsolver", rocm_config.rocm_toolkit_path),
++            ("hipsparse", hipsparse_path),
++            ("roctracer64", roctracer_path + "/lib"),
++            ("rocsolver", rocsolver_path),
+         ]
+     ]
+     if int(rocm_config.rocm_version_number) >= 40500:
+-        libs_paths.append(("hipsolver", _rocm_lib_paths(repository_ctx, "hipsolver", rocm_config.rocm_toolkit_path)))
+-        libs_paths.append(("hipblas", _rocm_lib_paths(repository_ctx, "hipblas", rocm_config.rocm_toolkit_path)))
++        libs_paths.append(("hipsolver", _rocm_lib_paths(repository_ctx, "hipsolver", hipsolver_path)))
++        libs_paths.append(("hipblas", _rocm_lib_paths(repository_ctx, "hipblas", hipblas_path)))
+
+     # hipblaslt may be absent even in versions of ROCm where it exists
+     # (it is not installed by default in some containers). Autodetect.
+@@ -560,12 +576,13 @@ def _create_local_rocm_repository(repository_ctx):
+     hipfft_or_rocfft = "rocfft" if rocm_version_number < 40100 else "hipfft"
+
+     # For ROCm 5.2 and above, find MIOpen and RCCL in the main rocm lib path
+-    miopen_path = rocm_config.rocm_toolkit_path + "/miopen" if rocm_version_number < 50200 else rocm_config.rocm_toolkit_path
+-    rccl_path = rocm_config.rocm_toolkit_path + "/rccl" if rocm_version_number < 50200 else rocm_config.rocm_toolkit_path
++    miopen_path = get_host_environ(repository_ctx, "MIOPEN_PATH") + "/miopen"
++    rccl_path = get_host_environ(repository_ctx, "RCCL_PATH") + "/rccl"
+
+     # Copy header and library files to execroot.
+     # rocm_toolkit_path
+     rocm_toolkit_path = rocm_config.rocm_toolkit_path
++    rocblas_path = get_host_environ(repository_ctx, "ROCBLAS_PATH")
+     copy_rules = [
+         make_copy_dir_rule(
+             repository_ctx,
+@@ -578,6 +595,53 @@ def _create_local_rocm_repository(repository_ctx):
+     # explicitly copy (into the local_config_rocm repo) the $ROCM_PATH/hiprand/include and
+     # $ROCM_PATH/rocrand/include dirs, only once the softlink to them in $ROCM_PATH/include
+     # dir has been removed. This removal will happen in a near-future ROCm release.
++    rocm_parts = [
++        "rocblas",
++       "rocprim",
++        "hipfft",
++        "hipsparse",
++        "rocsolver",
++        "hipsolver",
++        "hipblas",
++       "rccl",
++       "miopen",
++       "roctracer",
++    ]
++    rocm_includes = ""
++
++    for part in rocm_parts:
++        part_cap = part.upper().replace("-","_")
++        part_path = get_host_environ(repository_ctx, part_cap + "_PATH")
++        rocm_includes = rocm_includes + '":' + part + '-include",\n'
++        rocm_includes = rocm_includes + '":' + part + '-include-second",\n'
++        copy_rules.append(
++            make_copy_dir_rule(
++                repository_ctx,
++                name = part + "-include",
++                src_dir = part_path + "/include/" + part,
++                out_dir = "rocm/include",
++            ),
++        )
++        copy_rules.append(
++            make_copy_dir_rule(
++                repository_ctx,
++                name = part + "-include-second",
++                src_dir = part_path + "/include/" + part,
++                out_dir = "rocm/include/" + part,
++            ),
++        )
++    hipcub_path = get_host_environ(repository_ctx, "HIPCUB_PATH")
++    copy_rules.append(
++        make_copy_dir_rule(
++            repository_ctx,
++            name = "hipcub-include",
++            src_dir = hipcub_path + "/include/hipcub/",
++            out_dir = "rocm/include/hipcub/",
++        )
++    )
++    rocm_includes = rocm_includes + '":hipcub-include",\n'
++
++    hiprand_path = get_host_environ(repository_ctx, "HIPRAND_PATH")
+     hiprand_include = ""
+     hiprand_include_softlink = rocm_config.rocm_toolkit_path + "/include/hiprand"
+     softlink_exists = files_exist(repository_ctx, [hiprand_include_softlink], bash_bin)
+@@ -587,11 +651,12 @@ def _create_local_rocm_repository(repository_ctx):
+             make_copy_dir_rule(
+                 repository_ctx,
+                 name = "hiprand-include",
+-                src_dir = rocm_toolkit_path + "/hiprand/include",
++                src_dir = hiprand_path + "/include/hiprand",
+                 out_dir = "rocm/include/hiprand",
+             ),
+         )
+
++    rocrand_path = get_host_environ(repository_ctx, "ROCRAND_PATH")
+     rocrand_include = ""
+     rocrand_include_softlink = rocm_config.rocm_toolkit_path + "/include/rocrand"
+     softlink_exists = files_exist(repository_ctx, [rocrand_include_softlink], bash_bin)
+@@ -601,11 +666,10 @@ def _create_local_rocm_repository(repository_ctx):
+             make_copy_dir_rule(
+                 repository_ctx,
+                 name = "rocrand-include",
+-                src_dir = rocm_toolkit_path + "/rocrand/include",
++                src_dir = rocrand_path + "/include/rocrand",
+                 out_dir = "rocm/include/rocrand",
+             ),
+         )
+-
+     rocm_libs = _find_libs(repository_ctx, rocm_config, hipfft_or_rocfft, miopen_path, rccl_path, bash_bin)
+     rocm_lib_srcs = []
+     rocm_lib_outs = []
+@@ -620,7 +684,8 @@ def _create_local_rocm_repository(repository_ctx):
+         outs = rocm_lib_outs,
+     ))
+
+-    clang_offload_bundler_path = rocm_toolkit_path + "/llvm/bin/clang-offload-bundler"
++    llvm_amdgpu_path = get_host_environ(repository_ctx, "LLVM_PATH")
++    clang_offload_bundler_path = llvm_amdgpu_path + "/bin/clang-offload-bundler"
+
+     # copy files mentioned in third_party/gpus/rocm/BUILD
+     copy_rules.append(make_copy_files_rule(
+@@ -665,9 +730,7 @@ def _create_local_rocm_repository(repository_ctx):
+         "%{roctracer_lib}": rocm_libs["roctracer64"].file_name,
+         "%{rocsolver_lib}": rocm_libs["rocsolver"].file_name,
+         "%{copy_rules}": "\n".join(copy_rules),
+-        "%{rocm_headers}": ('":rocm-include",\n' +
+-                            hiprand_include +
+-                            rocrand_include),
++        "%{rocm_headers}": ('":rocm-include",\n' + rocrand_include + hiprand_include + rocm_includes),
+     }
+     if rocm_libs["hipblaslt"] != None:
+         repository_dict["%{hipblaslt_lib}"] = rocm_libs["hipblaslt"].file_name
+@@ -735,16 +798,18 @@ def _create_local_rocm_repository(repository_ctx):
+         tpl_paths["crosstool:hipcc_cc_toolchain_config.bzl"],
+     )
+
++    hip_path = get_host_environ(repository_ctx, "HIP_PATH")
++    hsa_rocr_dev_path = get_host_environ(repository_ctx, "HSA_ROCR_DEV_PATH")
+     repository_ctx.template(
+         "crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc",
+         tpl_paths["crosstool:clang/bin/crosstool_wrapper_driver_rocm"],
+         {
+             "%{cpu_compiler}": str(cc),
+-            "%{hipcc_path}": rocm_config.rocm_toolkit_path + "/bin/hipcc",
++            "%{hipcc_path}": hip_path + "/bin/hipcc",
+             "%{hipcc_env}": _hipcc_env(repository_ctx),
+-            "%{rocr_runtime_path}": rocm_config.rocm_toolkit_path + "/lib",
++            "%{rocr_runtime_path}": hsa_rocr_dev_path + "/lib",
+             "%{rocr_runtime_library}": "hsa-runtime64",
+-            "%{hip_runtime_path}": rocm_config.rocm_toolkit_path + "/lib",
++            "%{hip_runtime_path}": hip_path + "/lib",
+             "%{hip_runtime_library}": "amdhip64",
+             "%{crosstool_verbose}": _crosstool_verbose(repository_ctx),
+             "%{gcc_host_compiler_path}": str(cc),
diff --git a/var/spack/repos/builtin/packages/py-tensorflow/Find_ROCm_Components_Individiually.patch b/var/spack/repos/builtin/packages/py-tensorflow/Find_ROCm_Components_Individiually.patch
new file mode 100644
index 00000000000..58ac82b4293
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-tensorflow/Find_ROCm_Components_Individiually.patch
@@ -0,0 +1,388 @@
+diff --git a/tensorflow/tsl/platform/default/rocm_rocdl_path.cc b/tensorflow/tsl/platform/default/rocm_rocdl_path.cc
+index dfd9721b..88215316 100644
+--- a/tensorflow/tsl/platform/default/rocm_rocdl_path.cc
++++ b/tensorflow/tsl/platform/default/rocm_rocdl_path.cc
+@@ -40,6 +40,9 @@ string RocmRoot() {
+ #endif
+ }
+
+-string RocdlRoot() { return io::JoinPath(RocmRoot(), "amdgcn/bitcode"); }
++string RocdlRoot() {
++  const char* llvm_path_env = std::getenv("LLVM_PATH");
++  return io::JoinPath(llvm_path_env, "amdgcn/bitcode");
++}
+
+ }  // namespace tsl
+diff --git a/third_party/gpus/find_rocm_config.py b/third_party/gpus/find_rocm_config.py
+index cd64efe6..e38495e2 100644
+--- a/third_party/gpus/find_rocm_config.py
++++ b/third_party/gpus/find_rocm_config.py
+@@ -70,9 +70,10 @@ def _get_header_version(path, name):
+ def _find_rocm_config(rocm_install_path):
+
+   def rocm_version_numbers(path):
++    rocm_core_path = os.environ.get('ROCM_CORE_PATH')
+     possible_version_files = [
+-        "include/rocm-core/rocm_version.h",  # ROCm 5.2
+-        "include/rocm_version.h",  # ROCm 5.1 and prior
++        rocm_core_path + "/include/rocm-core/rocm_version.h",
++        rocm_core_path + "/include/rocm_version.h",  # ROCm 5.1 and prior
+     ]
+     version_file = None
+     for f in possible_version_files:
+@@ -101,9 +102,10 @@ def _find_rocm_config(rocm_install_path):
+ def _find_hipruntime_config(rocm_install_path):
+
+   def hipruntime_version_number(path):
++    hip_path = os.environ.get('HIP_PATH')
+     possible_version_files = [
+-        "include/hip/hip_version.h",  # ROCm 5.2
+-        "hip/include/hip/hip_version.h",  # ROCm 5.1 and prior
++        hip_path + "/include/hip/hip_version.h",
++        hip_path + "/hip/include/hip/hip_version.h",  # ROCm 5.1 and prior
+     ]
+     version_file = None
+     for f in possible_version_files:
+@@ -133,7 +135,9 @@ def _find_hipruntime_config(rocm_install_path):
+ def _find_miopen_config(rocm_install_path):
+
+   def miopen_version_numbers(path):
++    miopen_path = os.environ.get('MIOPEN_PATH')
+     possible_version_files = [
++        miopen_path + "/include/miopen/version.h",
+         "include/miopen/version.h",  # ROCm 5.2 and prior
+         "miopen/include/miopen/version.h",  # ROCm 5.1 and prior
+     ]
+@@ -164,7 +168,9 @@ def _find_miopen_config(rocm_install_path):
+ def _find_rocblas_config(rocm_install_path):
+
+   def rocblas_version_numbers(path):
++    rocblas_path = os.environ.get('ROCBLAS_PATH')
+     possible_version_files = [
++        rocblas_path + "/include/rocblas/internal/rocblas-version.h",
+         "include/rocblas/internal/rocblas-version.h",  # ROCm 5.2
+         "rocblas/include/internal/rocblas-version.h",  # ROCm 5.1 and prior
+     ]
+@@ -196,7 +202,9 @@ def _find_rocblas_config(rocm_install_path):
+ def _find_rocrand_config(rocm_install_path):
+
+   def rocrand_version_number(path):
++    rocrand_path = os.environ.get('ROCRAND_PATH')
+     possible_version_files = [
++        rocrand_path + "/include/rocrand/rocrand_version.h",
+         "include/rocrand/rocrand_version.h",  # ROCm 5.1
+         "rocrand/include/rocrand_version.h",  # ROCm 5.0 and prior
+     ]
+@@ -222,7 +230,9 @@ def _find_rocrand_config(rocm_install_path):
+ def _find_rocfft_config(rocm_install_path):
+
+   def rocfft_version_numbers(path):
++    rocfft_path = os.environ.get('ROCFFT_PATH')
+     possible_version_files = [
++        rocfft_path + "/include/rocfft/rocfft-version.h",
+         "include/rocfft/rocfft-version.h",  # ROCm 5.2
+         "rocfft/include/rocfft-version.h",  # ROCm 5.1 and prior
+     ]
+@@ -253,7 +263,9 @@ def _find_rocfft_config(rocm_install_path):
+ def _find_hipfft_config(rocm_install_path):
+
+   def hipfft_version_numbers(path):
++    hipfft_path = os.environ.get('HIPFFT_PATH')
+     possible_version_files = [
++        hipfft_path + "/include/hipfft/hipfft-version.h",
+         "include/hipfft/hipfft-version.h",  # ROCm 5.2
+         "hipfft/include/hipfft-version.h",  # ROCm 5.1 and prior
+     ]
+@@ -284,7 +296,9 @@ def _find_hipfft_config(rocm_install_path):
+ def _find_roctracer_config(rocm_install_path):
+
+   def roctracer_version_numbers(path):
++    roctracer_path = os.environ.get('ROCTRACER_PATH')
+     possible_version_files = [
++        roctracer_path + "/include/roctracer/roctracer.h",
+         "include/roctracer/roctracer.h",  # ROCm 5.2
+         "roctracer/include/roctracer.h",  # ROCm 5.1 and prior
+     ]
+@@ -316,7 +330,9 @@ def _find_roctracer_config(rocm_install_path):
+ def _find_hipsparse_config(rocm_install_path):
+
+   def hipsparse_version_numbers(path):
++    hipsparse_path = os.environ.get('HIPSPARSE_PATH')
+     possible_version_files = [
++        hipsparse_path + "/include/hipsparse/hipsparse-version.h",
+         "include/hipsparse/hipsparse-version.h",  # ROCm 5.2
+         "hipsparse/include/hipsparse-version.h",  # ROCm 5.1 and prior
+     ]
+@@ -346,7 +362,9 @@ def _find_hipsparse_config(rocm_install_path):
+ def _find_hipsolver_config(rocm_install_path):
+
+   def hipsolver_version_numbers(path):
++    hipsolver_path = os.environ.get('HIPSOLVER_PATH')
+     possible_version_files = [
++        hipsolver_path + "/include/hipsolver/internal/hipsolver-version.h",
+         "include/hipsolver/internal/hipsolver-version.h",  # ROCm 5.2
+         "hipsolver/include/internal/hipsolver-version.h",  # ROCm 5.1
+         "hipsolver/include/hipsolver-version.h",  # ROCm 5.0 and prior
+@@ -378,7 +396,9 @@ def _find_hipsolver_config(rocm_install_path):
+ def _find_rocsolver_config(rocm_install_path):
+
+   def rocsolver_version_numbers(path):
++    rocsolver_path = os.environ.get('ROCSOLVER_PATH')
+     possible_version_files = [
++        rocsolver_path + "/include/rocsolver/rocsolver-version.h",
+         "include/rocsolver/rocsolver-version.h",  # ROCm 5.2
+         "rocsolver/include/rocsolver-version.h",  # ROCm 5.1 and prior
+     ]
+diff --git a/third_party/gpus/find_rocm_config.py.gz.base64 b/third_party/gpus/find_rocm_config.py.gz.base64
+index fee67ec5..6e437b26 100644
+--- a/third_party/gpus/find_rocm_config.py.gz.base64
++++ b/third_party/gpus/find_rocm_config.py.gz.base64
+@@ -1 +1 @@
+-eJztW21v2zgS/q5fQSgoam9dJSlugUMOOcCbZlHftUlhZ7tYtIVB27TNrSz6SCppUPS/7wxJyRIt2YqjvX6JgTa2NPNwOC8Ph7Z4RC7E+l7yxVKTVyevTsjNkpEblighf43FHemneimkikg/jskQxRQZMsXkLZtFwVFwRN7yKYizGUmTGZNEg35/Tafwx93pkQ9MKi4S8io6IR0UCN2tsPsvQLgXKVnRe5IITVLFAIIrMucxI+zrlK014QmZitU65jSZMnLH9dIM40DADPKHgxATTUGagvwaPs2LcoRqYzC+llqvz46P7+7uImqMjYRcHMdWUB2/HVxcXo0uX4LBRuW3JGZKEcn+l3IJU53cE7oGe6Z0AlbG9I4ISehCMrinBdp7J7nmyaJHlJjrOyoZoMy40pJPUl1yVmYdzLkoAO6iCQn7IzIYheSX/mgw6gHG74ObN9e/3ZDf+8Nh/+pmcDki10NycX31enAzuL6CT7+S/tUf5L+Dq9c9wsBVMAz7upZoPxjJ0Y0mdGTEWMmAubAGqTWb8jmfwrySRUoXjCzELZMJTIesmVxxhcFUYN4MUGK+4ppqc2VrUjjMeauvIAzD95InmIbXFysYfiKpvEdjyJJRHH8GIZpqITkzNpJbm32QUgIMRMeaWd4rzVZREGDCq6nkkGeKUQm5oIwr6uAxMVUZpQcRR69pFcDFFabAjGl0VWJczGVmhAFaW/tRfyqSOV+k0jgQ9ZSeiVRHxqo1xUQXGThmiIsNptlSinSxxCRhyS2XIlmxRJNbKrlJyg7Y/278vn/zphsFgzkUF9yL+cwbkju39Ox0rB8yA405TEoTasl0Kk3YCVwCB03FjJX9p+kXZueVxeC+YDEUDd7K7aq0OyrixUJ8scGwvrfxzGJiA2GqfUnl7CXaM4MYaqj7QKWTYh7MpViRCVXOqY4YNrbl9kYEfLUxEdwDrBTkgsZNUJbHYq2PpZiuQhRJkf4o2KIh7nOaxjifOGUBZmsQQM0JCeET2TuhsnfAC+4dZFIQBNOYQp1emBBdopc7l4YCIVTds4CA9QrFYBQyXjA9dsON0ZQxTq1jxGysimYWlYwwJJWmcVxQAltfZ1lrPZ2F3IVtRZySTR3UjHCCMJ6PSM5r7QNxPidh7uMQQyhU5LIBLakG3Mh8LGh/BvmjOnljomQ0NkNvCXU3rtq6V3QYLj5Ccc3GrozHSbqaMNlZ0T+F7BHwGP4Btemy6P/TE3iRn4gRIy/wM35CafhkxIvD2CTPxuigFT2S0BXLwjN08QA2X4PBQOkABOomYODFiyJHudBg9cTIQ3Cfi0isWYYcyhBWiATKBpj9PEz1/OU/w671/wptAx9KFpm3HRke2YHIM0VedD7NXnRD8sxY1zP4XaMHkTXyFoXYMgAcczFaAGWtO6ddd9N5CbimY+S6AbqOcqjSYgU8z0YOv3035WZXtU/J8wgmB8gd4yPywsGWXyExfQUugMgz4AXTWXz7DuvmpyTMIEw61ENwoEM7F8sIyIJgN1sw9C4UDY3DbhZLMHZmM95ybUXineFUUdjcKmeV6jgRHBvyTnHgxVzGst45+ZibGvJkGqczZgodWFDad5lGtIQwQ4mYAv45elWtVyN9mi1YQhq9z+b/oi1gypVImLmOuTZHF1dbnSdF4aJfrH8KnuXnPMsTSKrsNvsKLZLqbCF0z/JpecZtieaCEyCGL1nWYkyLohnedjoWUiQ0XsqWdjMg4tg+A9wA+Zrll5dY1Q7qmgogji/OK1mhqNBzNPrhcjiC5m/8rv+f62Fo3WZZ5gCMwVWOsXYs8FAMIOaLNw7DFXkFU+JcKy4j61QVRQV7ZyuPLTNQ/ObcHFYghGcHkDngfQ+8VcKOVir2JV/LNNF8xRqUfEHYs+DAugdA/Le/4FGwmdJT3e+s+zeD92RoY/g3lP8RbMJhmSms5dDg2sb7K246oc3MlkT0PBiTVV6P3C0F2G27T4sFSJ2CiCUJaEOwGXlBSnew9LtODXoNyfit3SJmDYwlFdsavpzSeJpCLwjWMQkcoLjZnUPzL0zTbHHuKKy6UHe4XAoDNkkVNiwKu/A1hfnDvhK6f0gH3KeqhzLg1tweToBbTihz12mpiTO4GKitoi8SUG2RAw3VE0Blh1okoK0xSzS04tjeNaAgJ/i4vqPEQRbxuMQm7rVhIo9UchCnvAvriZkaMdPzd4Nr8F2ZlEzfnDNT3jUXwbvdh5YdDPT+8uqxleejHNZ8eCiPbT9qqqO6ASnVXJEBKlHCs0KwHt2OlMb2dx+TmKpmGxAj2d4eBOGOcXMkExpnF17ubU42mhapIcITH+zsVJzz2u1SHkgVEKlf3vZHLWxTyjAH71RKMC1sVioLqHa/UihMb8tSgdMuYZRH9xlDQiU1Ywwj2c7mxaEde6h+lZd4wih4ADWKJ0/0sIceZOE3kt30sJMPytnQuBSH/avXWSmWi7AM6GqnkKJe7VSkJLTaNbm6t88uD+UXynyum9UJCra3sAKa+9NoNUXxsvbTEnpgjYDvfugK6qWS0Xz4AuqjoObD108PxWg+bvmsqpLa1XNTex4BbKO0vnZuxva//WvGCE6wJUawaO7PfkZw4mXtJ0Y4hBGs734oI1gTPtiL7w7jgzLGYWxQwnj/SC6oqY9qLihVnffV29/NBaWx/e5ASzoFhzVqEJxsez2CBdy829khOOEt9Sc2eGB/YN32ozfZN8P+xeWwhW22D1TYaB9tsjZ/3kow+5v4kt4yQl01Z75w7XuRVk4e0yzUVExtv1CqRq9lqMRqvWsoWeA3Dgp/AWn4q6GTba99sICbd42aCKe0BfPUShzYSlj3/ehuwlrx6IaiDHNwT1GCaaGtqCmd2s6iVJZec1GJ1Xp/UbLAJw0R3zbqMDayLZKGAdx8PZ9fasYembb3Jf8+lNNdKPuUn74E3OKgjRcfTDxtko6x4PGkU4I5nHSKMG2QTnXp1ZNOsax90qnCap90ihb4+5rGrLORbW9f4yo+f9foG1CPJvYpP7Uq+/Y5BzJGm6wBsRpdv/3Qyj7HBzr4B0UPqIWfFB/EHX5xetuc/wN3+BY47th6Etp/kp2SGZ/iaQI80CHmthKNIQnDIyhuPjyZi31HDSpONQR5tnslVUVeewrg+Sg/LLI5HmKerMn3vwY8f7xmewwbK6bwfAhE6Pvm40f7wKoWIv7CtZEOP2cPwpZOJWQaUbqeUc064wYPm3drtJo8tVqnu+9Rszq9vQ+m7FDc/fv0DsWd3867LCkFwiuVz+Tf5+QfJ6cnJy5Nqp25d5ga8/Z9Ybgjfvu+Pmg+uZ/3TG7f6r9jfvtVC0RiIBx9rCisj6Y0tbw/y1fdL+y+l53GSIgSUrNZZ5tqIqCylep08+XSHGvrhM/UGXmm8PRKZ4Nk7HfHTQv1j6e63PKo7lVkj8NFeLiTdcJPyeVweD08g1L+lBTOkigtOwDYzdWAGDQeewkCiMV4jMdVxmNyfk7C8RjnOB4bNrbTDf4CEAHDYg==
+\ No newline at end of file
++eJztW21v2zgS/q5fQSgoKl8cJSlugUMOOcCbpqjv2iSws10smsKQbdrmVhZ9pJQ0KPrfb4akZIqW5NfbfkmA1rY083A4L4+GknhErvjiWbDpLCVvzt6ckfsZJfc0kVy8i/kT6WTpjAsZkk4ckx6KSdKjkopHOg69I++IfGAjEKdjkiVjKkgK+p1FNIIPc6ZNPlEhGU/Im/CMBCjgm1N+65+A8MwzMo+eScJTkkkKEEySCYspod9GdJESlpARny9iFiUjSp5YOlPDGBAwg/xhIPgwjUA6AvkF/JrYciRKlcH4N0vTxcXp6dPTUxgpY0MupqexFpSnH7pX1zf96xMwWKn8lsRUSiLofzMmYKrDZxItwJ5RNAQr4+iJcEGiqaBwLuVo75NgKUumbSL5JH2KBAWUMZOpYMMsLTkrtw7mbAuAu6KE+J0+6fZ98mun3+23AeP37v3729/uye+dXq9zc9+97pPbHrm6vXnbve/e3sCvd6Rz8wf5T/fmbZtQcBUMQ78tBNoPRjJ0owod6VNaMmDCtUFyQUdswkYwr2SaRVNKpvyRigSmQxZUzJnEYEowbwwoMZuzNErVkZVJ4TCXB/3zfN+/EyzBNLy9msPwQxGJZzSGzGiE448hRKOUC0aVjeRRZx+kFAcD0bFqls8ypfPQ8zDh5UgwyDNJIwG5IJUr6uAxMWUZpQ0RR6+l0oODc0yBMU3RVYlyMRO5EQpooe1H/RFPJmyaCeVA1JPpmGdpqKxaRJjoPAfHDDGxwTSbCZ5NZ5gkNHlkgidzmqTkMRJMJWUA9n8c3HXu37dCrzuB4oJzMRs7QzLjlraejvZDbqAyhwqhQi1omgkVdgKHwEEjPqZl/6XRV6rnlcfg2bIYigZPFXZV2h3aeDHnX3UwtO91PPOY6ECoap9FYnyC9owhhinUvSezoZ0HE8HnZBhJ41RDDEvbCntDAr5amgjuAVbyCkHlJijLU75ITwUfzX0UyZD+IrAlhbhPoizG+cQZ9TBbPQ9qjgsIH8+/cZl/A14w3yCTPM8bxRHU6ZUK0TV6ObhWFAihal14BKyXKAajkMGUpgMz3ABNGeDUAiWmY2WbaSspYUiqNIpjSwlsfZtnrfZ0HnITtjkxSjp1UDPECcJ4LiK5rLUPxNmE+IWPfQwhl6HJBrSkGnAp89nS/gLyR3XyykRBo1gNvSLUWrpq5ZztMLz4cMlSOjBlPEiy+ZCKYB79yUWbgMfwA9RGM9v/52fwR/5GlBg5xt/4C6XhlxK3h9FJno8RoBVtkkRzmoenZ+IBbL4Ag4HSAQjUVcDAi1c2R5nQYPXEyENwnvGQL2iO7AsfrhAJlA0w+6WfpZOTf/gt7f852gY+FDRUXwPhH+mByCtJjoOH8XHLJ6+UdW2F31J6EFklr1GILgPAUQfDKVDWIjhvmZPGS8A1gZJreei6iEGV2hXwOh/Z//5DlZu+qj0kr0OYHCAHykfk2MCW/3yi+gq8ACLPgBdUZ/H9B1w3HxI/h1DpUA/BgA71XDQjIAuC3XRK0btQNFHst/JYgrFjnfGaaysS7wKnisLqVDmrZGBEikIYcUFXqyCEnAleq0q4uu1dq3J4rX0L2SoZsGmBrLnyknwuJugAHwNZsGQUZ2OqCOMET53a1oUzv72ptq2D5anI45fwPL/8caGQvqj/bRvBxBueUHUcM3eCAaueTZFi1kG39P/kLM/2SZ51kKL5afoNGi4ZrCC0LoqJOsatiBaCQ6CZr3kNYIbYojneanJbCecrL+WNghoQcXTXAm6A7C+ytdIjLVVAxNDNZSWp2Aptw8Kfrnt96B0HHzv/vu352k+apHbA6N4UGAtDIttiQCJfvTcYhiMqiBbnWnEYSauqpirI3yvyGOMBit9NNPwKBP9ih2sB4P3wnIuMHq3EFTO2EFmSsjndgDEsYccCK3VBqI4w3nfvtqSKAswucziI/yrZoaSAgg1KL/TQRA8+RIv0dLg3owaLUJpZ4giW+nAxszoGaKN1e/8Nl7bQzOYXXnQ1po0p0DZ5mnEwVPe4GguQAktEcwk0O9jyHJPSGWSIllGDjkZQ9qgXonmbpLlHN6AnoygeZdBxgnVUAFVIpu4BwBKDq9Zc4zxFcG2H9MaLMldgw0xiWySx119EMH9YvcIaA+KPq2G5LVGuzG17nlxxQpnizkutosLFQK1wg81TtVwAbFXPE5V9sM1TK2OW2GrOsIncgKmMYEN3YyRqmOpj9/bu+mZLsrIhbb7Sx0+r6Mqvl7HY6Y3DTkrTaGwE8EJvTd0PRht8VyY51fEXTFf0+zZ4q7VtKZu02rOaXZTd+h4HZd/Op6biqnufUh3brFKJ4l9Ywdq7EyqN7a6bhnEkN1s6Kcnm1ZMSqV87/fqh099+5bQEdVY+eOIUF4YiieL8wEkj6WyiZdPQEmGpqZE2RHjhocZVmHHe1t1WffZsT1F5Yu6/MivD7Lw4K8EcYH1WWbi1SzSLEJxVWgXOYYmqPLrLVAIqaTOmUpL167Vcop6oep2bt9sT1RLUISo8cerYVctPdcI2qZRoSSk4ADWKZy9stIaNhPUQa8fbQrbVOvk2rnyVd6byyzVfBjSlalWEU6oVFQCrlJrSWLtEKQ/l1uVkkm5WlijY3D+gRH1Vvnt3v31RFpBOTcJx87G2Y6gWrW0TULys/dIb7FiN4LsDFOMWvYCTpUpz+1bARUHN7TsBB0Vp7tcIVBVgbR+wLGuHW1ZRDt4FLMd2b91uRjZGsIFsjET9bdvtycaGdG7eIiXoj2ayqRWtJhsjXtZ+IZtdyEb77q8lGz3mJ33w425UU8bYjWhKGHd70kxN6VXTTKmgnXut/2+aKY3t9jSpiEbgsI3aGiPb3NkYofrm5r7Xubrubd/f2MBOi6NPLb/VNjhVgrXtjRFeUX/hm7XPlwpfHe7p0paUs0y0/e92uEDW/Y6jZWYWLwpyql/mmEWPlESGL3I/mGWNTVxn+3Q6NTVZ2+yU6t3pdyqxDt7ylCxwux6JD9U2fF5tZJt7HyNU3/707zq9/ravuTjAThOkTy2/rW2FmhRqGyKjtALz0hZtTFOFz34aTRUW7N0clWF27o9KMAdokWqKtLZLKhGA0yhVYh28VypZ4NITjx836paWsmvoSQs10NPth09bd0sOsEtP6tTyuU5xaD1PbaJZT1i5tvNcaR3KeRPKOuWXG8HVtKd89lNpT1mwP+2VYHanPRvmELRXXfz1tGcTi0t7VViHpz3bAneVuDHvLWWbV4nNvAe1uxPvOcDOKtEwR/Ft7e3wJoXaVaPDT+uUX9qyytXjT+anZQLuv3p0gXZ+Wu4AHeB5+VYs5dKAs3j8C1jKtcCw1MrGCHdjS0TGbISbi3B/F5/o0lOGJBR3pJn5sGTC1+08qtjk5BXp7dRQFU1WZ3zhpNf9Yu/YcreYel2tuKugwIt31lbH0LGiEreLQYR+LH9+1i+gp5zHX5m+fe9/yV9sL21SyjXCbDGOUhoMNth70qrR2uQt9Drdde+E1umtfdurQbH55YsGxcYHNiZLSoFwSuUL+dcl+fvZ+dmZSZNqZ64dpsa8dTd6G+K37qbM5pP7Zc3k1vUZDfNbr2oRiYIw9DGP4IKoSjMVzxfFZfYrfW7nm7MSIrlI6ThYpZoQqGwug1ZxfVS7XAP/lbwgryRuZguWSMp+s/vcqn/c5Gmuh/JZhnp3bIh7vWngPyTXvd5t7wJK+SGxtpbJVAQA2CrUgBhS3AXneRCLwQB3rw0G5PKS+IMBznEwUGysp+v9DwcFMAM=
+\ No newline at end of file
+diff --git a/third_party/gpus/rocm_configure.bzl b/third_party/gpus/rocm_configure.bzl
+index 30760f4f..336c1e96 100644
+--- a/third_party/gpus/rocm_configure.bzl
++++ b/third_party/gpus/rocm_configure.bzl
+@@ -29,7 +29,6 @@ load(
+     "which",
+ )
+
+-
+ _GCC_HOST_COMPILER_PATH = "GCC_HOST_COMPILER_PATH"
+ _GCC_HOST_COMPILER_PREFIX = "GCC_HOST_COMPILER_PREFIX"
+ _ROCM_TOOLKIT_PATH = "ROCM_PATH"
+@@ -175,17 +174,23 @@ def _rocm_include_path(repository_ctx, rocm_config, bash_bin):
+     """
+     inc_dirs = []
+
++    hsa_path = get_host_environ(repository_ctx, "HSA_ROCR_DEV_PATH")
++    hip_path = get_host_environ(repository_ctx, "HIP_PATH")
++    rocprim_path = get_host_environ(repository_ctx, "ROCPRIM_PATH")
++    rocsolver_path = get_host_environ(repository_ctx, "ROCSOLVER_PATH")
++    rocblas_path = get_host_environ(repository_ctx, "ROCBLAS_PATH")
++    llvm_path = get_host_environ(repository_ctx, "LLVM_PATH")
+     # Add HSA headers (needs to match $HSA_PATH)
+-    inc_dirs.append(rocm_config.rocm_toolkit_path + "/hsa/include")
++    inc_dirs.append(hsa_path + "/include/hsa")
+
+     # Add HIP headers (needs to match $HIP_PATH)
+     inc_dirs.append(rocm_config.rocm_toolkit_path + "/hip/include")
+     if int(rocm_config.rocm_version_number) >= 50200:
+         inc_dirs.append(rocm_config.rocm_toolkit_path + "/include")
+-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/hip")
+-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/rocprim")
+-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/rocsolver")
+-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/rocblas")
++        inc_dirs.append(hip_path + "/include/hip")
++        inc_dirs.append(rocprim_path + "/include/rocprim")
++        inc_dirs.append(rocsolver_path + "/include/rocsolver")
++        inc_dirs.append(rocblas_path + "/include/rocblas")
+
+     # Add HIP-Clang headers (realpath relative to compiler binary)
+     rocm_toolkit_path = realpath(repository_ctx, rocm_config.rocm_toolkit_path, bash_bin)
+@@ -198,7 +203,7 @@ def _rocm_include_path(repository_ctx, rocm_config, bash_bin):
+     inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/14.0.0/include")
+     inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/15.0.0/include")
+     inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/16.0.0/include")
+-    inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/17.0.0/include")
++    inc_dirs.append(llvm_path + "/lib/clang/17.0.0/include/")
+     inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/17/include")
+     inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/18/include")
+
+@@ -224,7 +229,8 @@ def _amdgpu_targets(repository_ctx, rocm_toolkit_path, bash_bin):
+     """Returns a list of strings representing AMDGPU targets."""
+     amdgpu_targets_str = get_host_environ(repository_ctx, _TF_ROCM_AMDGPU_TARGETS)
+     if not amdgpu_targets_str:
+-        cmd = "%s/bin/rocm_agent_enumerator" % rocm_toolkit_path
++        rocminfo_path = get_host_environ(repository_ctx, "ROCMINFO_PATH")
++        cmd = rocminfo_path + "/bin/rocm_agent_enumerator"
+         result = execute(repository_ctx, [bash_bin, "-c", cmd])
+         targets = [target for target in result.stdout.strip().split("\n") if target != "gfx000"]
+         targets = {x: None for x in targets}
+@@ -345,23 +351,33 @@ def _find_libs(repository_ctx, rocm_config, hipfft_or_rocfft, miopen_path, rccl_
+     Returns:
+       Map of library names to structs of filename and path
+     """
++    hip_path = get_host_environ(repository_ctx, "HIP_PATH")
++    rocblas_path = get_host_environ(repository_ctx, "ROCBLAS_PATH")
++    hipfft_path = get_host_environ(repository_ctx, "HIPFFT_PATH")
++    hiprand_path = get_host_environ(repository_ctx, "HIPRAND_PATH")
++    hipsparse_path = get_host_environ(repository_ctx, "HIPSPARSE_PATH")
++    roctracer_path = get_host_environ(repository_ctx, "ROCTRACER_PATH")
++    rocsolver_path = get_host_environ(repository_ctx, "ROCSOLVER_PATH")
++    hipsolver_path = get_host_environ(repository_ctx, "HIPSOLVER_PATH")
++    hipblas_path = get_host_environ(repository_ctx, "HIPBLAS_PATH")
++
+     libs_paths = [
+         (name, _rocm_lib_paths(repository_ctx, name, path))
+         for name, path in [
+-            ("amdhip64", rocm_config.rocm_toolkit_path),
+-            ("rocblas", rocm_config.rocm_toolkit_path),
+-            (hipfft_or_rocfft, rocm_config.rocm_toolkit_path),
+-            ("hiprand", rocm_config.rocm_toolkit_path),
+-            ("MIOpen", miopen_path),
++            ("amdhip64", hip_path + "/lib"),
++            ("rocblas", rocblas_path),
++            (hipfft_or_rocfft, hipfft_path),
++            ("hiprand", hiprand_path),
++           ("MIOpen", miopen_path),
+             ("rccl", rccl_path),
+-            ("hipsparse", rocm_config.rocm_toolkit_path),
+-            ("roctracer64", rocm_config.rocm_toolkit_path),
+-            ("rocsolver", rocm_config.rocm_toolkit_path),
++            ("hipsparse", hipsparse_path),
++            ("roctracer64", roctracer_path + "/lib"),
++            ("rocsolver", rocsolver_path),
+         ]
+     ]
+     if int(rocm_config.rocm_version_number) >= 40500:
+-        libs_paths.append(("hipsolver", _rocm_lib_paths(repository_ctx, "hipsolver", rocm_config.rocm_toolkit_path)))
+-        libs_paths.append(("hipblas", _rocm_lib_paths(repository_ctx, "hipblas", rocm_config.rocm_toolkit_path)))
++        libs_paths.append(("hipsolver", _rocm_lib_paths(repository_ctx, "hipsolver", hipsolver_path)))
++        libs_paths.append(("hipblas", _rocm_lib_paths(repository_ctx, "hipblas", hipblas_path)))
+
+     # hipblaslt may be absent even in versions of ROCm where it exists
+     # (it is not installed by default in some containers). Autodetect.
+@@ -578,12 +594,13 @@ def _create_local_rocm_repository(repository_ctx):
+     hipfft_or_rocfft = "rocfft" if rocm_version_number < 40100 else "hipfft"
+
+     # For ROCm 5.2 and above, find MIOpen and RCCL in the main rocm lib path
+-    miopen_path = rocm_config.rocm_toolkit_path + "/miopen" if rocm_version_number < 50200 else rocm_config.rocm_toolkit_path
+-    rccl_path = rocm_config.rocm_toolkit_path + "/rccl" if rocm_version_number < 50200 else rocm_config.rocm_toolkit_path
++    miopen_path = get_host_environ(repository_ctx, "MIOPEN_PATH") + "/miopen"
++    rccl_path = get_host_environ(repository_ctx, "RCCL_PATH") + "/rccl"
+
+     # Copy header and library files to execroot.
+     # rocm_toolkit_path
+     rocm_toolkit_path = rocm_config.rocm_toolkit_path
++    rocblas_path = get_host_environ(repository_ctx, "ROCBLAS_PATH")
+     copy_rules = [
+         make_copy_dir_rule(
+             repository_ctx,
+@@ -597,6 +614,53 @@ def _create_local_rocm_repository(repository_ctx):
+     # explicitly copy (into the local_config_rocm repo) the $ROCM_PATH/hiprand/include and
+     # $ROCM_PATH/rocrand/include dirs, only once the softlink to them in $ROCM_PATH/include
+     # dir has been removed. This removal will happen in a near-future ROCm release.
++    rocm_parts = [
++        "rocblas",
++       "rocprim",
++        "hipfft",
++        "hipsparse",
++        "rocsolver",
++        "hipsolver",
++        "hipblas",
++       "rccl",
++       "miopen",
++       "roctracer",
++    ]
++    rocm_includes = ""
++
++    for part in rocm_parts:
++        part_cap = part.upper().replace("-","_")
++        part_path = get_host_environ(repository_ctx, part_cap + "_PATH")
++        rocm_includes = rocm_includes + '":' + part + '-include",\n'
++        rocm_includes = rocm_includes + '":' + part + '-include-second",\n'
++        copy_rules.append(
++            make_copy_dir_rule(
++                repository_ctx,
++                name = part + "-include",
++                src_dir = part_path + "/include/" + part,
++                out_dir = "rocm/include",
++            ),
++        )
++        copy_rules.append(
++            make_copy_dir_rule(
++                repository_ctx,
++                name = part + "-include-second",
++                src_dir = part_path + "/include/" + part,
++                out_dir = "rocm/include/" + part,
++            ),
++        )
++    hipcub_path = get_host_environ(repository_ctx, "HIPCUB_PATH")
++    copy_rules.append(
++        make_copy_dir_rule(
++            repository_ctx,
++            name = "hipcub-include",
++            src_dir = hipcub_path + "/include/hipcub/",
++            out_dir = "rocm/include/hipcub/",
++        )
++    )
++    rocm_includes = rocm_includes + '":hipcub-include",\n'
++
++    hiprand_path = get_host_environ(repository_ctx, "HIPRAND_PATH")
+     hiprand_include = ""
+     hiprand_include_softlink = rocm_config.rocm_toolkit_path + "/include/hiprand"
+     softlink_exists = files_exist(repository_ctx, [hiprand_include_softlink], bash_bin)
+@@ -606,11 +670,12 @@ def _create_local_rocm_repository(repository_ctx):
+             make_copy_dir_rule(
+                 repository_ctx,
+                 name = "hiprand-include",
+-                src_dir = rocm_toolkit_path + "/hiprand/include",
++                src_dir = hiprand_path + "/include/hiprand",
+                 out_dir = "rocm/include/hiprand",
+             ),
+         )
+
++    rocrand_path = get_host_environ(repository_ctx, "ROCRAND_PATH")
+     rocrand_include = ""
+     rocrand_include_softlink = rocm_config.rocm_toolkit_path + "/include/rocrand"
+     softlink_exists = files_exist(repository_ctx, [rocrand_include_softlink], bash_bin)
+@@ -620,11 +685,10 @@ def _create_local_rocm_repository(repository_ctx):
+             make_copy_dir_rule(
+                 repository_ctx,
+                 name = "rocrand-include",
+-                src_dir = rocm_toolkit_path + "/rocrand/include",
++                src_dir = rocrand_path + "/include/rocrand",
+                 out_dir = "rocm/include/rocrand",
+             ),
+         )
+-
+     rocm_libs = _find_libs(repository_ctx, rocm_config, hipfft_or_rocfft, miopen_path, rccl_path, bash_bin)
+     rocm_lib_srcs = []
+     rocm_lib_outs = []
+@@ -639,7 +703,8 @@ def _create_local_rocm_repository(repository_ctx):
+         outs = rocm_lib_outs,
+     ))
+
+-    clang_offload_bundler_path = rocm_toolkit_path + "/llvm/bin/clang-offload-bundler"
++    llvm_amdgpu_path = get_host_environ(repository_ctx, "LLVM_PATH")
++    clang_offload_bundler_path = llvm_amdgpu_path + "/bin/clang-offload-bundler"
+
+     # copy files mentioned in third_party/gpus/rocm/BUILD
+     copy_rules.append(make_copy_files_rule(
+@@ -683,9 +748,7 @@ def _create_local_rocm_repository(repository_ctx):
+         "%{roctracer_lib}": rocm_libs["roctracer64"].file_name,
+         "%{rocsolver_lib}": rocm_libs["rocsolver"].file_name,
+         "%{copy_rules}": "\n".join(copy_rules),
+-        "%{rocm_headers}": ('":rocm-include",\n' +
+-                            hiprand_include +
+-                            rocrand_include),
++        "%{rocm_headers}": ('":rocm-include",\n' + rocrand_include + hiprand_include + rocm_includes),
+     }
+     if rocm_libs["hipblaslt"] != None:
+         repository_dict["%{hipblaslt_lib}"] = rocm_libs["hipblaslt"].file_name
+@@ -749,16 +812,18 @@ def _create_local_rocm_repository(repository_ctx):
+         tpl_paths["crosstool:hipcc_cc_toolchain_config.bzl"],
+     )
+
++    hip_path = get_host_environ(repository_ctx, "HIP_PATH")
++    hsa_rocr_dev_path = get_host_environ(repository_ctx, "HSA_ROCR_DEV_PATH")
+     repository_ctx.template(
+         "crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc",
+         tpl_paths["crosstool:clang/bin/crosstool_wrapper_driver_rocm"],
+         {
+             "%{cpu_compiler}": str(cc),
+-            "%{hipcc_path}": rocm_config.rocm_toolkit_path + "/bin/hipcc",
++            "%{hipcc_path}": hip_path + "/bin/hipcc",
+             "%{hipcc_env}": _hipcc_env(repository_ctx),
+-            "%{rocr_runtime_path}": rocm_config.rocm_toolkit_path + "/lib",
++            "%{rocr_runtime_path}": hsa_rocr_dev_path + "/lib",
+             "%{rocr_runtime_library}": "hsa-runtime64",
+-            "%{hip_runtime_path}": rocm_config.rocm_toolkit_path + "/lib",
++            "%{hip_runtime_path}": hip_path + "/lib",
+             "%{hip_runtime_library}": "amdhip64",
+             "%{crosstool_verbose}": _crosstool_verbose(repository_ctx),
+             "%{gcc_host_compiler_path}": str(cc),
diff --git a/var/spack/repos/builtin/packages/py-tensorflow/package.py b/var/spack/repos/builtin/packages/py-tensorflow/package.py
index 6668507186e..1a3c3154a06 100644
--- a/var/spack/repos/builtin/packages/py-tensorflow/package.py
+++ b/var/spack/repos/builtin/packages/py-tensorflow/package.py
@@ -8,12 +8,36 @@
 
 from spack.package import *
 
+rocm_dependencies = [
+    "hip",
+    "rocrand",
+    "rocblas",
+    "rocfft",
+    "hipfft",
+    "rccl",
+    "hipsparse",
+    "rocprim",
+    "llvm-amdgpu",
+    "hsa-rocr-dev",
+    "rocminfo",
+    "hipsolver",
+    "hiprand",
+    "rocsolver",
+    "hipsolver",
+    "hipblas",
+    "hipcub",
+    "rocm-core",
+    "roctracer-dev",
+    "miopen-hip",
+]
+
 
 class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     """TensorFlow is an open source machine learning framework for everyone."""
 
     homepage = "https://www.tensorflow.org"
     url = "https://github.com/tensorflow/tensorflow/archive/v2.3.1.tar.gz"
+    git = "https://github.com/tensorflow/tensorflow.git"
 
     maintainers("adamjstewart", "aweits")
     import_modules = ["tensorflow"]
@@ -21,9 +45,19 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     license("Apache-2.0")
 
     version("2.16.1", sha256="c729e56efc945c6df08efe5c9f5b8b89329c7c91b8f40ad2bb3e13900bd4876d")
+    version(
+        "2.16-rocm-enhanced",
+        git="https://github.com/ROCm/tensorflow-upstream.git",
+        branch="r2.16-rocm-enhanced",
+    )
     version("2.15.1", sha256="f36416d831f06fe866e149c7cd752da410a11178b01ff5620e9f265511ed57cf")
     version("2.15.0", sha256="9cec5acb0ecf2d47b16891f8bc5bc6fbfdffe1700bdadc0d9ebe27ea34f0c220")
     version("2.14.1", sha256="6b31ed347ed7a03c45b906aa41628ac91c3db7c84cb816971400d470e58ba494")
+    version(
+        "2.14-rocm-enhanced",
+        git="https://github.com/ROCm/tensorflow-upstream.git",
+        branch="r2.14-rocm-enhanced-nohipblaslt-build",
+    )
     version("2.14.0", sha256="ce357fd0728f0d1b0831d1653f475591662ec5bca736a94ff789e6b1944df19f")
     version("2.13.1", sha256="89c07aebd4f41fbe0d08cc88aef00305542134f2f16d3b62918dc3c1182f33e2")
     version("2.13.0", sha256="e58c939079588623e6fa1d054aec2f90f95018266e0a970fd353a5244f5173dc")
@@ -286,20 +320,9 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     # depends_on('android-sdk', when='+android')
 
     with when("+rocm"):
-        depends_on("hip")
-        depends_on("rocrand")
-        depends_on("rocblas")
-        depends_on("rocfft")
-        depends_on("hipfft")
-        depends_on("rccl", when="+nccl")
-        depends_on("hipsparse")
-        depends_on("hipcub")
-        depends_on("rocsolver")
-        depends_on("rocprim")
-        depends_on("miopen-hip")
-        depends_on("llvm-amdgpu")
-        depends_on("hsa-rocr-dev")
-        depends_on("rocminfo")
+        for pkg_dep in rocm_dependencies:
+            depends_on(f"{pkg_dep}@6.0:", when="@2.14:")
+            depends_on(pkg_dep)
 
     # Check configure and configure.py to see when these variants are supported
     conflicts("+mkl", when="platform=darwin", msg="Darwin is not yet supported")
@@ -354,9 +377,11 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     conflicts("platform=darwin target=aarch64:", when="@:2.4")
     # https://github.com/tensorflow/tensorflow/pull/39225
     conflicts("target=aarch64:", when="@:2.2")
-    conflicts("~rocm", when="@2.7.4-rocm-enhanced,2.11.0-rocm-enhanced")
-    conflicts("+rocm", when="@:2.7.4-a,2.7.4.0:2.11.0-a,2.11.0.0:")
-
+    conflicts(
+        "~rocm",
+        when="@2.7.4-rocm-enhanced,2.11.0-rocm-enhanced,2.14-rocm-enhanced,2.16-rocm-enhanced",
+    )
+    conflicts("+rocm", when="@:2.7.4-a,2.7.4.0:2.11.0-a,2.11.0.0:2.14-a,2.14-z:2.16-a,2.16-z:")
     # wheel 0.40 upgrades vendored packaging, trips over tensorflow-io-gcs-filesystem identifier
     conflicts("^py-wheel@0.40:", when="@2.11:2.13")
 
@@ -403,6 +428,10 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension):
     # and https://github.com/abseil/abseil-cpp/issues/1665
     patch("absl_neon.patch", when="@2.16.1: target=aarch64:")
 
+    patch("Find_ROCm_Components_Individiually.2.16.patch", when="@2.16-rocm-enhanced +rocm")
+    patch("Find_ROCm_Components_Individiually.patch", when="@2.14-rocm-enhanced +rocm")
+    patch("set_jit_trueLT_false.patch", when="@2.14-rocm-enhanced: +rocm")
+
     phases = ["configure", "build", "install"]
 
     # https://www.tensorflow.org/install/source
@@ -512,6 +541,13 @@ def setup_build_environment(self, env):
         # Do you wish to build TensorFlow with ROCm support?
         if "+rocm" in spec:
             env.set("TF_NEED_ROCM", "1")
+            env.set("TF_HIPBLASLT", "0")
+            env.set("MIOPEN_PATH", spec["miopen-hip"].prefix)
+            env.set("ROCTRACER_PATH", spec["roctracer-dev"].prefix)
+            env.set("LLVM_PATH", spec["llvm-amdgpu"].prefix)
+            for pkg_dep in rocm_dependencies:
+                pkg_dep_cap = pkg_dep.upper().replace("-", "_")
+                env.set(f"{pkg_dep_cap}_PATH", spec[pkg_dep].prefix)
         else:
             env.set("TF_NEED_ROCM", "0")
 
@@ -704,6 +740,11 @@ def post_configure_fixes(self):
             with open(".tf_configure.bazelrc", mode="a") as f:
                 f.write('build --action_env LD_LIBRARY_PATH="' + slibs + '"')
 
+        if spec.satisfies("@2.16-rocm-enhanced +rocm"):
+            filter_file(
+                "/usr/lib/llvm-17/bin/clang", spec["llvm-amdgpu"].prefix.bin.clang, ".bazelrc"
+            )
+
         filter_file("build:opt --copt=-march=native", "", ".tf_configure.bazelrc")
         filter_file("build:opt --host_copt=-march=native", "", ".tf_configure.bazelrc")
 
diff --git a/var/spack/repos/builtin/packages/py-tensorflow/set_jit_trueLT_false.patch b/var/spack/repos/builtin/packages/py-tensorflow/set_jit_trueLT_false.patch
new file mode 100644
index 00000000000..ee58be0745d
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-tensorflow/set_jit_trueLT_false.patch
@@ -0,0 +1,22 @@
+diff --git a/tensorflow/core/kernels/mlir_generated/build_defs.bzl b/tensorflow/core/kernels/mlir_generated/build_defs.bzl
+index 1cc3d335577..327e6cda821 100644
+--- a/tensorflow/core/kernels/mlir_generated/build_defs.bzl
++++ b/tensorflow/core/kernels/mlir_generated/build_defs.bzl
+@@ -355,7 +355,7 @@ def _gen_kernel_library(
+                 extra_args = extra_args,
+                 host_triple = host_triple,
+                 gpu_archs = gpu_archs,
+-                jit = jit,
++                jit = True,
+                 max_supported_rank = max_supported_rank,
+                 mlir_op = "{op}_{name}_{platform}_{type}_{output_type}.mlir".format(
+                     op = op,
+@@ -366,7 +366,7 @@ def _gen_kernel_library(
+                 ),
+                 tile_size = typed_tile_size,
+                 unroll_factors = typed_unroll_factors,
+-                jit_i64_indexed_for_large_tensors = jit_i64_indexed_for_large_tensors,
++                jit_i64_indexed_for_large_tensors = False,
+             )
+ 
+             # We have to use a sh_test instead of build_test because it doesn't properly find the dependent targets.
diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index b87ee813055..aa666e37291 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -54,6 +54,7 @@ class Python(Package):
     version("3.11.2", sha256="2411c74bda5bbcfcddaf4531f66d1adc73f247f529aee981b029513aefdbf849")
     version("3.11.1", sha256="baed518e26b337d4d8105679caf68c5c32630d702614fc174e98cb95c46bdfa4")
     version("3.11.0", sha256="64424e96e2457abbac899b90f9530985b51eef2905951febd935f0e73414caeb")
+    version("3.10.14", sha256="cefea32d3be89c02436711c95a45c7f8e880105514b78680c14fe76f5709a0f6")
     version("3.10.13", sha256="698ec55234c1363bd813b460ed53b0f108877c7a133d48bde9a50a1eb57b7e65")
     version("3.10.12", sha256="a43cd383f3999a6f4a7db2062b2fc9594fefa73e175b3aedafa295a51a7bb65c")
     version("3.10.11", sha256="f3db31b668efa983508bd67b5712898aa4247899a346f2eb745734699ccd3859")
@@ -68,6 +69,7 @@ class Python(Package):
     version("3.10.2", sha256="3c0ede893011319f9b0a56b44953a3d52c7abf9657c23fb4bc9ced93b86e9c97")
     version("3.10.1", sha256="b76117670e7c5064344b9c138e141a377e686b9063f3a8a620ff674fa8ec90d3")
     version("3.10.0", sha256="c4e0cbad57c90690cb813fb4663ef670b4d0f587d8171e2c42bd4c9245bd2758")
+    version("3.9.19", sha256="f5f9ec8088abca9e399c3b62fd8ef31dbd2e1472c0ccb35070d4d136821aaf71")
     version("3.9.18", sha256="504ce8cfd59addc04c22f590377c6be454ae7406cb1ebf6f5a350149225a9354")
     version("3.9.17", sha256="8ead58f669f7e19d777c3556b62fae29a81d7f06a7122ff9bc57f7dd82d7e014")
     version("3.9.16", sha256="1ad539e9dbd2b42df714b69726e0693bc6b9d2d2c8e91c2e43204026605140c5")
@@ -87,6 +89,7 @@ class Python(Package):
     version("3.9.2", sha256="7899e8a6f7946748830d66739f2d8f2b30214dad956e56b9ba216b3de5581519")
     version("3.9.1", sha256="29cb91ba038346da0bd9ab84a0a55a845d872c341a4da6879f462e94c741f117")
     version("3.9.0", sha256="df796b2dc8ef085edae2597a41c1c0a63625ebd92487adaef2fed22b567873e8")
+    version("3.8.19", sha256="c7fa55a36e5c7a19ec37d8f90f60a2197548908c9ac8b31e7c0dbffdd470eeac")
     version("3.8.18", sha256="7c5df68bab1be81a52dea0cc2e2705ea00553b67107a301188383d7b57320b16")
     version("3.8.17", sha256="def428fa6cf61b66bcde72e3d9f7d07d33b2e4226f04f9d6fce8384c055113ae")
     version("3.8.16", sha256="71ca9d935637ed2feb59e90a368361dc91eca472a90acb1d344a2e8178ccaf10")
diff --git a/var/spack/repos/builtin/packages/rust/package.py b/var/spack/repos/builtin/packages/rust/package.py
index b9f8da825f1..9676d8a033b 100644
--- a/var/spack/repos/builtin/packages/rust/package.py
+++ b/var/spack/repos/builtin/packages/rust/package.py
@@ -75,12 +75,14 @@ class Rust(Package):
     depends_on("rust-bootstrap@nightly", type="build", when="@nightly")
 
     # Stable version dependencies
+    depends_on("rust-bootstrap", type="build")
     depends_on("rust-bootstrap@1.59:1.60", type="build", when="@1.60")
     depends_on("rust-bootstrap@1.64:1.65", type="build", when="@1.65")
     depends_on("rust-bootstrap@1.69:1.70", type="build", when="@1.70")
     depends_on("rust-bootstrap@1.72:1.73", type="build", when="@1.73")
     depends_on("rust-bootstrap@1.73:1.74", type="build", when="@1.74")
     depends_on("rust-bootstrap@1.74:1.75", type="build", when="@1.75")
+    depends_on("rust-bootstrap@1.77:1.78", type="build", when="@1.78")
 
     # src/llvm-project/llvm/cmake/modules/CheckCompilerVersion.cmake
     conflicts("%gcc@:7.3", when="@1.73:", msg="Host GCC version must be at least 7.4")
diff --git a/var/spack/repos/builtin/packages/vecgeom/package.py b/var/spack/repos/builtin/packages/vecgeom/package.py
index 48284544866..e04d29d4227 100644
--- a/var/spack/repos/builtin/packages/vecgeom/package.py
+++ b/var/spack/repos/builtin/packages/vecgeom/package.py
@@ -21,6 +21,11 @@ class Vecgeom(CMakePackage, CudaPackage):
     maintainers("drbenmorgan", "sethrj")
 
     version("master", branch="master")
+    version(
+        "1.2.8",
+        url="https://gitlab.cern.ch/VecGeom/VecGeom/uploads/db11697eb81d6f369e9ded1078de946b/VecGeom-v1.2.8.tar.gz",
+        sha256="769f59e8377f8268e253a9b2a3eee86868a9ebc1fa66c968b96e19c31440c12b",
+    )
     version(
         "1.2.7",
         url="https://gitlab.cern.ch/VecGeom/VecGeom/uploads/e4172cca4f6f731ef15e2780ecbb1645/VecGeom-v1.2.7.tar.gz",
diff --git a/var/spack/repos/builtin/packages/zsh/package.py b/var/spack/repos/builtin/packages/zsh/package.py
index 11ca3f8cce1..913c83a73e0 100644
--- a/var/spack/repos/builtin/packages/zsh/package.py
+++ b/var/spack/repos/builtin/packages/zsh/package.py
@@ -17,6 +17,8 @@ class Zsh(AutotoolsPackage):
 
     license("custom")
 
+    version("5.9", sha256="9b8d1ecedd5b5e81fbf1918e876752a7dd948e05c1a0dba10ab863842d45acd5")
+    version("5.8.1", sha256="b6973520bace600b4779200269b1e5d79e5f505ac4952058c11ad5bbf0dd9919")
     version("5.8", sha256="dcc4b54cc5565670a65581760261c163d720991f0d06486da61f8d839b52de27")
     version("5.7.1", sha256="7260292c2c1d483b2d50febfa5055176bd512b32a8833b116177bf5f01e77ee8")
     version("5.6.2", sha256="a50bd66c0557e8eca3b8fa24e85d0de533e775d7a22df042da90488623752e9e")