diff --git a/mlx/backend/metal/fence.cpp b/mlx/backend/metal/fence.cpp
index 54ce8ea8d..949ce7e23 100644
--- a/mlx/backend/metal/fence.cpp
+++ b/mlx/backend/metal/fence.cpp
@@ -34,6 +34,12 @@ Fence::Fence(const Stream& stream) : stream_(stream) {
   }
 }
 
+Fence::~Fence() {
+  if (use_fast_) {
+    cpu_value()[0] = INT_MAX;
+  }
+}
+
 void Fence::wait_gpu(array& x) {
   gpu_count_++;
   auto& d = metal::device(stream_.device);
diff --git a/mlx/backend/metal/fence.h b/mlx/backend/metal/fence.h
index 28e04e643..7d53d469e 100644
--- a/mlx/backend/metal/fence.h
+++ b/mlx/backend/metal/fence.h
@@ -21,6 +21,7 @@ namespace mlx::core {
 class Fence {
  public:
   Fence(const Stream& stream);
+  ~Fence();
 
   void update_gpu(const array& x);
   void wait_gpu(array& x);