diff --git a/mlx/backend/metal/allocator.cpp b/mlx/backend/metal/allocator.cpp
index f2c95be20..8f5b28226 100644
--- a/mlx/backend/metal/allocator.cpp
+++ b/mlx/backend/metal/allocator.cpp
@@ -20,6 +20,9 @@ Allocator& allocator() {
 }
 
 void* Buffer::raw_ptr() {
+  if (!ptr_) {
+    return nullptr;
+  }
   return static_cast<MTL::Buffer*>(ptr_)->contents();
 }
 
diff --git a/mlx/backend/no_metal/allocator.cpp b/mlx/backend/no_metal/allocator.cpp
index 27e2ea06f..0429ea53a 100644
--- a/mlx/backend/no_metal/allocator.cpp
+++ b/mlx/backend/no_metal/allocator.cpp
@@ -10,6 +10,9 @@ Allocator& allocator() {
 }
 
 void* Buffer::raw_ptr() {
+  if (!ptr_) {
+    return nullptr;
+  }
   return static_cast<size_t*>(ptr_) + 1;
 }