diff --git a/mlx/distributed/mpi/mpi.cpp b/mlx/distributed/mpi/mpi.cpp
index 77b346037..ed0d66b3f 100644
--- a/mlx/distributed/mpi/mpi.cpp
+++ b/mlx/distributed/mpi/mpi.cpp
@@ -69,6 +69,7 @@ struct MPIWrapper {
     if (version.find("Open MPI") == std::string::npos) {
       std::cerr << "[mpi] MPI found but it does not appear to be Open MPI."
                 << "MLX requires Open MPI but this is " << version << std::endl;
+      libmpi_handle_ = nullptr;
       return;
     }