diff --git a/python/src/CMakeLists.txt b/python/src/CMakeLists.txt
index bcba3a8a5..a82c6bb6c 100644
--- a/python/src/CMakeLists.txt
+++ b/python/src/CMakeLists.txt
@@ -27,7 +27,11 @@ nanobind_add_module(
   ${CMAKE_CURRENT_SOURCE_DIR}/utils.cpp)
 
 if(NOT MLX_PYTHON_BINDINGS_OUTPUT_DIRECTORY)
-  set(MLX_PYTHON_BINDINGS_OUTPUT_DIRECTORY ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
+  if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY)
+    set(MLX_PYTHON_BINDINGS_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
+  else()
+    set(MLX_PYTHON_BINDINGS_OUTPUT_DIRECTORY ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
+  endif()
 endif()
 
 set_target_properties(