diff --git a/python/mlx/optimizers.py b/python/mlx/optimizers.py
index 601d87b03..d737d5d86 100644
--- a/python/mlx/optimizers.py
+++ b/python/mlx/optimizers.py
@@ -423,6 +423,8 @@ class Adamax(Adam):
         self, learning_rate: float, betas: List[float] = [0.9, 0.999], eps: float = 1e-8
     ):
         super().__init__(learning_rate, betas, eps)
+        if not 0.0 <= eps:
+            raise ValueError(f"Epsilon value should be >=0, {self.eps} was provided instead")
 
     def apply_single(
         self, gradient: mx.array, parameter: mx.array, state: OptimizerState