diff --git a/llms/mlx_lm/models/cohere2.py b/llms/mlx_lm/models/cohere2.py
index b1c9f2da..bc6ad276 100644
--- a/llms/mlx_lm/models/cohere2.py
+++ b/llms/mlx_lm/models/cohere2.py
@@ -157,7 +157,8 @@ class CohereModel(nn.Module):
         h = self.embed_tokens(inputs)
 
         if mask is None:
-            mask = create_attention_mask(h, cache, layer_idx=self.args.sliding_window_pattern - 1)
+            j = self.args.sliding_window_pattern
+            mask = create_attention_mask(h, cache[j - 1: j])
 
         if cache is None:
             cache = [None] * len(self.layers)