From 9f10728145828fba08d797b43e77b5ee7e63729f Mon Sep 17 00:00:00 2001
From: Awni Hannun <awni@apple.com>
Date: Thu, 27 Jun 2024 06:38:19 -0700
Subject: [PATCH] fix yi (#852)

---
 llms/mlx_lm/tokenizer_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llms/mlx_lm/tokenizer_utils.py b/llms/mlx_lm/tokenizer_utils.py
index 7e251a09..6caad629 100644
--- a/llms/mlx_lm/tokenizer_utils.py
+++ b/llms/mlx_lm/tokenizer_utils.py
@@ -120,7 +120,7 @@ class SPMStreamingDetokenizer(StreamingDetokenizer):
         self.trim_space = trim_space
 
         # Extract the tokens in a list from id to text
-        self.tokenmap = [None] * len(tokenizer.vocab)
+        self.tokenmap = [""] * (max(tokenizer.vocab.values()) + 1)
         for value, tokenid in tokenizer.vocab.items():
             self.tokenmap[tokenid] = value