fix yi (#852)

2025-09-01 04:14:38 +08:00 · 2024-06-27 06:38:19 -07:00
parent 7979b84a9e
commit 9f10728145
1 changed files with 1 additions and 1 deletions
--- a/llms/mlx_lm/tokenizer_utils.py
+++ b/llms/mlx_lm/tokenizer_utils.py
@@ -120,7 +120,7 @@ class SPMStreamingDetokenizer(StreamingDetokenizer):
        self.trim_space = trim_space

        # Extract the tokens in a list from id to text
-        self.tokenmap = [None] * len(tokenizer.vocab)
+        self.tokenmap = [""] * (max(tokenizer.vocab.values()) + 1)
        for value, tokenid in tokenizer.vocab.items():
            self.tokenmap[tokenid] = value