From d8ca500136c98343494adc95593c3298e09c7f5c Mon Sep 17 00:00:00 2001 From: Isaac Aronson Date: Sat, 19 Oct 2024 17:46:48 -0500 Subject: [PATCH] Handle empty string case in maybe_trim_space --- llms/mlx_lm/tokenizer_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llms/mlx_lm/tokenizer_utils.py b/llms/mlx_lm/tokenizer_utils.py index d8694d86..4c4979b6 100644 --- a/llms/mlx_lm/tokenizer_utils.py +++ b/llms/mlx_lm/tokenizer_utils.py @@ -193,7 +193,9 @@ class BPEStreamingDetokenizer(StreamingDetokenizer): self.tokens = [] def _maybe_trim_space(self, current_text): - if current_text[0] != " ": + if len(current_text) < 1 or current_text is None: + return current_text + elif current_text[0] != " ": return current_text elif not self.text: return current_text[1:]