From 743763bc2e4113dcf3c478058623cb02e770a237 Mon Sep 17 00:00:00 2001 From: aronson Date: Sun, 20 Oct 2024 22:46:43 -0500 Subject: [PATCH] Handle empty string case in maybe_trim_space (#1055) * Handle empty string case in maybe_trim_space * nit --------- Co-authored-by: Awni Hannun --- llms/mlx_lm/tokenizer_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llms/mlx_lm/tokenizer_utils.py b/llms/mlx_lm/tokenizer_utils.py index d8694d86..78ec2ff8 100644 --- a/llms/mlx_lm/tokenizer_utils.py +++ b/llms/mlx_lm/tokenizer_utils.py @@ -193,7 +193,9 @@ class BPEStreamingDetokenizer(StreamingDetokenizer): self.tokens = [] def _maybe_trim_space(self, current_text): - if current_text[0] != " ": + if len(current_text) == 0: + return current_text + elif current_text[0] != " ": return current_text elif not self.text: return current_text[1:]