From d7775e67a616ed9397bc68f3f1bc2859b6c73dc8 Mon Sep 17 00:00:00 2001 From: Awni Hannun Date: Sun, 20 Oct 2024 19:20:50 -0700 Subject: [PATCH] nit --- llms/mlx_lm/tokenizer_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llms/mlx_lm/tokenizer_utils.py b/llms/mlx_lm/tokenizer_utils.py index 4c4979b6..78ec2ff8 100644 --- a/llms/mlx_lm/tokenizer_utils.py +++ b/llms/mlx_lm/tokenizer_utils.py @@ -193,7 +193,7 @@ class BPEStreamingDetokenizer(StreamingDetokenizer): self.tokens = [] def _maybe_trim_space(self, current_text): - if len(current_text) < 1 or current_text is None: + if len(current_text) == 0: return current_text elif current_text[0] != " ": return current_text