From 6ef5ca4ce59174764dbaf3b436b66e35f8d594fb Mon Sep 17 00:00:00 2001 From: Angelos Katharopoulos Date: Thu, 12 Dec 2024 10:37:29 -0800 Subject: [PATCH] Change the last_segment logic in tokenizer utils --- llms/mlx_lm/tokenizer_utils.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/llms/mlx_lm/tokenizer_utils.py b/llms/mlx_lm/tokenizer_utils.py index 36a32f9f..60cfb555 100644 --- a/llms/mlx_lm/tokenizer_utils.py +++ b/llms/mlx_lm/tokenizer_utils.py @@ -3,8 +3,6 @@ from functools import partial from transformers import AutoTokenizer -REPLACEMENT_CHAR = "\ufffd" - class StreamingDetokenizer: """The streaming detokenizer interface so that we can detokenize one token at a time. @@ -51,11 +49,9 @@ class StreamingDetokenizer: def last_segment(self): """Return the last segment of readable text since last time this property was accessed.""" text = self.text - if text and text[-1] != REPLACEMENT_CHAR: - segment = text[self.offset :] - self.offset = len(text) - return segment - return "" + segment = text[self.offset :] + self.offset = len(text) + return text class NaiveStreamingDetokenizer(StreamingDetokenizer):