diff --git a/llms/mlx_lm/tokenizer_utils.py b/llms/mlx_lm/tokenizer_utils.py index 9c5d3208..0eb961a3 100644 --- a/llms/mlx_lm/tokenizer_utils.py +++ b/llms/mlx_lm/tokenizer_utils.py @@ -158,9 +158,6 @@ class BPEStreamingDetokenizer(StreamingDetokenizer): _space_matches = (".", "?", "!", ",", "n't", "'m", "'s", "'ve", "'re") def __init__(self, tokenizer): - - self.tokenizer = tokenizer - self.clean_spaces = tokenizer.clean_up_tokenization_spaces # Extract the tokens in a list from id to text