diff --git a/llms/mlx_lm/tokenizer_utils.py b/llms/mlx_lm/tokenizer_utils.py index 78ec2ff8..0cbc3b9b 100644 --- a/llms/mlx_lm/tokenizer_utils.py +++ b/llms/mlx_lm/tokenizer_utils.py @@ -169,7 +169,7 @@ class BPEStreamingDetokenizer(StreamingDetokenizer): """ _byte_decoder = None - _space_matches = (".", "?", "!", ",", "'", "n't", "'m", "'s", "'ve", "'re") + _space_matches = (".", "?", "!", ",", "n't", "'m", "'s", "'ve", "'re") def __init__(self, tokenizer): diff --git a/llms/tests/test_tokenizers.py b/llms/tests/test_tokenizers.py index 7b4828b1..03445c1f 100644 --- a/llms/tests/test_tokenizers.py +++ b/llms/tests/test_tokenizers.py @@ -51,6 +51,9 @@ class TestTokenizers(unittest.TestCase): tokens = tokenizer.encode("3 3") check(tokens) + tokens = tokenizer.encode("import 'package:flutter/material.dart';") + check(tokens) + def test_tokenizers(self): tokenizer_repos = [ ("mlx-community/Qwen1.5-0.5B-Chat-4bit", BPEStreamingDetokenizer),