From 00b67e2f9737ae9fc9245952c0788c356b5d852d Mon Sep 17 00:00:00 2001 From: Awni Hannun Date: Sat, 26 Oct 2024 17:41:55 -0700 Subject: [PATCH] fix + test --- llms/mlx_lm/tokenizer_utils.py | 2 +- llms/tests/test_tokenizers.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/llms/mlx_lm/tokenizer_utils.py b/llms/mlx_lm/tokenizer_utils.py index 78ec2ff8..0cbc3b9b 100644 --- a/llms/mlx_lm/tokenizer_utils.py +++ b/llms/mlx_lm/tokenizer_utils.py @@ -169,7 +169,7 @@ class BPEStreamingDetokenizer(StreamingDetokenizer): """ _byte_decoder = None - _space_matches = (".", "?", "!", ",", "'", "n't", "'m", "'s", "'ve", "'re") + _space_matches = (".", "?", "!", ",", "n't", "'m", "'s", "'ve", "'re") def __init__(self, tokenizer): diff --git a/llms/tests/test_tokenizers.py b/llms/tests/test_tokenizers.py index 7b4828b1..03445c1f 100644 --- a/llms/tests/test_tokenizers.py +++ b/llms/tests/test_tokenizers.py @@ -51,6 +51,9 @@ class TestTokenizers(unittest.TestCase): tokens = tokenizer.encode("3 3") check(tokens) + tokens = tokenizer.encode("import 'package:flutter/material.dart';") + check(tokens) + def test_tokenizers(self): tokenizer_repos = [ ("mlx-community/Qwen1.5-0.5B-Chat-4bit", BPEStreamingDetokenizer),