fix spm decoder multi-byte (#1092)

2025-09-01 12:49:50 +08:00 · 2024-11-05 06:06:26 -08:00
parent 4394633ce0
commit 6fd1f70f73
2 changed files with 20 additions and 23 deletions
--- a/llms/tests/test_tokenizers.py
+++ b/llms/tests/test_tokenizers.py
@@ -42,6 +42,9 @@ class TestTokenizers(unittest.TestCase):
            text += detokenizer.last_segment
            self.assertEqual(text, expected_text)

+        tokens = tokenizer.encode("こんにちは！私の名前はAI")
+        check(tokens)
+
        tokens = tokenizer.encode("a ,b")
        check(tokens)