Fix detokenizer space match for quote (#1072)

* fix + test

* remove transformer flax/torch warning

* format
This commit is contained in:
Awni Hannun
2024-10-27 15:06:07 -07:00
committed by GitHub
parent ab4bf05c6e
commit 8fe9539af7
3 changed files with 9 additions and 1 deletions

View File

@@ -51,6 +51,9 @@ class TestTokenizers(unittest.TestCase):
tokens = tokenizer.encode("3 3")
check(tokens)
tokens = tokenizer.encode("import 'package:flutter/material.dart';")
check(tokens)
def test_tokenizers(self):
tokenizer_repos = [
("mlx-community/Qwen1.5-0.5B-Chat-4bit", BPEStreamingDetokenizer),