mirror of
https://github.com/ml-explore/mlx-examples.git
synced 2025-12-14 17:28:59 +08:00
Fix detokenizer space match for quote (#1072)
* fix + test * remove transformer flax/torch warning * format
This commit is contained in:
@@ -51,6 +51,9 @@ class TestTokenizers(unittest.TestCase):
|
||||
tokens = tokenizer.encode("3 3")
|
||||
check(tokens)
|
||||
|
||||
tokens = tokenizer.encode("import 'package:flutter/material.dart';")
|
||||
check(tokens)
|
||||
|
||||
def test_tokenizers(self):
|
||||
tokenizer_repos = [
|
||||
("mlx-community/Qwen1.5-0.5B-Chat-4bit", BPEStreamingDetokenizer),
|
||||
|
||||
Reference in New Issue
Block a user