mirror of
https://github.com/ml-explore/mlx-examples.git
synced 2025-09-01 12:49:50 +08:00
Fix detokenizer space match for quote (#1072)
* fix + test * remove transformer flax/torch warning * format
This commit is contained in:
@@ -51,6 +51,9 @@ class TestTokenizers(unittest.TestCase):
|
||||
tokens = tokenizer.encode("3 3")
|
||||
check(tokens)
|
||||
|
||||
tokens = tokenizer.encode("import 'package:flutter/material.dart';")
|
||||
check(tokens)
|
||||
|
||||
def test_tokenizers(self):
|
||||
tokenizer_repos = [
|
||||
("mlx-community/Qwen1.5-0.5B-Chat-4bit", BPEStreamingDetokenizer),
|
||||
|
Reference in New Issue
Block a user