zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-08-30 02:53:41 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix + test

Browse Source
This commit is contained in:
Awni Hannun 2024-10-26 17:41:55 -07:00
parent ab4bf05c6e
commit 00b67e2f97
2 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
llms/mlx_lm/tokenizer_utils.py
View File

@ -169,7 +169,7 @@ class BPEStreamingDetokenizer(StreamingDetokenizer):
""" """
_byte_decoder = None _byte_decoder = None
_space_matches = (".", "?", "!", ",", "'", "n't", "'m", "'s", "'ve", "'re") _space_matches = (".", "?", "!", ",", "n't", "'m", "'s", "'ve", "'re")
def __init__(self, tokenizer): def __init__(self, tokenizer):

3
llms/tests/test_tokenizers.py
View File

@ -51,6 +51,9 @@ class TestTokenizers(unittest.TestCase):
tokens = tokenizer.encode("3 3") tokens = tokenizer.encode("3 3")
check(tokens) check(tokens)
tokens = tokenizer.encode("import 'package:flutter/material.dart';")
check(tokens)
def test_tokenizers(self): def test_tokenizers(self):
tokenizer_repos = [ tokenizer_repos = [
("mlx-community/Qwen1.5-0.5B-Chat-4bit", BPEStreamingDetokenizer), ("mlx-community/Qwen1.5-0.5B-Chat-4bit", BPEStreamingDetokenizer),