zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-09-01 04:14:38 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix detokenizer space match for quote (#1072)

Browse Source 
* fix + test

* remove transformer flax/torch warning

* format
This commit is contained in:
Awni Hannun
2024-10-27 15:06:07 -07:00
committed by GitHub
parent ab4bf05c6e
commit 8fe9539af7
3 changed files with 9 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
llms/mlx_lm/__init__.py
View File

@@ -1,4 +1,9 @@
# Copyright © 2023-2024 Apple Inc.
import os
from ._version import __version__
os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"
from .utils import convert, generate, load, stream_generate

2
llms/mlx_lm/tokenizer_utils.py
View File

@@ -169,7 +169,7 @@ class BPEStreamingDetokenizer(StreamingDetokenizer):
"""
_byte_decoder = None
_space_matches = (".", "?", "!", ",", "'", "n't", "'m", "'s", "'ve", "'re")
_space_matches = (".", "?", "!", ",", "n't", "'m", "'s", "'ve", "'re")
def __init__(self, tokenizer):