zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-06-27 11:21:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'ml-explore:main' into adding-support-for-mamba2

Browse Source
This commit is contained in:
Gökdeniz Gülmez 2024-10-21 15:14:19 +02:00 committed by GitHub
commit 0ef73f3a2d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
llms/mlx_lm/tokenizer_utils.py
View File

@ -193,7 +193,9 @@ class BPEStreamingDetokenizer(StreamingDetokenizer):
self.tokens = [] self.tokens = []
def _maybe_trim_space(self, current_text): def _maybe_trim_space(self, current_text):
if current_text[0] != " ": if len(current_text) == 0:
return current_text
elif current_text[0] != " ":
return current_text return current_text
elif not self.text: elif not self.text:
return current_text[1:] return current_text[1:]