zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-08-29 18:26:37 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

nit

Browse Source
This commit is contained in:
Awni Hannun 2024-10-20 19:20:50 -07:00
parent d8ca500136
commit d7775e67a6
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
llms/mlx_lm/tokenizer_utils.py
View File

@ -193,7 +193,7 @@ class BPEStreamingDetokenizer(StreamingDetokenizer):
self.tokens = []
def _maybe_trim_space(self, current_text):
if len(current_text) < 1 or current_text is None:
if len(current_text) == 0:
return current_text
elif current_text[0] != " ":
return current_text