zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-12-15 01:42:31 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Handle empty string case in maybe_trim_space

Browse Source
This commit is contained in:
Isaac Aronson
2024-10-19 17:46:48 -05:00
parent f491d473a3
commit d8ca500136
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
llms/mlx_lm/tokenizer_utils.py
View File

@@ -193,7 +193,9 @@ class BPEStreamingDetokenizer(StreamingDetokenizer):
self.tokens = []
def _maybe_trim_space(self, current_text):
if current_text[0] != " ":
if len(current_text) < 1 or current_text is None:
return current_text
elif current_text[0] != " ":
return current_text
elif not self.text:
return current_text[1:]