zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-08-30 02:53:41 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix bug

Browse Source
This commit is contained in:
Angelos Katharopoulos 2024-12-12 11:08:23 -08:00
parent 6ef5ca4ce5
commit 62391bec30
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
llms/mlx_lm/tokenizer_utils.py
View File

@ -51,7 +51,7 @@ class StreamingDetokenizer:
text = self.text text = self.text
segment = text[self.offset :] segment = text[self.offset :]
self.offset = len(text) self.offset = len(text)
return text return segment
class NaiveStreamingDetokenizer(StreamingDetokenizer): class NaiveStreamingDetokenizer(StreamingDetokenizer):