zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-08-29 18:17:07 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

nit

Browse Source
This commit is contained in:
Awni Hannun 2024-12-17 09:10:26 -08:00
parent eb81e97963
commit 0fb0b6b4e6
1 changed files with 0 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
llms/mlx_lm/tokenizer_utils.py
View File

@ -198,9 +198,6 @@ class BPEStreamingDetokenizer(StreamingDetokenizer):
return current_text[1:] return current_text[1:]
return current_text return current_text
def _is_single_space(self, v):
return
def add_token(self, token): def add_token(self, token):
self.tokens.append(token) self.tokens.append(token)
v = self.tokenmap[token] v = self.tokenmap[token]