zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-06-24 09:21:18 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Set finish_reason in response (#592)

Browse Source
This commit is contained in:
Matt Wronkiewicz 2024-03-19 20:21:26 -07:00 committed by GitHub
parent 6c3d4c8ba2
commit 373dd6f2a2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
llms/mlx_lm/server.py
View File

@ -238,6 +238,7 @@ class APIHandler(BaseHTTPRequestHandler):
A list of stop words passed to the stopping_criteria function
"""
tokens = []
finish_reason = "length"
for (token, _), _ in zip(
generate_step(
prompt=prompt,
@ -255,12 +256,13 @@ class APIHandler(BaseHTTPRequestHandler):
tokens, stop_id_sequences, TOKENIZER.eos_token_id
)
if stop_condition.stop_met:
finish_reason = "stop"
if stop_condition.trim_length:
tokens = tokens[: -stop_condition.trim_length]
break
text = TOKENIZER.decode(tokens)
response = self.generate_response(text, "stop", len(prompt), len(tokens))
response = self.generate_response(text, finish_reason, len(prompt), len(tokens))
response_json = json.dumps(response).encode()