some fixes for pipeline parallel deep seek r1 (#1216)

This commit is contained in:
Awni Hannun
2025-01-21 19:40:29 -08:00
committed by GitHub
parent df1406735b
commit 9a3ddc3e65
2 changed files with 14 additions and 6 deletions

View File

@@ -22,6 +22,11 @@ import mlx.core as mx
from mlx_lm import load, stream_generate
parser = argparse.ArgumentParser(description="LLM pipelined inference example")
parser.add_argument(
"--model",
default="mlx-community/DeepSeek-R1-3bit",
help="HF repo or path to local model.",
)
parser.add_argument(
"--prompt",
"-p",
@@ -37,9 +42,7 @@ parser.add_argument(
)
args = parser.parse_args()
model_repo = "mlx-community/DeepSeek-V3-3bit"
model, tokenizer = load(model_repo, lazy=True)
model, tokenizer = load(args.model, lazy=True)
messages = [{"role": "user", "content": args.prompt}]
prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True)