From 0532ba17fafd547143271847544aad78163eb8e9 Mon Sep 17 00:00:00 2001 From: "chenguangjian.jk" Date: Sat, 19 Oct 2024 16:23:49 +0800 Subject: [PATCH] mlx_lm.server --model mlx-community/Qwen2.5-7B-Instruct-8bit --trust-remote-code --port 8722 --- llms/mlx_lm/Makefile | 5 ++++- llms/mlx_lm/README.md | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/llms/mlx_lm/Makefile b/llms/mlx_lm/Makefile index a740268e..60f3f5bc 100644 --- a/llms/mlx_lm/Makefile +++ b/llms/mlx_lm/Makefile @@ -5,7 +5,10 @@ cd_mlx_lm: cd llms/mlx_lm run: - mlx_lm.server --model mlx-community/Ministral-8B-Instruct-2410-8bit --trust-remote-code --port 8722 + #mlx_lm.server --model mlx-community/Qwen2.5-7B-Instruct-4bit --trust-remote-code --port 8722 + mlx_lm.server --model mlx-community/Qwen2.5-7B-Instruct-8bit --trust-remote-code --port 8722 + #mlx_lm.server --model mlx-community/gemma-2-9b-it-8bit --trust-remote-code --port 8722 + #mlx_lm.server --model mlx-community/Ministral-8B-Instruct-2410-8bit --trust-remote-code --port 8722 k: ./kill.sh diff --git a/llms/mlx_lm/README.md b/llms/mlx_lm/README.md index f2a86460..4938c460 100644 --- a/llms/mlx_lm/README.md +++ b/llms/mlx_lm/README.md @@ -88,6 +88,8 @@ For example: # https://huggingface.co/mlx-community/Ministral-8B-Instruct-2410-8bit mlx_lm.server --model mlx-community/Ministral-8B-Instruct-2410-8bit --trust-remote-code --port 8722 +mlx_lm.server --model mlx-community/Qwen2.5-7B-Instruct-8bitt --trust-remote-code --port 8722 +mlx_lm.server --model mlx-community/Qwen2.5-7B-Instruct-4bit --trust-remote-code --port 8722 mlx_lm.server --model mlx-community/Meta-Llama-3.1-8B-Instruct-8bit --trust-remote-code --port 8722 mlx_lm.server --model mlx-community/Mistral-Nemo-Instruct-2407-8bit --trust-remote-code --port 8722 mlx_lm.server --model mlx-community/Mistral-7B-Instruct-v0.3-4bit --trust-remote-code --port 8722