From cffd0ed79e105891ba7509c6fbe4fc0f4af80477 Mon Sep 17 00:00:00 2001 From: "chenguangjian.jk" Date: Sun, 20 Oct 2024 03:08:27 +0800 Subject: [PATCH] run: mlx_lm.server --model mlx-community/Qwen2.5-14B-Instruct-8bit --trust-remote-code --port 8722 --- llms/mlx_lm/Makefile | 5 ----- llms/mlx_lm/README.md | 9 +++++++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/llms/mlx_lm/Makefile b/llms/mlx_lm/Makefile index 5bcec668..78de1a46 100644 --- a/llms/mlx_lm/Makefile +++ b/llms/mlx_lm/Makefile @@ -5,12 +5,7 @@ cd_mlx_lm: cd llms/mlx_lm run: - #mlx_lm.server --model mlx-community/Qwen2.5-7B-Instruct-4bit --trust-remote-code --port 8722 - #mlx_lm.server --model mlx-community/Qwen2.5-7B-Instruct-8bit --trust-remote-code --port 8722 - #mlx_lm.server --model mlx-community/Qwen2.5-14B-Instruct-4bit --trust-remote-code --port 8722 mlx_lm.server --model mlx-community/Qwen2.5-14B-Instruct-8bit --trust-remote-code --port 8722 - #mlx_lm.server --model mlx-community/Ministral-8B-Instruct-2410-4bit --trust-remote-code --port 8722 - #mlx_lm.server --model mlx-community/Ministral-8B-Instruct-2410-8bit --trust-remote-code --port 8722 k: ./kill.sh diff --git a/llms/mlx_lm/README.md b/llms/mlx_lm/README.md index 4938c460..9558fc51 100644 --- a/llms/mlx_lm/README.md +++ b/llms/mlx_lm/README.md @@ -94,6 +94,15 @@ mlx_lm.server --model mlx-community/Meta-Llama-3.1-8B-Instruct-8bit --trust-remo mlx_lm.server --model mlx-community/Mistral-Nemo-Instruct-2407-8bit --trust-remote-code --port 8722 mlx_lm.server --model mlx-community/Mistral-7B-Instruct-v0.3-4bit --trust-remote-code --port 8722 mlx_lm.server --model mlx-community/internlm2_5-7b-chat-8bit --trust-remote-code --port 8722 + +# for run +run: + mlx_lm.server --model mlx-community/Qwen2.5-14B-Instruct-8bit --trust-remote-code --port 8722 + #mlx_lm.server --model mlx-community/Qwen2.5-7B-Instruct-4bit --trust-remote-code --port 8722 + #mlx_lm.server --model mlx-community/Qwen2.5-7B-Instruct-8bit --trust-remote-code --port 8722 + #mlx_lm.server --model mlx-community/Qwen2.5-14B-Instruct-4bit --trust-remote-code --port 8722 + #mlx_lm.server --model mlx-community/Ministral-8B-Instruct-2410-4bit --trust-remote-code --port 8722 + #mlx_lm.server --model mlx-community/Ministral-8B-Instruct-2410-8bit --trust-remote-code --port 8722 ``` This will start a text generation server on port `8080` of the `localhost`