zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-06-24 17:31:18 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fine tune deepseek (#932)

Browse Source
This commit is contained in:
Awni Hannun 2024-08-22 10:41:21 -07:00 committed by GitHub
parent 0164d2058b
commit 58591a1b41
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
llms/mlx_lm/tuner/utils.py
View File

@ -128,6 +128,16 @@ def linear_to_lora_layers(
keys = set(["norm_attn_norm.attn.Wqkv", "ffn.router.layer"]) keys = set(["norm_attn_norm.attn.Wqkv", "ffn.router.layer"])
elif model.model_type == "internlm2": elif model.model_type == "internlm2":
keys = set(["attention.wqkv", "attention.wo"]) keys = set(["attention.wqkv", "attention.wo"])
elif model.model_type == "deepseek_v2":
keys = set(
[
"self_attn.q_proj",
"self_attn.q_a_proj",
"self_attn.q_b_proj",
"self_attn.kv_a_proj_with_mqa",
"self_attn.kv_b_proj",
]
)
else: else:
raise ValueError(f"Lora does not support {model.model_type}") raise ValueError(f"Lora does not support {model.model_type}")