mirror of
https://github.com/ml-explore/mlx-examples.git
synced 2025-10-23 22:18:06 +08:00
Add support for OpenELM (#719)
* add openELM * update splitting logic * update qkv logic and, transformer and MLP block * code formatting and fix args * fix array slicing and remove unused var :) * add to tuner * use mx.split for slicing qkv * merge with phi3 * remove rope scaling logic * code formatting
This commit is contained in:
@@ -87,7 +87,7 @@ def linear_to_lora_layers(
|
||||
keys.add("mlp.shared_expert_gate")
|
||||
elif model.model_type == "olmo":
|
||||
keys = set(["att_proj"])
|
||||
elif model.model_type == "phi3":
|
||||
elif model.model_type in ["phi3", "openelm"]:
|
||||
keys = set(["self_attn.qkv_proj"])
|
||||
elif model.model_type == "phi-msft":
|
||||
keys = set(["mixer.Wqkv", "moe.gate"])
|
||||
|
Reference in New Issue
Block a user