zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-09-01 04:14:38 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Some improvements to LoRA (#528)

Browse Source 
* set cache_limit

* remove set cache_limit

* cleanup

* add gradient checkpointing

* fix sort

* mokey patch call for checkpoint

* fix example config
This commit is contained in:
Awni Hannun
2024-03-12 20:02:03 -07:00
committed by GitHub
parent e56d9015ef
commit 39084e81c2
4 changed files with 68 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
llms/mlx_lm/tuner/utils.py
View File

@@ -35,9 +35,6 @@ def linear_to_lora_layers(
lin, r=config["rank"], alpha=config["alpha"], scale=config["scale"]
)
# If the lora_parameters are set, we assume the keys
# are correct for the given model
keys = config.get("keys", None)
if keys is not None:
keys = set(keys)
@@ -53,7 +50,7 @@ def linear_to_lora_layers(
]:
keys = set(["self_attn.q_proj", "self_attn.v_proj"])
if model.model_type == "mixtral":
keys.add(["block_sparse_moe.gate"])
keys.add("block_sparse_moe.gate")
elif model.model_type == "olmo":
keys = set(["att_proj"])
elif model.model_type == "phi-msft":