zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-09-01 04:14:38 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix lora for qwen moe (#743)

Browse Source 
* fix lora for qwen moe

* use max seq length in test as well
This commit is contained in:
Awni Hannun
2024-05-02 21:55:09 -07:00
committed by GitHub
parent 5079af62db
commit 92430df0a0
2 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
llms/mlx_lm/models/qwen2_moe.py
View File

@@ -141,7 +141,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
if self.training:
inds = np.array(inds)
y = mx.zeros((B, ne, D), x.dtype)
y = mx.zeros((B * L, ne, D), x.dtype)
for e, expert in enumerate(self.experts):
idx1, idx2 = map(mx.array, np.where(inds == e))
if idx1.size == 0: