zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-09-01 04:14:38 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix argpartition call in Mixtral and other MOES (#676)

Browse Source 
* Update mixtral.py

* fix all moes

---------

Co-authored-by: yuhai-china <yuhai.china@gmail.com>
This commit is contained in:
Awni Hannun
2024-04-12 11:00:56 -07:00
committed by GitHub
parent 9c5554d8ee
commit d3f8e4aee9
4 changed files with 4 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
llms/mlx_lm/models/dbrx.py
View File

@@ -143,7 +143,7 @@ class SparseMoeBlock(nn.Module):
gates = self.router(x)
gates = mx.softmax(gates.astype(mx.float32), axis=-1)
inds = mx.stop_gradient(mx.argpartition(-gates, kth=ne, axis=-1)[:, :ne])
inds = mx.stop_gradient(mx.argpartition(-gates, kth=ne - 1, axis=-1)[:, :ne])
scores = mx.take_along_axis(gates, inds, axis=-1)
scores = scores / mx.linalg.norm(scores, ord=1, axis=-1, keepdims=True)
scores = scores.astype(x.dtype)