zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-09-17 09:08:10 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: prevent gradients from flowing through the reference model's logits

Browse Source
This commit is contained in:
Goekdeniz-Guelmez
2025-02-09 17:02:58 +01:00
parent 54179901b5
commit a527cdb39b
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
llms/mlx_lm/tuner/grpo_trainer.py
View File

@@ -189,7 +189,7 @@ def compute_kl(logprobs1, logprobs2):
def compute_policy_ratio(current_logprobs, ref_logprobs): def compute_policy_ratio(current_logprobs, ref_logprobs):
return mx.exp(mx.array(current_logprobs - ref_logprobs, dtype=mx.float32)) return mx.exp(mx.array(current_logprobs - mx.stop_gradient(ref_logprobs), dtype=mx.float32))
def grpo_loss( def grpo_loss(