From 71d9f8cc38020118bd384328ee530e17746600f6 Mon Sep 17 00:00:00 2001 From: Chime Ogbuji Date: Wed, 6 Nov 2024 20:29:12 -0500 Subject: [PATCH] Fix --- llms/mlx_lm/tuner/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llms/mlx_lm/tuner/trainer.py b/llms/mlx_lm/tuner/trainer.py index 7d1821d4..392030cb 100644 --- a/llms/mlx_lm/tuner/trainer.py +++ b/llms/mlx_lm/tuner/trainer.py @@ -166,7 +166,7 @@ def iterate_delineated_batches( for j in batch_idx[i]: prompt, completion = dataset.get_prompt_and_completion(j) prompt_lengths.append(input_length(prompt, completion, tokenizer)) - full_sequence = [tokenizer.encode(dataset[j]) for j in batch_idx[i]] + full_sequence = tokenizer.encode(dataset[j]) if full_sequence[-1] != tokenizer.eos_token_id: full_sequence.append(tokenizer.eos_token_id) batch.append(full_sequence)