zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-09-09 02:22:26 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

1. Add user warning for sequences over 2048 tokens in iterate_batches.

Browse Source
This commit is contained in:
wyanzhao
2023-12-20 23:35:16 -08:00
parent 3efb1cc2cc
commit d8a4920e66
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lora/lora.py
View File

@@ -209,6 +209,10 @@ def iterate_batches(dset, tokenizer, batch_size, train=False):
for j in range(batch_size) for j in range(batch_size)
] ]
lengths = [len(x) for x in batch] lengths = [len(x) for x in batch]
# Check if any sequence is longer than 2048 tokens
if max(lengths) > 2048:
print("Warning: Some sequences are longer than 2048 tokens. Consider pre-splitting your data to save memory.")
# Pad to the max length # Pad to the max length
batch_arr = np.zeros((batch_size, max(lengths)), np.int32) batch_arr = np.zeros((batch_size, max(lengths)), np.int32)