zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-09-01 04:14:38 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

1. Add user warning for sequences over 2048 tokens in iterate_batches. (#166)

Browse Source
This commit is contained in:
wyanzhao
2023-12-21 06:29:31 -08:00
committed by GitHub
parent 58f409feb0
commit 18f0a96cee
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lora/lora.py
View File

@@ -209,6 +209,10 @@ def iterate_batches(dset, tokenizer, batch_size, train=False):
for j in range(batch_size)
]
lengths = [len(x) for x in batch]
# Check if any sequence is longer than 2048 tokens
if max(lengths) > 2048:
print("Warning: Some sequences are longer than 2048 tokens. Consider pre-splitting your data to save memory.")
# Pad to the max length
batch_arr = np.zeros((batch_size, max(lengths)), np.int32)