zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-06-26 18:51:18 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

1. Add user warning for sequences over 2048 tokens in iterate_batches. (#166)

Browse Source
This commit is contained in:
wyanzhao 2023-12-21 06:29:31 -08:00 committed by GitHub
parent 43b6522af2
commit 22620de3ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lora/lora.py
View File

@ -210,6 +210,10 @@ def iterate_batches(dset, tokenizer, batch_size, train=False):
] ]
lengths = [len(x) for x in batch] lengths = [len(x) for x in batch]
# Check if any sequence is longer than 2048 tokens
if max(lengths) > 2048:
print("Warning: Some sequences are longer than 2048 tokens. Consider pre-splitting your data to save memory.")
# Pad to the max length # Pad to the max length
batch_arr = np.zeros((batch_size, max(lengths)), np.int32) batch_arr = np.zeros((batch_size, max(lengths)), np.int32)
for j in range(batch_size): for j in range(batch_size):