From bb261aadcb9a5113b13e6f88791ccbbbc2c81c11 Mon Sep 17 00:00:00 2001 From: Goekdeniz-Guelmez Date: Sat, 1 Mar 2025 12:42:39 +0100 Subject: [PATCH] updates --- llms/mlx_lm/tuner/datasets.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/llms/mlx_lm/tuner/datasets.py b/llms/mlx_lm/tuner/datasets.py index 59e9b232..97513c78 100644 --- a/llms/mlx_lm/tuner/datasets.py +++ b/llms/mlx_lm/tuner/datasets.py @@ -2,18 +2,9 @@ import itertools import json import types from pathlib import Path -from typing import Any, Dict, List, Optional, Union - +from typing import Any, Dict, List, Union from transformers import PreTrainedTokenizer -from typing import List, Dict, Union -from transformers import PreTrainedTokenizer - -from typing import List, Dict, Union -from transformers import PreTrainedTokenizer - -from typing import List, Dict, Union -from transformers import PreTrainedTokenizer class ORPODataset: def __init__( @@ -368,7 +359,7 @@ def load_dataset(args, tokenizer: PreTrainedTokenizer): train, valid, test = load_local_dataset(args, data_path, tokenizer, args) else: print(f"Loading Hugging Face dataset {args.data}.") - train, valid, test = load_hf_dataset(args.data, tokenizer, args) + train, valid, test = load_hf_dataset(args, args.data, tokenizer, args) if args.train and len(train) == 0: raise ValueError(