zhangyiss/mlx-examples
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx-examples.git synced 2025-08-29 12:26:07 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add support for multiturn fewshot examples and chat templates

Browse Source 
Added two new arguments to the evaluation script: `--fewshot-as-multiturn` and `--apply-chat-template` which correspond to lm_eval options of similar names and are very often used to ensure apples-to-apples comparisons of lm_evaluation results
This commit is contained in:
Chime Ogbuji 2024-12-23 10:45:17 -05:00 committed by Awni Hannun
parent c4833a2f55
commit d352074e73
1 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
llms/mlx_lm/evaluate.py
View File

@ -332,6 +332,18 @@ def main():
type=float, type=float,
) )
parser.add_argument("--seed", type=int, default=123, help="Random seed.") parser.add_argument("--seed", type=int, default=123, help="Random seed.")
parser.add_argument(
"--fewshot-as-multiturn",
action="store_true",
help="Whether to provide the fewshot examples as a multiturn conversation or a single user turn.",
default=False,
)
parser.add_argument(
"--apply-chat-template",
action="store_true",
help="Specifies whether to apply a chat template to the prompt",
default=False,
)
args = parser.parse_args() args = parser.parse_args()
output_dir = Path(args.output_dir) output_dir = Path(args.output_dir)
@ -347,6 +359,8 @@ def main():
results = lm_eval.simple_evaluate( results = lm_eval.simple_evaluate(
model=lm, model=lm,
tasks=args.tasks, tasks=args.tasks,
fewshot_as_multiturn=args.fewshot_as_multiturn,
apply_chat_template=args.apply_chat_template,
num_fewshot=args.num_shots, num_fewshot=args.num_shots,
limit=args.limit, limit=args.limit,
random_seed=args.seed, random_seed=args.seed,