Nits (#2500)

* fix test task * dont call lm.chat_template each time
EleutherAI · Nov 20, 2024 · 867413f · 867413f
1 parent 62b4364
commit 867413f
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/lm_eval/evaluator.py b/lm_eval/evaluator.py
@@ -294,7 +294,9 @@ def _adjust_config(task_dict):
             model_source=model,
             model_args=model_args,
             system_instruction=system_instruction,
-            chat_template=lm.chat_template(apply_chat_template),
+            chat_template=lm.chat_template(apply_chat_template)
+            if apply_chat_template
+            else None,
             fewshot_as_multiturn=fewshot_as_multiturn,
         )
 

diff --git a/tests/test_tasks.py b/tests/test_tasks.py
@@ -15,7 +15,7 @@
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 task_manager = tasks.TaskManager()
 # Default Task
-TASKS = ["metabench_winogrande"]
+TASKS = ["arc_easy"]
 
 
 def task_class():