Updated winogrande and hellaswag benchmarks

deepvk · Mar 2, 2024 · 694b40a · 694b40a
1 parent 6513e98
commit 694b40a
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 4 deletions.
diff --git a/lm_eval/tasks/hellaswag/hellaswag_ru.yaml b/lm_eval/tasks/hellaswag/hellaswag_ru.yaml
@@ -13,9 +13,6 @@ doc_to_text: "{{query}}"
 doc_to_target: "{{label}}"
 doc_to_choice: "choices"
 metric_list:
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true
   - metric: acc_norm
     aggregation: mean
     higher_is_better: true

diff --git a/lm_eval/tasks/winogrande/default_ru.yaml b/lm_eval/tasks/winogrande/default_ru.yaml
@@ -3,6 +3,6 @@ task: winogrande_ru
 dataset_path: deepvk/winogrande-ru
 dataset_name: null
 dataset_kwargs:
-  revision: vk-edited
+  revision: second-iteration
 num_fewshot: 5
 fewshot_split: train