From e97503015f24117dc65d3fc11d8546083f1270d2 Mon Sep 17 00:00:00 2001
From: Kirill Semin <k.semin@vk.team>
Date: Mon, 5 Feb 2024 20:04:15 +0300
Subject: [PATCH] Added Hellaswag

---
 lm_eval/tasks/arc/arc_easy.yaml        | 1 +
 lm_eval/tasks/arc/arc_easy_ru.yaml     | 2 ++
 lm_eval/tasks/gsm8k/gsm8k-ru.yaml      | 2 +-
 lm_eval/tasks/hellaswag/hellaswag.yaml | 1 +
 4 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/lm_eval/tasks/arc/arc_easy.yaml b/lm_eval/tasks/arc/arc_easy.yaml
index 9c0d312bac..e551863b7e 100644
--- a/lm_eval/tasks/arc/arc_easy.yaml
+++ b/lm_eval/tasks/arc/arc_easy.yaml
@@ -12,6 +12,7 @@ doc_to_target: "{{choices.label.index(answerKey)}}"
 doc_to_choice: "{{choices.text}}"
 should_decontaminate: true
 doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+num_fewshot: 25
 metric_list:
   - metric: acc
     aggregation: mean
diff --git a/lm_eval/tasks/arc/arc_easy_ru.yaml b/lm_eval/tasks/arc/arc_easy_ru.yaml
index 24a93b6c69..06c0692922 100644
--- a/lm_eval/tasks/arc/arc_easy_ru.yaml
+++ b/lm_eval/tasks/arc/arc_easy_ru.yaml
@@ -2,3 +2,5 @@ include: arc_easy.yaml
 task: arc_easy_ru
 dataset_path: deepvk/ARC-easy-ru
 dataset_name: null
+doc_to_text: "Вопрос: {{question}}\nОтвет:"
+doc_to_decontamination_query: "Вопрос: {{question}}\nОтвет:"
\ No newline at end of file
diff --git a/lm_eval/tasks/gsm8k/gsm8k-ru.yaml b/lm_eval/tasks/gsm8k/gsm8k-ru.yaml
index b523038fd5..394765c913 100644
--- a/lm_eval/tasks/gsm8k/gsm8k-ru.yaml
+++ b/lm_eval/tasks/gsm8k/gsm8k-ru.yaml
@@ -1,7 +1,7 @@
 include: gsm8k.yaml
 group:
   - math_word_problems
-task: gsm8k-ru
+task: gsm8k_ru
 dataset_path: deepvk/gsm8k-ru
 dataset_name: null
 doc_to_text: "Вопрос: {{question}}\nОтвет:"
diff --git a/lm_eval/tasks/hellaswag/hellaswag.yaml b/lm_eval/tasks/hellaswag/hellaswag.yaml
index ec627da7d4..37a4298072 100644
--- a/lm_eval/tasks/hellaswag/hellaswag.yaml
+++ b/lm_eval/tasks/hellaswag/hellaswag.yaml
@@ -7,6 +7,7 @@ output_type: multiple_choice
 training_split: train
 validation_split: validation
 test_split: null
+num_fewshot: 10
 process_docs: !function utils.process_docs
 doc_to_text: "{{query}}"
 doc_to_target: "{{label}}"