[m_mmul] added multilingual evaluation from alexandrainst/m_mmlu (Ele…

…utherAI#1358)
deepvk · Feb 12, 2024 · b69c67c · b69c67c
1 parent 7397b96
commit b69c67c
Show file tree

Hide file tree

Showing 34 changed files with 178 additions and 0 deletions.
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/_default_yaml b/lm_eval/tasks/okapi/mmlu_multilingual/_default_yaml
@@ -0,0 +1,17 @@
+group:
+  - m_mmlu
+dataset_path: alexandrainst/m_mmlu
+test_split: test
+fewshot_split: train
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:"
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: answer
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/_generate_configs.py b/lm_eval/tasks/okapi/mmlu_multilingual/_generate_configs.py
@@ -0,0 +1,33 @@
+import yaml
+import datasets
+
+from tqdm import tqdm
+
+
+def main() -> None:
+    dataset_path = "alexandrainst/m_mmlu"
+
+    # Removed hy and sk subdataset because the original dataset is broken
+    # I created this PR https://huggingface.co/datasets/alexandrainst/m_mmlu/discussions/3
+    # on the dataset for the authors, in case it will be accepeted the filter can be removed
+    keys_without_hy_sk = list(filter(lambda k: ('hy' not in k and  'sk' not in k), 
+                                     datasets.get_dataset_infos(dataset_path).keys()))
+
+    for task in tqdm():
+        file_name = f"m_mmlu_{task}.yaml"
+        try:
+            with open(f"{file_name}", "w") as f:
+                f.write("# Generated by _generate_configs.py\n")
+                yaml.dump(
+                    {
+                        "include": "_default_yaml",
+                        "task": f"{dataset_path.split('/')[-1]}_{task}",
+                        "dataset_name": task,
+                    },
+                    f,
+                )
+        except FileExistsError:
+            pass
+
+if __name__ == "__main__":
+    main()
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ar.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ar.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: ar
+include: _default_yaml
+task: m_mmlu_ar
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_bn.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_bn.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: bn
+include: _default_yaml
+task: m_mmlu_bn
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ca.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ca.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: ca
+include: _default_yaml
+task: m_mmlu_ca
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_da.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_da.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: da
+include: _default_yaml
+task: m_mmlu_da
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_de.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_de.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: de
+include: _default_yaml
+task: m_mmlu_de
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_en.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_en.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: en
+include: _default_yaml
+task: m_mmlu_en
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_es.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_es.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: es
+include: _default_yaml
+task: m_mmlu_es
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_eu.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_eu.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: eu
+include: _default_yaml
+task: m_mmlu_eu
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_fr.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_fr.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: fr
+include: _default_yaml
+task: m_mmlu_fr
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_gu.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_gu.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: gu
+include: _default_yaml
+task: m_mmlu_gu
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hi.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hi.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: hi
+include: _default_yaml
+task: m_mmlu_hi
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hr.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hr.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: hr
+include: _default_yaml
+task: m_mmlu_hr
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hu.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hu.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: hu
+include: _default_yaml
+task: m_mmlu_hu
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_id.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_id.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: id
+include: _default_yaml
+task: m_mmlu_id
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_is.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_is.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: is
+include: _default_yaml
+task: m_mmlu_is
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_it.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_it.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: it
+include: _default_yaml
+task: m_mmlu_it
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_kn.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_kn.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: kn
+include: _default_yaml
+task: m_mmlu_kn
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ml.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ml.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: ml
+include: _default_yaml
+task: m_mmlu_ml
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_mr.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_mr.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: mr
+include: _default_yaml
+task: m_mmlu_mr
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nb.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nb.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: nb
+include: _default_yaml
+task: m_mmlu_nb
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ne.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ne.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: ne
+include: _default_yaml
+task: m_mmlu_ne
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nl.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nl.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: nl
+include: _default_yaml
+task: m_mmlu_nl
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_pt.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_pt.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: pt
+include: _default_yaml
+task: m_mmlu_pt
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ro.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ro.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: ro
+include: _default_yaml
+task: m_mmlu_ro
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ru.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ru.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: ru
+include: _default_yaml
+task: m_mmlu_ru
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sr.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sr.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: sr
+include: _default_yaml
+task: m_mmlu_sr
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sv.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sv.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: sv
+include: _default_yaml
+task: m_mmlu_sv
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ta.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ta.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: ta
+include: _default_yaml
+task: m_mmlu_ta
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_te.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_te.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: te
+include: _default_yaml
+task: m_mmlu_te
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_uk.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_uk.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: uk
+include: _default_yaml
+task: m_mmlu_uk
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_vi.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_vi.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: vi
+include: _default_yaml
+task: m_mmlu_vi
diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_zh.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_zh.yaml
@@ -0,0 +1,4 @@
+# Generated by _generate_configs.py
+dataset_name: zh
+include: _default_yaml
+task: m_mmlu_zh