From 8d9322ab93d73186facf43cdbff4578167e6b67a Mon Sep 17 00:00:00 2001
From: Kirill Semin <k.semin@vk.team>
Date: Tue, 20 Feb 2024 20:16:16 +0300
Subject: [PATCH] Added script to eval all the models in a directory

---
 eval_checkpoints.sh | 64 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 eval_checkpoints.sh

diff --git a/eval_checkpoints.sh b/eval_checkpoints.sh
new file mode 100644
index 0000000000..ff6ec4eb48
--- /dev/null
+++ b/eval_checkpoints.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Check if two arguments are provided
+if [ $# -ne 2 ]; then
+  echo "Usage: $0 <models_directory> <csv_output>"
+  exit 1
+fi
+
+# Directory containing model folders
+models_directory="$1"
+
+# Check if the directory exists
+if [ ! -d "$models_directory" ]; then
+  echo "Error: Directory $models_directory does not exist."
+  exit 1
+fi
+
+# Concatenate models_directory with csv_output
+csv_output="$models_directory/$2"
+
+# Function to extract model names from res.csv
+extract_model_names() {
+  if [ -f "$1" ]; then
+    awk -F',' 'NR>1 {print $1}' "$1"
+  else
+    echo ""
+  fi
+}
+
+# Get existing model names from res.csv or an empty string if it doesn't exist
+existing_models=$(extract_model_names "$csv_output")
+
+# Create a directory to store output files
+output_dir="output_tables"
+mkdir -p "$output_dir"
+
+# Populate model_args_list with subdirectories of the provided directory
+model_args_list=()
+for model_dir in "$models_directory"/*/; do
+  model_name="$models_directory${model_dir#$models_directory}"
+  # Check if the model has already been evaluated
+  if ! grep -q "^$model_name$" <<< "$existing_models"; then
+    model_args_list+=("pretrained=$model_name")
+  else
+    echo "Model $model_name already evaluated. Skipping..."
+  fi
+done
+
+# Iterate over the list and run the command for new models
+for model_args_value in "${model_args_list[@]}"; do
+  command="accelerate launch -m  lm_eval --model hf --tasks winogrande_ru,arc_challenge_ru,hellaswag_ru,mmlu_ru,gsm8k_ru,truthfulqa_mc2_ru --batch_size 4 --model_args $model_args_value --output_csv $csv_output"
+  output_file="$output_dir/output_${model_args_value//\//_}.txt"  # Replace "/" with "_"
+  echo "Running command: $command"
+  $command > "$output_file"  # Redirect output to file
+done
+
+# Print the contents of the output files
+for output_file in "$output_dir"/*.txt; do
+  echo -e "\nContents of $output_file:"
+  cat "$output_file"
+done
+
+# Clean-up
+rm -r "$output_dir"