From 0994de67c8ae64954d51b8fdb251c26ff95c5907 Mon Sep 17 00:00:00 2001
From: Ilyas Moutawwakil <57442720+IlyasMoutawwakil@users.noreply.github.com>
Date: Sat, 11 May 2024 18:21:38 +0200
Subject: [PATCH] Gather llm perf benchmarks (#198)

---
 .../update_llm_perf_leaderboard.yaml          | 38 +++++++++++++++++++
 .../update_open_llm_leaderboard.yaml          |  2 +-
 llm_perf/update_llm_perf_leaderboard.py       | 38 +++++++++++++++++++
 3 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/update_llm_perf_leaderboard.yaml
 create mode 100644 llm_perf/update_llm_perf_leaderboard.py

diff --git a/.github/workflows/update_llm_perf_leaderboard.yaml b/.github/workflows/update_llm_perf_leaderboard.yaml
new file mode 100644
index 00000000..445bff77
--- /dev/null
+++ b/.github/workflows/update_llm_perf_leaderboard.yaml
@@ -0,0 +1,38 @@
+name: Update LLM Perf Leaderboard
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - gather-llm-perf-benchmarks
+  schedule:
+    - cron: "0 */6 * * *"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  update_llm_perf_leaderboard:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+
+      - name: Install requirements
+        run: |
+          pip install --upgrade pip
+          pip install huggingface_hub[hf_transfer]
+          pip install .
+
+      - name: Update Open LLM Leaderboard
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_HUB_ENABLE_HF_TRANSFER: 1
+        run: |
+          python llm_perf/update_llm_perf_leaderboard.py
diff --git a/.github/workflows/update_open_llm_leaderboard.yaml b/.github/workflows/update_open_llm_leaderboard.yaml
index 30fb40ca..21361316 100644
--- a/.github/workflows/update_open_llm_leaderboard.yaml
+++ b/.github/workflows/update_open_llm_leaderboard.yaml
@@ -3,7 +3,7 @@ name: Update Open LLM Leaderboard
 on:
   workflow_dispatch:
   schedule:
-    - cron: "0 0 * * *"
+    - cron: "0 */6 * * *"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
diff --git a/llm_perf/update_llm_perf_leaderboard.py b/llm_perf/update_llm_perf_leaderboard.py
new file mode 100644
index 00000000..41e2dd6b
--- /dev/null
+++ b/llm_perf/update_llm_perf_leaderboard.py
@@ -0,0 +1,38 @@
+from glob import glob
+from tempfile import TemporaryDirectory
+
+import pandas as pd
+from huggingface_hub import create_repo, snapshot_download, upload_file
+from tqdm import tqdm
+
+from optimum_benchmark import Benchmark
+
+
+def gather_benchmarks(subset: str, machine: str):
+    pull_repo_id = f"optimum-benchmark/llm-perf-pytorch-cuda-{subset}-{machine}"
+
+    snapshot = snapshot_download(repo_type="dataset", repo_id=pull_repo_id, allow_patterns=["**/benchmark.json"])
+
+    dfs = []
+
+    for file in tqdm(glob(f"{snapshot}/**/benchmark.json", recursive=True)):
+        dfs.append(Benchmark.from_json(file).to_dataframe())
+
+    benchmarks = pd.concat(dfs, ignore_index=True)
+
+    tmp_dir = TemporaryDirectory()
+    push_repo_id = "optimum-benchmark/llm-perf-leaderboard"
+    file_name = f"llm-perf-leaderboard-{subset}-{machine}.csv"
+
+    benchmarks.to_csv(f"{tmp_dir.name}/{file_name}", index=False)
+
+    create_repo(repo_id=push_repo_id, repo_type="dataset", private=True, exist_ok=True)
+    upload_file(
+        path_or_fileobj=f"{tmp_dir.name}/{file_name}", path_in_repo=file_name, repo_id=push_repo_id, repo_type="dataset"
+    )
+    tmp_dir.cleanup()
+
+
+for subset in ["unquantized", "bnb", "awq", "gptq"]:
+    for machine in ["1xA10"]:
+        gather_benchmarks(subset, machine)