Merge branch 'main' of https://github.com/huggingface/optimum-benchmark…

… into main

.github/workflows/update_llm_perf_leaderboard.yaml

@@ -0,0 +1,38 @@
+name: Update LLM Perf Leaderboard
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - gather-llm-perf-benchmarks
+  schedule:
+    - cron: "0 */6 * * *"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  update_llm_perf_leaderboard:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+
+      - name: Install requirements
+        run: |
+          pip install --upgrade pip
+          pip install huggingface_hub[hf_transfer]
+          pip install .
+
+      - name: Update Open LLM Leaderboard
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_HUB_ENABLE_HF_TRANSFER: 1
+        run: |
+          python llm_perf/update_llm_perf_leaderboard.py

.github/workflows/update_open_llm_leaderboard.yaml

@@ -3,7 +3,7 @@ name: Update Open LLM Leaderboard
 on:
   workflow_dispatch:
   schedule:
-    - cron: "0 0 * * *"
+    - cron: "0 */6 * * *"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

llm_perf/update_llm_perf_leaderboard.py

@@ -0,0 +1,38 @@
+from glob import glob
+from tempfile import TemporaryDirectory
+
+import pandas as pd
+from huggingface_hub import create_repo, snapshot_download, upload_file
+from tqdm import tqdm
+
+from optimum_benchmark import Benchmark
+
+
+def gather_benchmarks(subset: str, machine: str):
+    pull_repo_id = f"optimum-benchmark/llm-perf-pytorch-cuda-{subset}-{machine}"
+
+    snapshot = snapshot_download(repo_type="dataset", repo_id=pull_repo_id, allow_patterns=["**/benchmark.json"])
+
+    dfs = []
+
+    for file in tqdm(glob(f"{snapshot}/**/benchmark.json", recursive=True)):
+        dfs.append(Benchmark.from_json(file).to_dataframe())
+
+    benchmarks = pd.concat(dfs, ignore_index=True)
+
+    tmp_dir = TemporaryDirectory()
+    push_repo_id = "optimum-benchmark/llm-perf-leaderboard"
+    file_name = f"llm-perf-leaderboard-{subset}-{machine}.csv"
+
+    benchmarks.to_csv(f"{tmp_dir.name}/{file_name}", index=False)
+
+    create_repo(repo_id=push_repo_id, repo_type="dataset", private=True, exist_ok=True)
+    upload_file(
+        path_or_fileobj=f"{tmp_dir.name}/{file_name}", path_in_repo=file_name, repo_id=push_repo_id, repo_type="dataset"
+    )
+    tmp_dir.cleanup()
+
+
+for subset in ["unquantized", "bnb", "awq", "gptq"]:
+    for machine in ["1xA10"]:
+        gather_benchmarks(subset, machine)