From f9709c5cbbac772af34e973cd83cb4dc0a45d7ff Mon Sep 17 00:00:00 2001
From: IlyasMoutawwakil <moutawwakil.ilyas.tsi@gmail.com>
Date: Sat, 11 May 2024 18:35:41 +0200
Subject: [PATCH] order list

---
 llm_perf/utils.py | 103 ++++++++++++++++++++--------------------------
 1 file changed, 44 insertions(+), 59 deletions(-)

diff --git a/llm_perf/utils.py b/llm_perf/utils.py
index 071b66b2..8d5145a6 100644
--- a/llm_perf/utils.py
+++ b/llm_perf/utils.py
@@ -11,50 +11,45 @@
 GENERATE_KWARGS = {"max_new_tokens": 64, "min_new_tokens": 64}
 
 
-CANONICAL_ORGANIZATIONS = [
-    # big companies
-    *["google", "facebook", "meta", "meta-llama", "microsoft", "Intel", "TencentARC", "Salesforce"],
-    # collectives
-    *["EleutherAI", "tiiuae", "NousResearch", "Open-Orca"],
-    # HF related
-    ["bigcode", "HuggingFaceH4", "huggyllama"],
-    # community members
-    ["teknium"],
-    # startups
-    *[
-        "mistral-community",
-        "openai-community",
-        "togethercomputer",
-        "stabilityai",
-        "CohereForAI",
-        "databricks",
-        "mistralai",
-        "internlm",
-        "Upstage",
-        "xai-org",
-        "Phind",
-        "01-ai",
-        "Deci",
-        "Qwen",
-    ],
-]
-
-
 OPEN_LLM_LIST = OPEN_LLM_LEADERBOARD.drop_duplicates(subset=["Model"])["Model"].tolist()
 PRETRAINED_OPEN_LLM_LIST = (
     OPEN_LLM_LEADERBOARD[OPEN_LLM_LEADERBOARD["Type"] == "pretrained"]
     .drop_duplicates(subset=["Model"])["Model"]
     .tolist()
 )
-# CANONICAL_PRETRAINED_OPEN_LLM_LIST = sorted(
-#     [model for model in PRETRAINED_OPEN_LLM_LIST if model.split("/")[0] in CANONICAL_ORGANIZATIONS]
-# )
-
+# CANONICAL_ORGANIZATIONS = [
+#     # big companies
+#     *["google", "facebook", "meta", "meta-llama", "microsoft", "Intel", "TencentARC", "Salesforce"],
+#     # collectives
+#     *["EleutherAI", "tiiuae", "NousResearch", "Open-Orca"],
+#     # HF related
+#     ["bigcode", "HuggingFaceH4", "huggyllama"],
+#     # community members
+#     ["teknium"],
+#     # startups
+#     *[
+#         "mistral-community",
+#         "openai-community",
+#         "togethercomputer",
+#         "stabilityai",
+#         "CohereForAI",
+#         "databricks",
+#         "mistralai",
+#         "internlm",
+#         "Upstage",
+#         "xai-org",
+#         "Phind",
+#         "01-ai",
+#         "Deci",
+#         "Qwen",
+#     ],
+# ]
+# CANONICAL_PRETRAINED_OPEN_LLM_LIST = [model for model in PRETRAINED_OPEN_LLM_LIST if model.split("/")[0] in CANONICAL_ORGANIZATIONS]
 CANONICAL_PRETRAINED_OPEN_LLM_LIST = [
     "01-ai/Yi-34B",
     "01-ai/Yi-6B",
-    "Deci/DeciCoder-1b",
     "Deci/DeciLM-7B",
+    "Deci/DeciCoder-1b",
     "EleutherAI/gpt-j-6b",
     "EleutherAI/gpt-neo-1.3B",
     "EleutherAI/gpt-neo-125m",
@@ -63,56 +58,48 @@
     "EleutherAI/polyglot-ko-12.8b",
     "EleutherAI/pythia-1.3b",
     "EleutherAI/pythia-1.4b",
-    # "EleutherAI/pythia-1.4b-deduped",
     "EleutherAI/pythia-12b",
-    # "EleutherAI/pythia-12b-deduped",
     "EleutherAI/pythia-160m",
-    # "EleutherAI/pythia-160m-deduped",
-    # "EleutherAI/pythia-1b-deduped",
     "EleutherAI/pythia-2.7b",
-    # "EleutherAI/pythia-2.8b-deduped",
     "EleutherAI/pythia-410m",
-    # "EleutherAI/pythia-410m-deduped",
     "EleutherAI/pythia-6.7b",
-    # "EleutherAI/pythia-6.9b-deduped",
     "EleutherAI/pythia-70m",
-    # "EleutherAI/pythia-70m-deduped",
+    "Qwen/Qwen-7B",
     "Qwen/Qwen-14B",
     "Qwen/Qwen-72B",
-    "Qwen/Qwen-7B",
     "Qwen/Qwen1.5-0.5B",
     "Qwen/Qwen1.5-1.8B",
-    "Qwen/Qwen1.5-110B",
+    "Qwen/Qwen1.5-4B",
+    "Qwen/Qwen1.5-7B",
     "Qwen/Qwen1.5-14B",
     "Qwen/Qwen1.5-32B",
-    "Qwen/Qwen1.5-4B",
     "Qwen/Qwen1.5-72B",
-    "Qwen/Qwen1.5-7B",
-    # "Qwen/Qwen1.5-7B-Chat",
+    "Qwen/Qwen1.5-110B",
     "Qwen/Qwen1.5-MoE-A2.7B",
     "Qwen/Qwen2-beta-14B",
     "Qwen/Qwen2-beta-72B",
-    "Salesforce/codegen-16B-nl",
-    # "Salesforce/codegen-6B-multi",
     "Salesforce/codegen-6B-nl",
+    "Salesforce/codegen-16B-nl",
     "TencentARC/Mistral_Pro_8B_v0.1",
     "databricks/dbrx-base",
     "facebook/opt-125m",
-    "facebook/opt-13b",
-    "facebook/opt-2.7b",
-    "facebook/opt-30b",
     "facebook/opt-350m",
+    "facebook/opt-2.7b",
     "facebook/opt-6.7b",
+    "facebook/opt-13b",
+    "facebook/opt-30b",
     "facebook/opt-66b",
-    "facebook/xglm-4.5B",
     "facebook/xglm-564M",
+    "facebook/xglm-4.5B",
     "facebook/xglm-7.5B",
+    "google/gemma-2b",
     "google/gemma-7b",
     "google/recurrentgemma-2b",
+    "google/recurrentgemma-7b",
     "internlm/internlm-20b",
     "internlm/internlm2-20b",
-    "meta-llama/Llama-2-13b-hf",
     "meta-llama/Llama-2-7b-hf",
+    "meta-llama/Llama-2-13b-hf",
     "meta-llama/Meta-Llama-3-8B",
     "meta-llama/Meta-Llama-3-70B",
     "microsoft/phi-1_5",
@@ -127,12 +114,10 @@
     "stabilityai/stablelm-3b-4e1t",
     "stabilityai/stablelm-base-alpha-3b",
     "stabilityai/stablelm-base-alpha-7b",
-    # "stabilityai/stablelm-base-alpha-7b-v2",
-    "tiiuae/falcon-180B",
-    "tiiuae/falcon-40b",
-    "tiiuae/falcon-7b",
     "tiiuae/falcon-rw-1b",
-    # "togethercomputer/RedPajama-INCITE-7B-Base",
+    "tiiuae/falcon-7b",
+    "tiiuae/falcon-40b",
+    "tiiuae/falcon-180B",
     "togethercomputer/RedPajama-INCITE-Base-3B-v1",
     "togethercomputer/RedPajama-INCITE-Base-7B-v0.1",
 ]