From 97d6b17a9a3d20ec3c1de000295bf3769aa900b2 Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Mon, 18 Dec 2023 20:03:50 -0800 Subject: [PATCH] Add Mistral-7B-Instruct-v0.1 from huggingface. (#2010) Summary: Add Mistral-7B-Instruct-v0.1 from huggingface. See https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1 Pull Request resolved: https://github.com/pytorch/benchmark/pull/2010 Reviewed By: aaronenyeshi Differential Revision: D50692789 Pulled By: xuzhao9 fbshipit-source-id: 7aa3c455882726ac3b1a50c2ca349eee9c34c938 --- requirements.txt | 3 ++- .../mistral_7b_instruct/__init__.py | 19 +++++++++++++++++++ .../mistral_7b_instruct/install.py | 9 +++++++++ .../mistral_7b_instruct/metadata.yaml | 11 +++++++++++ .../models/phi_1_5/requirements.txt | 3 ++- .../framework/huggingface/model_factory.py | 6 ++++-- 6 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 torchbenchmark/canary_models/mistral_7b_instruct/__init__.py create mode 100644 torchbenchmark/canary_models/mistral_7b_instruct/install.py create mode 100644 torchbenchmark/canary_models/mistral_7b_instruct/metadata.yaml diff --git a/requirements.txt b/requirements.txt index 3f5713c243..6f46e3d736 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,8 @@ pytest-benchmark requests tabulate git+https://github.com/huggingface/pytorch-image-models.git@730b907 -git+https://github.com/huggingface/transformers.git@6c26faa +# this version of transformers is required as per this page https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1 +transformers==4.34.1 MonkeyType psutil pyyaml diff --git a/torchbenchmark/canary_models/mistral_7b_instruct/__init__.py b/torchbenchmark/canary_models/mistral_7b_instruct/__init__.py new file mode 100644 index 0000000000..8241db7038 --- /dev/null +++ b/torchbenchmark/canary_models/mistral_7b_instruct/__init__.py @@ -0,0 +1,19 @@ +from torchbenchmark.tasks import NLP +from torchbenchmark.util.framework.huggingface.model_factory import HuggingFaceModel + +class Model(HuggingFaceModel): + task = NLP.LANGUAGE_MODELING + # DEFAULT_TRAIN_BSIZE not specified since we're not implementing a train test + # DEFAULT_TRAIN_BSIZE = 1 + DEFAULT_EVAL_BSIZE = 1 + + def __init__(self, test, device, batch_size=None, extra_args=[]): + super().__init__(name="mistral_7b_instruct", test=test, device=device, batch_size=batch_size, extra_args=extra_args) + + def train(self): + return NotImplementedError("Not implemented") + + def eval(self): + if (self.device == "cpu"): + raise NotImplementedError("mistral_7b_instruct model is too slow on CPU - skip CPU test.") + super().eval() diff --git a/torchbenchmark/canary_models/mistral_7b_instruct/install.py b/torchbenchmark/canary_models/mistral_7b_instruct/install.py new file mode 100644 index 0000000000..1cbb0df344 --- /dev/null +++ b/torchbenchmark/canary_models/mistral_7b_instruct/install.py @@ -0,0 +1,9 @@ +import subprocess +import sys +import os +from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model + +if __name__ == '__main__': + patch_transformers() + model_name = os.path.basename(os.path.dirname(os.path.abspath(__file__))) + cache_model(model_name) \ No newline at end of file diff --git a/torchbenchmark/canary_models/mistral_7b_instruct/metadata.yaml b/torchbenchmark/canary_models/mistral_7b_instruct/metadata.yaml new file mode 100644 index 0000000000..6a91c2469b --- /dev/null +++ b/torchbenchmark/canary_models/mistral_7b_instruct/metadata.yaml @@ -0,0 +1,11 @@ +devices: + - device: NVIDIA A10G + - device: NVIDIA A100-SXM4-40GB +eval_batch_size: 1 +eval_benchmark: false +eval_deterministic: false +eval_nograd: true +train_benchmark: false +train_deterministic: false +not_implemented: + - device: cpu \ No newline at end of file diff --git a/torchbenchmark/models/phi_1_5/requirements.txt b/torchbenchmark/models/phi_1_5/requirements.txt index 8fdd8e958c..bd5f4ab230 100644 --- a/torchbenchmark/models/phi_1_5/requirements.txt +++ b/torchbenchmark/models/phi_1_5/requirements.txt @@ -1 +1,2 @@ -einops \ No newline at end of file +einops +flash_attn diff --git a/torchbenchmark/util/framework/huggingface/model_factory.py b/torchbenchmark/util/framework/huggingface/model_factory.py index 4700cfe4d8..4d23251999 100644 --- a/torchbenchmark/util/framework/huggingface/model_factory.py +++ b/torchbenchmark/util/framework/huggingface/model_factory.py @@ -36,6 +36,8 @@ 'llama_v2_13b' : (512,512, 'AutoConfig.from_pretrained("meta-llama/Llama-2-13b-hf")', 'AutoModelForCausalLM'), 'llama_v2_70b' : (512, 512, 'AutoConfig.from_pretrained("meta-llama/Llama-2-70b-hf")', 'AutoModelForMaskedLM'), 'phi_1_5' : (512, 512, 'AutoConfig.from_pretrained("microsoft/phi-1_5", trust_remote_code=True)', 'AutoModelForCausalLM'), + # as per this page https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1 trust_remote_code=True is not required + 'mistral_7b_instruct' : (128, 128, 'AutoConfig.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1")', 'AutoModelForCausalLM'), 'hf_Yi' : (512, 512, 'AutoConfig.from_pretrained("01-ai/Yi-6B", trust_remote_code=True)', 'AutoModelForCausalLM'), } @@ -89,8 +91,8 @@ def __init__(self, name, test, device, batch_size=None, extra_args=[]): config.num_buckets = 128 class_ctor = getattr(transformers, class_models[name][3]) kwargs = {} - remote_code_required = ['hf_Falcon_7b', 'hf_MPT_7b_instruct', 'phi_1_5', 'hf_Yi'] - if name in remote_code_required: + hugging_face_models_requiring_trust_remote_code = ["hf_Falcon_7b", "hf_MPT_7b_instruct", "phi_1_5", "hf_Yi"] + if name in hugging_face_models_requiring_trust_remote_code: kwargs["trust_remote_code"] = True self.model = class_ctor.from_config(config, **kwargs).to(device) self.optimizer = optim.Adam(