diff --git a/torchbenchmark/models/llava/__init__.py b/torchbenchmark/models/llava/__init__.py new file mode 100644 index 0000000000..39e61070fc --- /dev/null +++ b/torchbenchmark/models/llava/__init__.py @@ -0,0 +1,17 @@ +from torchbenchmark.tasks import NLP +from torchbenchmark.util.framework.huggingface.model_factory import HuggingFaceModel + +class Model(HuggingFaceModel): + task = NLP.LANGUAGE_MODELING + # DEFAULT_TRAIN_BSIZE not specified since we're not implementing a train test + # DEFAULT_TRAIN_BSIZE = 1 + DEFAULT_EVAL_BSIZE = 1 + + def __init__(self, test, device, batch_size=None, extra_args=[]): + super().__init__(name="llava", test=test, device=device, batch_size=batch_size, extra_args=extra_args) + + def train(self): + return NotImplementedError("Not implemented") + + def eval(self): + super().eval() \ No newline at end of file diff --git a/torchbenchmark/models/llava/install.py b/torchbenchmark/models/llava/install.py new file mode 100644 index 0000000000..1a49905932 --- /dev/null +++ b/torchbenchmark/models/llava/install.py @@ -0,0 +1,13 @@ +import subprocess +import sys +import os +from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model + +def pip_install_requirements(): + subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) + +if __name__ == '__main__': + pip_install_requirements() + patch_transformers() + model_name = os.path.basename(os.path.dirname(os.path.abspath(__file__))) + cache_model(model_name) \ No newline at end of file diff --git a/torchbenchmark/models/llava/metadata.yaml b/torchbenchmark/models/llava/metadata.yaml new file mode 100644 index 0000000000..149a408edc --- /dev/null +++ b/torchbenchmark/models/llava/metadata.yaml @@ -0,0 +1,10 @@ +devices: + NVIDIA A100-SXM4-40GB: + eval_batch_size: 1 +eval_benchmark: false +eval_deterministic: false +eval_nograd: true +train_benchmark: false +train_deterministic: false +not_implemented: + - device: NVIDIA A10G diff --git a/torchbenchmark/models/llava/requirements.txt b/torchbenchmark/models/llava/requirements.txt new file mode 100644 index 0000000000..8fdd8e958c --- /dev/null +++ b/torchbenchmark/models/llava/requirements.txt @@ -0,0 +1 @@ +einops \ No newline at end of file diff --git a/torchbenchmark/models/moondream/__init__.py b/torchbenchmark/models/moondream/__init__.py new file mode 100644 index 0000000000..aedaa7ab32 --- /dev/null +++ b/torchbenchmark/models/moondream/__init__.py @@ -0,0 +1,17 @@ +from torchbenchmark.tasks import NLP +from torchbenchmark.util.framework.huggingface.model_factory import HuggingFaceModel + +class Model(HuggingFaceModel): + task = NLP.LANGUAGE_MODELING + # DEFAULT_TRAIN_BSIZE not specified since we're not implementing a train test + # DEFAULT_TRAIN_BSIZE = 1 + DEFAULT_EVAL_BSIZE = 1 + + def __init__(self, test, device, batch_size=None, extra_args=[]): + super().__init__(name="moondream", test=test, device=device, batch_size=batch_size, extra_args=extra_args) + + def train(self): + return NotImplementedError("Not implemented") + + def eval(self): + super().eval() \ No newline at end of file diff --git a/torchbenchmark/models/moondream/install.py b/torchbenchmark/models/moondream/install.py new file mode 100644 index 0000000000..1a49905932 --- /dev/null +++ b/torchbenchmark/models/moondream/install.py @@ -0,0 +1,13 @@ +import subprocess +import sys +import os +from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model + +def pip_install_requirements(): + subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) + +if __name__ == '__main__': + pip_install_requirements() + patch_transformers() + model_name = os.path.basename(os.path.dirname(os.path.abspath(__file__))) + cache_model(model_name) \ No newline at end of file diff --git a/torchbenchmark/models/moondream/metadata.yaml b/torchbenchmark/models/moondream/metadata.yaml new file mode 100644 index 0000000000..149a408edc --- /dev/null +++ b/torchbenchmark/models/moondream/metadata.yaml @@ -0,0 +1,10 @@ +devices: + NVIDIA A100-SXM4-40GB: + eval_batch_size: 1 +eval_benchmark: false +eval_deterministic: false +eval_nograd: true +train_benchmark: false +train_deterministic: false +not_implemented: + - device: NVIDIA A10G diff --git a/torchbenchmark/models/moondream/requirements.txt b/torchbenchmark/models/moondream/requirements.txt new file mode 100644 index 0000000000..8fdd8e958c --- /dev/null +++ b/torchbenchmark/models/moondream/requirements.txt @@ -0,0 +1 @@ +einops \ No newline at end of file diff --git a/torchbenchmark/util/framework/huggingface/model_factory.py b/torchbenchmark/util/framework/huggingface/model_factory.py index b852717f01..81c8f99676 100644 --- a/torchbenchmark/util/framework/huggingface/model_factory.py +++ b/torchbenchmark/util/framework/huggingface/model_factory.py @@ -8,7 +8,7 @@ from torchbenchmark.util.model import BenchmarkModel from torchbenchmark.tasks import NLP import transformers -from transformers import AutoConfig, ReformerConfig, BertConfig, GenerationConfig, WhisperConfig, LlamaConfig +from transformers import AutoConfig, ReformerConfig, BertConfig, GenerationConfig, WhisperConfig, LlamaConfig, PhiConfig from typing import Tuple class_models = { @@ -33,11 +33,13 @@ # default num_hidden_layers=32 but that OOMs, feel free to change this config to something more real 'llama_v2_7b_16h' : (128,512, 'LlamaConfig(num_hidden_layers=16)', 'AutoModelForCausalLM'), 'hf_MPT_7b_instruct': (512, 512, 'AutoConfig.from_pretrained("mosaicml/mpt-7b-instruct", trust_remote_code=True)', 'AutoModelForCausalLM'), + 'llava' : (512,512, 'AutoConfig.from_pretrained("liuhaotian/llava-v1.5-13b")', 'LlavaForConditionalGeneration'), 'llama_v2_7b' : (512,512, 'AutoConfig.from_pretrained("meta-llama/Llama-2-7b-hf")', 'AutoModelForCausalLM'), 'llama_v2_13b' : (512,512, 'AutoConfig.from_pretrained("meta-llama/Llama-2-13b-hf")', 'AutoModelForCausalLM'), 'llama_v2_70b' : (512, 512, 'AutoConfig.from_pretrained("meta-llama/Llama-2-70b-hf")', 'AutoModelForMaskedLM'), 'phi_1_5' : (512, 512, 'AutoConfig.from_pretrained("microsoft/phi-1_5", trust_remote_code=True)', 'AutoModelForCausalLM'), 'phi_2' : (512, 512, 'AutoConfig.from_pretrained("microsoft/phi-2", trust_remote_code=True)', 'AutoModelForCausalLM'), + 'moondream' : (512, 512, 'PhiConfig.from_pretrained("vikhyatk/moondream1")', 'PhiForCausalLM'), # as per this page https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1 trust_remote_code=True is not required 'mistral_7b_instruct' : (128, 128, 'AutoConfig.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1")', 'AutoModelForCausalLM'), 'hf_Yi' : (512, 512, 'AutoConfig.from_pretrained("01-ai/Yi-6B", trust_remote_code=True)', 'AutoModelForCausalLM'), @@ -96,7 +98,10 @@ def __init__(self, name, test, device, batch_size=None, extra_args=[]): hugging_face_models_requiring_trust_remote_code = ["hf_Falcon_7b", "hf_MPT_7b_instruct", "phi_1_5", "phi_2", "hf_Yi"] if name in hugging_face_models_requiring_trust_remote_code: kwargs["trust_remote_code"] = True - self.model = class_ctor.from_config(config, **kwargs).to(device) + if hasattr(class_ctor, "from_config"): + self.model = class_ctor.from_config(config, **kwargs).to(device) + else: + self.model = class_ctor(config, **kwargs).to(device) self.optimizer = optim.Adam( self.model.parameters(), lr=0.001, diff --git a/torchbenchmark/util/framework/huggingface/patch_hf.py b/torchbenchmark/util/framework/huggingface/patch_hf.py index f23055be64..0e9f645b6c 100644 --- a/torchbenchmark/util/framework/huggingface/patch_hf.py +++ b/torchbenchmark/util/framework/huggingface/patch_hf.py @@ -5,8 +5,8 @@ import subprocess import sys from .model_factory import class_models -from transformers import AutoConfig, ReformerConfig, BigBirdConfig, BertConfig, WhisperConfig, LlamaConfig - +from transformers import AutoConfig, ReformerConfig, BigBirdConfig, BertConfig, WhisperConfig, LlamaConfig, PhiConfig +import inspect PATCH_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "patches") @@ -14,7 +14,11 @@ def cache_model(name: str, **kwargs): import transformers model_config = eval(class_models[name][2]) model_ctor = getattr(transformers, class_models[name][3]) - model_ctor.from_config(model_config, **kwargs) + if (model_config.__class__.__name__ is "PhiConfig" or "LlavaConfig"): + model_ctor(model_config, **kwargs) + else: + model_ctor.from_config(model_config, **kwargs) + def patch_transformers(): import transformers