From 37bf22cb3449fe8253904bb8aa8c28e9fe2df2b4 Mon Sep 17 00:00:00 2001 From: Margaret Qian Date: Fri, 30 Jun 2023 10:06:32 -0700 Subject: [PATCH 1/2] fix --- examples/inference-deployments/mpt/mpt_30b_ft.yaml | 3 +++ examples/inference-deployments/mpt/mpt_30b_instruct_ft.yaml | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/inference-deployments/mpt/mpt_30b_ft.yaml b/examples/inference-deployments/mpt/mpt_30b_ft.yaml index 9ccec3ab8..c3abb6e42 100644 --- a/examples/inference-deployments/mpt/mpt_30b_ft.yaml +++ b/examples/inference-deployments/mpt/mpt_30b_ft.yaml @@ -20,7 +20,10 @@ model: downloader: examples.inference-deployments.mpt.mpt_ft_handler.download_convert download_parameters: hf_path: mosaicml/mpt-30b + gpus: 2 + force_conversion: true model_handler: examples.inference-deployments.mpt.mpt_ft_handler.MPTFTModelHandler model_parameters: model_name_or_path: mosaicml/mpt-30b ft_lib_path: /code/FasterTransformer/build/lib/libth_transformer.so + gpus: 2 diff --git a/examples/inference-deployments/mpt/mpt_30b_instruct_ft.yaml b/examples/inference-deployments/mpt/mpt_30b_instruct_ft.yaml index 43db81061..b9178b99e 100644 --- a/examples/inference-deployments/mpt/mpt_30b_instruct_ft.yaml +++ b/examples/inference-deployments/mpt/mpt_30b_instruct_ft.yaml @@ -2,7 +2,7 @@ name: mpt-30b-instruct-ft compute: gpus: 2 gpu_type: a100_40gb -image: mosaicml/inference:0.1.1 +image: mosaicml/inference:0.1.4 replicas: 1 command: | export PYTHONPATH=/code/llm-foundry:/code/examples:/code @@ -20,7 +20,10 @@ model: downloader: examples.inference-deployments.mpt.mpt_ft_handler.download_convert download_parameters: hf_path: mosaicml/mpt-30b-instruct + gpus: 2 + force_conversion: true model_handler: examples.inference-deployments.mpt.mpt_ft_handler.MPTFTModelHandler model_parameters: model_name_or_path: mosaicml/mpt-30b-instruct ft_lib_path: /code/FasterTransformer/build/lib/libth_transformer.so + gpus: 2 From b6d30e8bc655a5a6cde253fbb17a68801b6f48d4 Mon Sep 17 00:00:00 2001 From: Margaret Qian Date: Fri, 30 Jun 2023 10:07:08 -0700 Subject: [PATCH 2/2] bump image --- examples/inference-deployments/mpt/mpt_30b_ft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/inference-deployments/mpt/mpt_30b_ft.yaml b/examples/inference-deployments/mpt/mpt_30b_ft.yaml index c3abb6e42..11e8ea3ed 100644 --- a/examples/inference-deployments/mpt/mpt_30b_ft.yaml +++ b/examples/inference-deployments/mpt/mpt_30b_ft.yaml @@ -2,7 +2,7 @@ name: mpt-30b-ft compute: gpus: 2 gpu_type: a100_40gb -image: mosaicml/inference:0.1.1 +image: mosaicml/inference:0.1.4 replicas: 1 command: | export PYTHONPATH=/code/llm-foundry:/code/examples:/code