Skip to content

Commit

Permalink
Fix 1383 Llama model on transformers=4.41[WIP] (#11280)
Browse files Browse the repository at this point in the history
  • Loading branch information
songhappy authored Jun 21, 2024
1 parent 475b021 commit 7507000
Show file tree
Hide file tree
Showing 2 changed files with 790 additions and 84 deletions.
31 changes: 22 additions & 9 deletions python/llm/src/ipex_llm/transformers/convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -980,19 +980,32 @@ def _optimize_post(model, lightweight_bmm=False):
convert_forward(model,
transformers.models.llama.modeling_llama.LlamaDecoderLayer,
llama_decoder_forward)

if version.parse(trans_version) >= version.parse("4.36.0"):
# transformers version >= 4.36.0
from ipex_llm.transformers.models.llama import llama_attention_forward_4_38
if version.parse(trans_version) >= version.parse("4.38.0"):
from ipex_llm.transformers.models.llama import llama_model_forward_4_38
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaModel,
llama_model_forward_4_38)
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaAttention,
llama_attention_forward_4_38)
if version.parse(trans_version) >= version.parse("4.41.0"):
from ipex_llm.transformers.models.llama import llama_model_forward_4_41
from ipex_llm.transformers.models.llama import llama_attention_forward_4_41
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaModel,
llama_model_forward_4_41)
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaAttention,
llama_attention_forward_4_41)
else:
from ipex_llm.transformers.models.llama import llama_model_forward_4_38
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaModel,
llama_model_forward_4_38)
convert_forward(
model,
transformers.models.llama.modeling_llama.LlamaAttention,
llama_attention_forward_4_38)
else:
from ipex_llm.transformers.models.llama import llama_model_forward_4_36
convert_forward(
Expand Down
Loading

0 comments on commit 7507000

Please sign in to comment.