ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

support latest phi3 (#11721)

Browse source
This commit is contained in:
Yishuo Wang 2024-08-06 15:52:55 +08:00 committed by GitHub
parent 11650b6f81
commit 929675aa6b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
python/llm/src/ipex_llm/transformers/models/phi3.py
View file

@ -36,9 +36,7 @@ import torch
import warnings import warnings
from torch import nn from torch import nn
from ipex_llm.transformers.models.utils import ( from ipex_llm.transformers.models.utils import should_use_fuse_rope, rotate_half
rotate_half, should_use_fuse_rope,
)
from ipex_llm.transformers.models.utils import mlp_fusion_check, SILU from ipex_llm.transformers.models.utils import mlp_fusion_check, SILU
from ipex_llm.transformers.models.utils import use_sdp, use_sdp_causal from ipex_llm.transformers.models.utils import use_sdp, use_sdp_causal
from ipex_llm.transformers.models.utils import use_quantize_kv_cache, restore_fp8_kv_cache from ipex_llm.transformers.models.utils import use_quantize_kv_cache, restore_fp8_kv_cache
@ -63,7 +61,7 @@ def pre_compute_inv_freq(module: torch.nn.Module):
module.base ** module.base **
(torch.arange(0, module.dim, 2, dtype=torch.int64).float() / module.dim) (torch.arange(0, module.dim, 2, dtype=torch.int64).float() / module.dim)
) )
elif module.__class__.__name__ == "Phi3SuScaledRotaryEmbedding": else:
inv_freq_shape = torch.arange(0, module.dim, 2, dtype=torch.int64).float() / module.dim inv_freq_shape = torch.arange(0, module.dim, 2, dtype=torch.int64).float() / module.dim
short_ext_factors = torch.tensor(module.short_factor, dtype=torch.float32) short_ext_factors = torch.tensor(module.short_factor, dtype=torch.float32)
module.inv_freq = 1.0 / (short_ext_factors * module.base ** inv_freq_shape) module.inv_freq = 1.0 / (short_ext_factors * module.base ** inv_freq_shape)