ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

use quantize kv and sdp in phi3-mini (#10973)

Browse source
This commit is contained in:
Yishuo Wang 2024-05-09 15:16:18 +08:00 committed by GitHub
parent e3159c45e4
commit 697ca79eca
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
python/llm/src/ipex_llm/transformers/models/phi3.py
View file

@ -228,10 +228,9 @@ def model_forward_wrapper(origin_model_forward):
output_hidden_states: Optional[bool] = None,
return_dict: Optional[bool] = None,
):
# IPEX-LLM OPT: kv cache but no sdp (its head_dim 96, cannot use sdp)
# IPEX-LLM OPT: kv cache and quantize kv cache and sdp
use_cache = use_cache if use_cache is not None else self.config.use_cache
use_quantize_kv = (use_quantize_kv_cache(self.layers[0].mlp.down_proj, input_ids) and
self.config.hidden_size // self.config.num_attention_heads in [64, 128])
use_quantize_kv = use_quantize_kv_cache(self.layers[0].mlp.down_proj, input_ids)
if use_cache:
if use_quantize_kv and not isinstance(past_key_values, DynamicFp8Cache):
past_key_values = DynamicFp8Cache.from_legacy_cache(past_key_values)