ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Update IPEX_LLM_PERFORMANCE_MODE (#11823)

Browse source
This commit is contained in:
Yuwen Hu 2024-08-16 09:48:36 +08:00 committed by GitHub
parent 5a80fd2633
commit 9e9086cc2a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 11 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
python/llm/src/ipex_llm/transformers/models/utils.py
View file

@ -487,6 +487,10 @@ def update_past_key_value(past_key_value, key_states, value_states,
def should_use_compresskv(x: torch.Tensor, prompt_len: int): def should_use_compresskv(x: torch.Tensor, prompt_len: int):
use_compress_kv = os.environ.get("IPEX_LLM_COMPRESS_KV_CACHE", None) use_compress_kv = os.environ.get("IPEX_LLM_COMPRESS_KV_CACHE", None)
perf_mode = os.environ.get("IPEX_LLM_PERFORMANCE_MODE", None)
if perf_mode == "1":
return False
else:
if use_compress_kv is None: if use_compress_kv is None:
return ( return (
get_xpu_device_type(x) == "mtl" get_xpu_device_type(x) == "mtl"