ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix No module named 'transformers.cache_utils' with transformers < 4.36 (#10835)

Browse source 
* update sdp condition

* update

* fix

* fix 431 error

* revert sdp & style fix

* fix

* meet comments
This commit is contained in:
Yina Chen 2024-04-22 14:05:50 +08:00 committed by GitHub
parent c6e868f7ad
commit 3daad242b8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 14 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
python/llm/src/ipex_llm/transformers/lookup.py
View file

@ -316,7 +316,8 @@ def lookup_generate(self,
if max_of_max_matched != max_matched: if max_of_max_matched != max_matched:
output_ids = output_ids[:, :max_matched] output_ids = output_ids[:, :max_matched]
new_cache_size = max_of_max_matched - max_matched new_cache_size = max_of_max_matched - max_matched
past_key_values = _crop_past_key_values(self, past_key_values, new_cache_size) past_key_values = _crop_past_key_values(self, past_key_values,
new_cache_size)
input_ids = torch.cat((input_ids, output_ids), dim=-1) input_ids = torch.cat((input_ids, output_ids), dim=-1)

1
python/llm/src/ipex_llm/transformers/speculative.py
View file

@ -451,6 +451,7 @@ def _check_and_extend_kv_cache(past_key_values, max_step_draft, kv_alloc_block_l
def _crop_past_key_values(self, past_key_values, new_cache_size, _enable_ipex=False): def _crop_past_key_values(self, past_key_values, new_cache_size, _enable_ipex=False):
if version.parse(trans_version) >= version.parse("4.36.0"):
from ipex_llm.transformers.kv import DynamicFp8Cache, DynamicNormalCache from ipex_llm.transformers.kv import DynamicFp8Cache, DynamicNormalCache
if isinstance(past_key_values, (DynamicFp8Cache, DynamicNormalCache)): if isinstance(past_key_values, (DynamicFp8Cache, DynamicNormalCache)):
if hasattr(past_key_values, "_seen_tokens"): if hasattr(past_key_values, "_seen_tokens"):