ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix No module named 'transformers.cache_utils' with transformers < 4.36 (#10835)

Browse source 
* update sdp condition

* update

* fix

* fix 431 error

* revert sdp & style fix

* fix

* meet comments
This commit is contained in:
Yina Chen 2024-04-22 14:05:50 +08:00 committed by GitHub
parent c6e868f7ad
commit 3daad242b8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 14 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
python/llm/src/ipex_llm/transformers/lookup.py
View file

@ -316,7 +316,8 @@ def lookup_generate(self,
if max_of_max_matched != max_matched: if max_of_max_matched != max_matched:
output_ids = output_ids[:, :max_matched] output_ids = output_ids[:, :max_matched]
new_cache_size = max_of_max_matched - max_matched new_cache_size = max_of_max_matched - max_matched
past_key_values = _crop_past_key_values(self, past_key_values, new_cache_size) past_key_values = _crop_past_key_values(self, past_key_values,
new_cache_size)
input_ids = torch.cat((input_ids, output_ids), dim=-1) input_ids = torch.cat((input_ids, output_ids), dim=-1)

23
python/llm/src/ipex_llm/transformers/speculative.py
View file

@ -451,19 +451,20 @@ def _check_and_extend_kv_cache(past_key_values, max_step_draft, kv_alloc_block_l
def _crop_past_key_values(self, past_key_values, new_cache_size, _enable_ipex=False): def _crop_past_key_values(self, past_key_values, new_cache_size, _enable_ipex=False):
from ipex_llm.transformers.kv import DynamicFp8Cache, DynamicNormalCache if version.parse(trans_version) >= version.parse("4.36.0"):
if isinstance(past_key_values, (DynamicFp8Cache, DynamicNormalCache)): from ipex_llm.transformers.kv import DynamicFp8Cache, DynamicNormalCache
if hasattr(past_key_values, "_seen_tokens"): if isinstance(past_key_values, (DynamicFp8Cache, DynamicNormalCache)):
past_key_values._seen_tokens -= new_cache_size if hasattr(past_key_values, "_seen_tokens"):
else: past_key_values._seen_tokens -= new_cache_size
past_key_values.seen_tokens -= new_cache_size else:
past_key_values.seen_tokens -= new_cache_size
for i, k in enumerate(past_key_values.key_cache): for i, k in enumerate(past_key_values.key_cache):
past_key_values.key_cache[i] = k[:, :, :-new_cache_size, :] past_key_values.key_cache[i] = k[:, :, :-new_cache_size, :]
for i, v in enumerate(past_key_values.value_cache): for i, v in enumerate(past_key_values.value_cache):
past_key_values.value_cache[i] = v[:, :, :-new_cache_size, :] past_key_values.value_cache[i] = v[:, :, :-new_cache_size, :]
return past_key_values return past_key_values
if _enable_ipex: if _enable_ipex:
cur_len = past_key_values[0][0].size(1) cur_len = past_key_values[0][0].size(1)