ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

empty cache only for 1st token but rest token to speed up (#11665)

Browse source
This commit is contained in:
Heyang Sun 2024-07-26 16:46:21 +08:00 committed by GitHub
parent fc7f8feb83
commit ba01b85c13
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 6 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
python/llm/src/ipex_llm/transformers/pipeline_parallel.py
View file

@ -959,9 +959,13 @@ def llama_causallm_forward_4_37_lowmem(
logits = [F.linear(hidden_states, lm_head_slices[i]) for i in range(self.config.pretraining_tp)] # noqa logits = [F.linear(hidden_states, lm_head_slices[i]) for i in range(self.config.pretraining_tp)] # noqa
logits = torch.cat(logits, dim=-1) logits = torch.cat(logits, dim=-1)
else: else:
torch.xpu.empty_cache() # Only empty cache for first token
if hidden_states.shape[1] > 1:
torch.xpu.empty_cache()
logits = self.lm_head(hidden_states) logits = self.lm_head(hidden_states)
torch.xpu.empty_cache() # Only empty cache for first token
if hidden_states.shape[1] > 1:
torch.xpu.empty_cache()
# logits = logits.float() # logits = logits.float()
# ipex-llm change ends # ipex-llm change ends