ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix starcoder first token perf (#10612)

Browse source 
* add bias check

* update
This commit is contained in:
Kai Huang 2024-04-02 09:21:38 +08:00 committed by GitHub
parent e567956121
commit 0a95c556a1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
python/llm/src/ipex_llm/transformers/low_bit_linear.py
View file

@ -538,7 +538,7 @@ class LowBitLinear(nn.Linear):
# The condition makes sure that empty cache only takes effect if this layer is lm_head. # The condition makes sure that empty cache only takes effect if this layer is lm_head.
# For other models like llama, lm_cache will be applied as well # For other models like llama, lm_cache will be applied as well
# since performance isn't impacted. # since performance isn't impacted.
self.is_lm_head = self.in_len * self.out_len >= 30000 * 4096 self.is_lm_head = self.in_len * self.out_len >= 32000 * 4096 and self.bias is None
self.low_memory_mode = self.is_lm_head self.low_memory_mode = self.is_lm_head
def forward(self, x: torch.Tensor): def forward(self, x: torch.Tensor):