ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Update lm_head optimization for Qwen2 7B (#12090)

Browse source
This commit is contained in:
Yuwen Hu 2024-09-18 17:02:02 +08:00 committed by GitHub
parent ee33b93464
commit f7fb3c896c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 3 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
python/llm/src/ipex_llm/transformers/npu_models/convert_mp.py
View file

@ -89,10 +89,10 @@ def optimize_llm_pre(model: torch.nn.Module, qtype):
from ipex_llm.transformers.npu_models.qwen2_mp import split_mlp_down_proj
model.apply(split_mlp_down_proj)
# for Qwen2-7B-Insturct, divide lm_head into 7 parts
# for Qwen2-7B-Insturct, divide lm_head into 14 parts
if model.config.hidden_size == 3584 and model.config.vocab_size == 152064 and \
not cpu_lm_head:
new_lm_head = SlicedLMHead(model.lm_head.weight, split_num=7,
new_lm_head = SlicedLMHead(model.lm_head.weight, split_num=14,
bias=model.lm_head.bias)
del model.lm_head
model.lm_head = new_lm_head
@ -192,7 +192,7 @@ def optimize_llm(
from ipex_llm.transformers.npu_models.qwen2_mp import qwen2_casullm_forward
convert_forward(model, Qwen2ForCausalLM, qwen2_casullm_forward)
# for Qwen2-7B-Insturct, divide lm_head into 7 parts
# for Qwen2-7B-Insturct, divide lm_head into 14 parts
if model.config.hidden_size == 3584 and model.config.vocab_size == 152064 and \
isinstance(model.lm_head, SlicedLMHead):
model.lm_head.get_fused_lm_head()