ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Update for vllm one card padding (#12058)

Browse source
This commit is contained in:
Wang, Jian4 2024-09-11 10:52:55 +08:00 committed by GitHub
parent c5fdfde1bd
commit 30a8680645
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
python/llm/src/ipex_llm/vllm/xpu/model_convert.py
View file

@ -143,6 +143,7 @@ def padding_mlp(module: torch.nn.Module):
new_gate_up_weight[:intermediate_size, :] = gate_up_weight[:intermediate_size, :]
new_gate_up_weight[padding_intermediate_size:padding_intermediate_size+intermediate_size, :] = gate_up_weight[intermediate_size:, :] # noqa
getattr(module, mlp_gate_up_name).output_size_per_partition = padding_intermediate_size * 2
getattr(module, mlp_gate_up_name).output_size = padding_intermediate_size * 2
getattr(module, mlp_gate_up_name).weight = \
torch.nn.Parameter(new_gate_up_weight, requires_grad=False)
@ -151,4 +152,5 @@ def padding_mlp(module: torch.nn.Module):
dtype=down_weight.dtype, device=down_weight.device)
new_down_weight[:, :intermediate_size] = down_weight
getattr(module, mlp_down_name).input_size_per_partition = padding_intermediate_size
getattr(module, mlp_down_name).input_size = padding_intermediate_size
getattr(module, mlp_down_name).weight = torch.nn.Parameter(new_down_weight, requires_grad=False)