ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

vLLM: set convert_to_half to False by default (#13172)

Browse source 
* init

* remove

* fix
This commit is contained in:
Xiangyu Tian 2025-05-21 18:41:28 +08:00 committed by GitHub
parent 1576347892
commit 154af7d7f7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
python/llm/src/ipex_llm/transformers/convert.py
View file

@ -293,6 +293,7 @@ def convert_vllm(module, qtype, in_features, out_features, mp_group, cur_qtype,
mp_group=mp_group,
optimize_lm_head=optimize_lm_head,
enable_scale_search=enable_scale_search,
conver_to_half=False,
)
return new_linear
@ -589,6 +590,7 @@ def _replace_with_low_bit_linear(model, qtype, modules_to_not_convert=None,
optimize_lm_head=False,
act_order=act_order,
enable_scale_search=enable_scale_search,
conver_to_half=False,
)
device = module.qweight.data.device
invalidInputError(device.type != "meta",

2
python/llm/src/ipex_llm/transformers/low_bit_linear.py
View file

@ -654,7 +654,7 @@ class LowBitLinear(nn.Linear):
else:
w = self.weight.data
if use_batch_forward(x_2d, self.weight.qtype, self.out_len):
if use_batch_forward(x_2d, self.weight.qtype, self.out_len) and self.conver_to_half:
import xe_batch
result = xe_batch.batch_forward(x_2d, w, self.qtype)
elif not is_training and self.conver_to_half \