ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Enable internvl2-8b on vllm(#12645)

Browse source
This commit is contained in:
Wang, Jian4 2025-01-03 14:49:36 +08:00 committed by GitHub
parent 8fd2dcba86
commit 6711a48a36
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
python/llm/src/ipex_llm/vllm/xpu/model_convert.py
View file

@ -105,6 +105,8 @@ def get_load_function(low_bit):
if "minicpm-v" in self.model_config.model.lower(): if "minicpm-v" in self.model_config.model.lower():
from ipex_llm.transformers.models.minicpmv import merge_qkv from ipex_llm.transformers.models.minicpmv import merge_qkv
self.model.vpm.apply(merge_qkv) self.model.vpm.apply(merge_qkv)
if "internvl2" in self.model_config.model.lower():
modules = ["vision_model", "mlp1"]
optimize_model(self.model, low_bit=low_bit, torch_dtype=self.model_config.dtype, optimize_model(self.model, low_bit=low_bit, torch_dtype=self.model_config.dtype,
modules_to_not_convert=modules) modules_to_not_convert=modules)
self.model = self.model.to(device=self.device_config.device, self.model = self.model.to(device=self.device_config.device,