ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix qwen nan value issue on vllm (#12971)

Browse source 
* add to fix qwen nan value issue

* update
This commit is contained in:
Wang, Jian4 2025-03-14 14:43:54 +08:00 committed by GitHub
parent cd109bb061
commit c9ecb7a113
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 6 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
python/llm/src/ipex_llm/vllm/xpu/model_convert.py
View file

@ -107,6 +107,12 @@ def get_load_function(low_bit):
modules = ["35.mlp", "36.mlp", "37.mlp", "38.mlp", "39.mlp"] modules = ["35.mlp", "36.mlp", "37.mlp", "38.mlp", "39.mlp"]
else: else:
modules = None modules = None
not_convert_o_proj = os.getenv("IPEX_LLM_NOT_CONVERT_O_PROJ", None)
if not_convert_o_proj is not None:
# only use to avoid nan value in o_proj running DeepSeek-R1-Distill-Qwen-14B
modules = ["o_proj"]
else:
modules = None
if "minicpm" in self.vllm_config.model_config.model.lower(): if "minicpm" in self.vllm_config.model_config.model.lower():
modules = ["vpm", "resampler"] modules = ["vpm", "resampler"]
if "internvl2" in self.vllm_config.model_config.model.lower(): if "internvl2" in self.vllm_config.model_config.model.lower():