diff --git a/python/llm/src/ipex_llm/vllm/xpu/model_convert.py b/python/llm/src/ipex_llm/vllm/xpu/model_convert.py
index 7979bfbc..065652e7 100644
--- a/python/llm/src/ipex_llm/vllm/xpu/model_convert.py
+++ b/python/llm/src/ipex_llm/vllm/xpu/model_convert.py
@@ -250,7 +250,8 @@ def get_load_function(low_bit):
             from ipex_llm import optimize_model
             import os
             not_convert_last_mlp = os.getenv("IPEX_LLM_NOT_CONVERT_LAST_MLP", None)
-            if not_convert_last_mlp is not None:
+            is_glm4_model = "glm-4" in self.model_config.model.lower()
+            if not_convert_last_mlp is not None or is_glm4_model:
                 # only use to avoid nan value in last mlp forward running glm4-9b-chat
                 modules = ["35.mlp", "36.mlp", "37.mlp", "38.mlp", "39.mlp"]
             else: