diff --git a/python/llm/src/ipex_llm/transformers/pipeline_parallel.py b/python/llm/src/ipex_llm/transformers/pipeline_parallel.py
index 0812a12a..bbe04f12 100644
--- a/python/llm/src/ipex_llm/transformers/pipeline_parallel.py
+++ b/python/llm/src/ipex_llm/transformers/pipeline_parallel.py
@@ -53,7 +53,7 @@ class DummyLayer(nn.Module):
         super().__init__()
         # to avoid AttributeError in https://github.com/intel-analytics/ipex-llm/blob/main/
         # python/llm/src/ipex_llm/transformers/models/llama.py#L2076
-        self.weight = torch.randn(1,)
+        self.weight = nn.Parameter(torch.empty(0,), requires_grad=False)
 
     def forward(self, x):
         return x