From 90162264a3d29c85b54d8cb312ec7a31cc7e26e2 Mon Sep 17 00:00:00 2001
From: Jin Qiao <89779290+JinBridger@users.noreply.github.com>
Date: Tue, 24 Oct 2023 17:12:13 +0800
Subject: [PATCH] LLM: replace torch.float32 with auto type (#9261)

---
 python/llm/src/bigdl/llm/serving/bigdl_llm_model.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/llm/src/bigdl/llm/serving/bigdl_llm_model.py b/python/llm/src/bigdl/llm/serving/bigdl_llm_model.py
index 6c8cc780..c4716b5e 100644
--- a/python/llm/src/bigdl/llm/serving/bigdl_llm_model.py
+++ b/python/llm/src/bigdl/llm/serving/bigdl_llm_model.py
@@ -104,7 +104,7 @@ def load_model(
         device, load_8bit, cpu_offloading
     )
     if device == "cpu":
-        kwargs = {"torch_dtype": torch.float32}
+        kwargs = {"torch_dtype": "auto"}
         if CPU_ISA in ["avx512_bf16", "amx"]:
             try:
                 import intel_extension_for_pytorch as ipex