LLM: replace torch.float32 with auto type (#9261)

2023-10-24 17:12:13 +08:00 · 2023-10-24 17:12:13 +08:00 · 90162264a3
commit 90162264a3
parent bd5215d75b
1 changed files with 1 additions and 1 deletions
--- a/python/llm/src/bigdl/llm/serving/bigdl_llm_model.py
+++ b/python/llm/src/bigdl/llm/serving/bigdl_llm_model.py
@ -104,7 +104,7 @@ def load_model(
        device, load_8bit, cpu_offloading
    )
    if device == "cpu":
-        kwargs = {"torch_dtype": torch.float32}
+        kwargs = {"torch_dtype": "auto"}
        if CPU_ISA in ["avx512_bf16", "amx"]:
            try:
                import intel_extension_for_pytorch as ipex