Add half precision for fastchat models (#11130)

2024-05-24 15:41:14 +08:00 · 2024-05-24 15:41:14 +08:00 · 7f772c5a4f
commit 7f772c5a4f
parent 65f4212f89
1 changed files with 2 additions and 0 deletions
--- a/python/llm/src/ipex_llm/transformers/loader.py
+++ b/python/llm/src/ipex_llm/transformers/loader.py
@ -91,6 +91,8 @@ def load_model(

    if device == "xpu":
        import intel_extension_for_pytorch as ipex
+        print("Convert model to half precision...")
+        model = model.half()
        model = model.to('xpu')

    return model, tokenizer