enable inference mode for deepspeed tp serving (#11742)

2024-08-08 14:38:30 +08:00 · 2024-08-08 14:38:30 +08:00 · 107f7aafd0
commit 107f7aafd0
parent 9e65cf00b3
1 changed files with 3 additions and 1 deletions
--- a/python/llm/example/GPU/Deepspeed-AutoTP-FastAPI/serving.py
+++ b/python/llm/example/GPU/Deepspeed-AutoTP-FastAPI/serving.py
@ -121,6 +121,8 @@ def load_model(model_path, low_bit):
    current_accel = XPU_Accelerator()
    set_accelerator(current_accel)

+    model=model.eval()
+    
    # Move model back to xpu
    model = model.to(f"xpu:{local_rank}")
    model = BenchmarkWrapper(model)