ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix vLLM CPU api_server params (#11384)

Browse source
This commit is contained in:
Xiangyu Tian 2024-06-21 13:00:06 +08:00 committed by GitHub
parent 21fc781fce
commit b30bf7648e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
python/llm/src/ipex_llm/vllm/cpu/entrypoints/openai/api_server.py
View file

@ -175,7 +175,9 @@ if __name__ == "__main__":
served_model_names = [args.model] served_model_names = [args.model]
engine_args = AsyncEngineArgs.from_cli_args(args) engine_args = AsyncEngineArgs.from_cli_args(args)
engine = IPEXLLMAsyncLLMEngine.from_engine_args( engine = IPEXLLMAsyncLLMEngine.from_engine_args(
engine_args, usage_context=UsageContext.OPENAI_API_SERVER) engine_args, usage_context=UsageContext.OPENAI_API_SERVER,
load_in_low_bit=args.load_in_low_bit,
)
openai_serving_chat = OpenAIServingChat(engine, served_model_names, openai_serving_chat = OpenAIServingChat(engine, served_model_names,
args.response_role, args.response_role,
args.lora_modules, args.lora_modules,