ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix lightweight-serving no streaming resp on mtl (#11822)

Browse source
This commit is contained in:
Wang, Jian4 2024-08-16 09:43:03 +08:00 committed by GitHub
parent e70ae0638e
commit 5a80fd2633
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
python/llm/src/ipex_llm/serving/fastapi/api_server.py
View file

@ -125,6 +125,7 @@ async def chat_stream_generator(local_model, delta_text_queue, request_id):
model_name = local_model.model_name
index = 0
while True:
await asyncio.sleep(0)
if not hasattr(delta_text_queue, 'empty'):
delta_text, remain = get_queue_next_token(delta_text_queue)
else:
@ -168,6 +169,7 @@ async def completion_stream_generator(local_model, delta_text_queue, request_id)
model_name = local_model.model_name
index = 0
while True:
await asyncio.sleep(0)
if not hasattr(delta_text_queue, 'empty'):
delta_text, remain = get_queue_next_token(delta_text_queue)
else: