ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix /generate_stream api in Pipeline Parallel FastAPI (#11569)

Browse source
This commit is contained in:
Xiangyu Tian 2024-07-12 13:19:42 +08:00 committed by GitHub
parent a945500a98
commit 0981b72275
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 5 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
python/llm/example/GPU/Pipeline-Parallel-FastAPI/pipeline_serving.py
View file

@ -192,7 +192,6 @@ async def generate(prompt_request: PromptRequest):
return request_id, "".join(output_str) return request_id, "".join(output_str)
@app.post("/generate_stream/")
async def generate_stream(prompt_request: PromptRequest): async def generate_stream(prompt_request: PromptRequest):
request_id = str(uuid.uuid4()) + "stream" request_id = str(uuid.uuid4()) + "stream"
await local_model.waiting_requests.put((request_id, prompt_request)) await local_model.waiting_requests.put((request_id, prompt_request))
@ -211,6 +210,11 @@ async def generate_stream(prompt_request: PromptRequest):
content=cur_generator, media_type="text/event-stream" content=cur_generator, media_type="text/event-stream"
) )
@app.post("/generate_stream/")
async def generate_stream_api(prompt_request: PromptRequest):
request_id, result = await generate_stream(prompt_request)
return result
DEFAULT_SYSTEM_PROMPT = """\ DEFAULT_SYSTEM_PROMPT = """\
""" """