ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix (#12390)

Browse source
This commit is contained in:
Guancheng Fu 2024-11-27 10:41:58 +08:00 committed by GitHub
parent cb7b08948b
commit 8331875f34
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docker/llm/serving/xpu/docker/vllm_offline_inference.py
View file

@ -54,6 +54,8 @@ llm = LLM(model="YOUR_MODEL",
disable_async_output_proc=True, disable_async_output_proc=True,
distributed_executor_backend="ray", distributed_executor_backend="ray",
max_model_len=2000, max_model_len=2000,
trust_remote_code=True,
block_size=8,
max_num_batched_tokens=2000) max_num_batched_tokens=2000)
# Generate texts from the prompts. The output is a list of RequestOutput objects # Generate texts from the prompts. The output is a list of RequestOutput objects
# that contain the prompt, generated text, and other information. # that contain the prompt, generated text, and other information.