ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

add stream interval option to entrypoint (#9498)

Browse source
This commit is contained in:
Lilac09 2023-11-21 09:47:32 +08:00 committed by GitHub
parent c5cb3ab82e
commit 566ec85113
1 changed files with 7 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
docker/llm/serving/cpu/docker/entrypoint.sh
View file

@ -14,6 +14,7 @@ usage() {
echo "WORKER_HOST (default: localhost)." echo "WORKER_HOST (default: localhost)."
echo "WORKER_PORT (default: 21002)." echo "WORKER_PORT (default: 21002)."
echo "MODEL_PATH (default: empty)." echo "MODEL_PATH (default: empty)."
echo "STREAM_INTERVAL (default: 1)."
exit 1 exit 1
} }
@ -83,6 +84,7 @@ model_path=""
mode="" mode=""
omp_num_threads="" omp_num_threads=""
dispatch_method="shortest_queue" # shortest_queue or lottery dispatch_method="shortest_queue" # shortest_queue or lottery
stream_interval=1
# Update rootCA config if needed # Update rootCA config if needed
update-ca-certificates update-ca-certificates
@ -157,6 +159,10 @@ else
dispatch_method=$DISPATCH_METHOD dispatch_method=$DISPATCH_METHOD
fi fi
if [[ -n $STREAM_INTERVAL ]]; then
stream_interval=$STREAM_INTERVAL
fi
controller_address="http://$controller_host:$controller_port" controller_address="http://$controller_host:$controller_port"
# Execute logic based on options # Execute logic based on options
if [[ $mode == "controller" ]]; then if [[ $mode == "controller" ]]; then
@ -194,7 +200,7 @@ else
fi fi
echo "Worker address: $worker_address" echo "Worker address: $worker_address"
echo "Controller address: $controller_address" echo "Controller address: $controller_address"
python3 -m fastchat.serve.model_worker --model-path $model_path --device cpu --host $worker_host --port $worker_port --worker-address $worker_address --controller-address $controller_address python3 -m fastchat.serve.model_worker --model-path $model_path --device cpu --host $worker_host --port $worker_port --worker-address $worker_address --controller-address $controller_address --stream-interval $stream_interval
fi fi
fi fi