ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Remove tgi parameter validation (#11688)

Browse source 
* remove validation

* add min warm up

* remove no need source
This commit is contained in:
Wang, Jian4 2024-07-30 16:37:44 +08:00 committed by GitHub
parent 670ad887fc
commit b119825152
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 2 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docker/llm/serving/xpu/docker/start-lightweight_serving-service.sh
View file

@ -1,5 +1,4 @@
cd /llm/lightweight_serving cd /llm/lightweight_serving
model_path="/llm/models/Llama-2-7b-chat-hf" model_path="/llm/models/Llama-2-7b-chat-hf"
low_bit="sym_int4" low_bit="sym_int4"
source /opt/intel/1ccl-wks/setvars.sh
python lightweight_serving.py --repo-id-or-model-path $model_path --low-bit $low_bit python lightweight_serving.py --repo-id-or-model-path $model_path --low-bit $low_bit

2
docker/llm/serving/xpu/docker/vllm_online_benchmark.py
View file

@ -281,6 +281,8 @@ for MAX_CONCURRENT_REQUESTS in [max_batch]:
NUM_WARMUP = 2 * MAX_CONCURRENT_REQUESTS NUM_WARMUP = 2 * MAX_CONCURRENT_REQUESTS
NUM_REQUESTS = 5 * MAX_CONCURRENT_REQUESTS # 总请求次数 NUM_REQUESTS = 5 * MAX_CONCURRENT_REQUESTS # 总请求次数
# to avoid warm_up time out
benchmark(LLM_URLS, MODEL, PROMPT_1024, 2, 1, 32, is_warmup = True)
benchmark(LLM_URLS, MODEL, PROMPT, NUM_WARMUP, MAX_CONCURRENT_REQUESTS, MAX_TOKENS, is_warmup = True) benchmark(LLM_URLS, MODEL, PROMPT, NUM_WARMUP, MAX_CONCURRENT_REQUESTS, MAX_TOKENS, is_warmup = True)
# 运行benchmark # 运行benchmark

30
python/llm/src/ipex_llm/serving/fastapi/tgi_protocol.py
View file

@ -31,33 +31,3 @@ class Parameters(BaseModel):
top_k: Optional[int] = None top_k: Optional[int] = None
top_p: Optional[float] = None top_p: Optional[float] = None
typical_p: Optional[float] = None typical_p: Optional[float] = None
@field_validator("repetition_penalty")
def valid_repetition_penalty(cls, v):
if v is not None and v <= 0:
invalidInputError(False, "`repetition_penalty` must be strictly positive")
return v
@field_validator("temperature")
def valid_temp(cls, v):
if v is not None and v <= 0:
invalidInputError(False, "`temperature` must be strictly positive")
return v
@field_validator("top_k")
def valid_top_k(cls, v):
if v is not None and v <= 0:
invalidInputError(False, "`top_k` must be strictly positive")
return v
@field_validator("top_p")
def valid_top_p(cls, v):
if v is not None and (v <= 0 or v >= 1.0):
invalidInputError(False, "`top_p` must be > 0.0 and < 1.0")
return v
@field_validator("typical_p")
def valid_typical_p(cls, v):
if v is not None and (v <= 0 or v >= 1.0):
invalidInputError(False, "`typical_p` must be > 0.0 and < 1.0")
return v