ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

initial support of IPEX_LLM_PERFORMANCE_MODE (#11754)

Browse source 
* add perf mode

* update

* fix style
This commit is contained in:
Ruonan Wang 2024-08-09 14:04:09 +03:00 committed by GitHub
parent 4b9c57cc60
commit 66fe2ee464
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 9 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
python/llm/dev/benchmark/all-in-one/run.py
View file

@ -53,8 +53,8 @@ def run_model_in_thread(model, in_out, tokenizer, result, warm_up, num_beams, in
for i in range(num_trials + warm_up):
st = time.perf_counter()
if lookahead:
output_ids = model.generate(input_ids, lookahead=3, do_sample=False, max_matching_ngram_size=2, max_new_tokens=out_len,
min_new_tokens=out_len, num_beams=num_beams)
output_ids = model.generate(input_ids, lookahead=2, do_sample=False, max_matching_ngram_size=2, max_new_tokens=out_len,
min_new_tokens=out_len, num_beams=num_beams)
else:
output_ids = model.generate(input_ids, do_sample=False, max_new_tokens=out_len,
min_new_tokens=out_len, num_beams=num_beams)
@ -67,8 +67,8 @@ def run_model_in_thread(model, in_out, tokenizer, result, warm_up, num_beams, in
torch.xpu.empty_cache()
actual_out_len = output_ids.shape[1] - actual_in_len
if i >= warm_up:
if lookahead:
result[in_out].append([model.first_token_time, (end - st - model.first_token_time)/model.n_token_generated, 0,
if lookahead or os.environ.get("IPEX_LLM_PERFORMANCE_MODE", None) == "1":
result[in_out].append([model.first_token_time, (end - st - model.first_token_time)/(model.n_token_generated - 1), 0,
actual_in_len, actual_out_len, load_time, 0])
else:
result[in_out].append([model.first_cost, model.rest_cost_mean, model.encoder_time,
@ -510,7 +510,7 @@ def run_transformer_int4_gpu(repo_id,
load_time = end - st
print(">> loading of model costs {}s and {}GB".format(load_time, torch.xpu.memory.memory_reserved()/(1024**3)))
if not lookahead:
if not lookahead and os.environ.get("IPEX_LLM_PERFORMANCE_MODE", None) != "1":
model = BenchmarkWrapper(model)
result = {}

4
python/llm/src/ipex_llm/transformers/lookup.py
View file

@ -21,6 +21,7 @@
#
from typing import Callable, List, Optional, Tuple
import os
import torch
import time
import copy
@ -54,6 +55,9 @@ def generate(
**kwargs,
):
lookahead = kwargs.pop("lookahead", None)
perf_mode = os.environ.get("IPEX_LLM_PERFORMANCE_MODE", None)
if perf_mode == "1" and lookahead is None:
lookahead = 2 # default to 2 now
if lookahead:
from ipex_llm.transformers.convert import get_enable_ipex
_enable_ipex = get_enable_ipex()