Add OpenVINO performance tests to all-in-one benchmark (#12238)
* add-openvino-to-all-in-one * update on openvino API * Update save_openvino.py * Update save_openvino.py * Update save_openvino.py * update on run.py and save_openvino * update references * Create openvino-requirements.txt * fix on comments * Small updates * Small fix * Fix --------- Co-authored-by: Yuwen Hu <yuwen.hu@intel.com>
This commit is contained in:
Zijie Li
GitHub
parent
ae57e23e4f
commit
f7f62a3fef
4 changed files with 209 additions and 8 deletions
|
|
@ -60,23 +60,38 @@ test_api:
|
|||
# - "speculative_cpu" # on Intel CPU, inference with self-speculative decoding
|
||||
# - "deepspeed_transformer_int4_cpu" # on Intel CPU, deepspeed autotp inference
|
||||
# - "transformers_int4_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4)
|
||||
# - "transformers_openvino" # on Intel GPU, use OpenVINO. Please make sure you have used the save_openvino.py to save the converted OpenVINO model
|
||||
cpu_embedding: False # whether put embedding to CPU
|
||||
streaming: False # whether output in streaming way (only available now for gpu win related test_api)
|
||||
use_fp16_torch_dtype: True # whether use fp16 for non-linear layer (only available now for "pipeline_parallel_gpu" test_api)
|
||||
task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
|
||||
|
||||
group_size: 64 # group_size when converting OpenVINO model (only available or "transformers_openvino" test_api)
|
||||
```
|
||||
|
||||
## (Optional) Save model in low bit
|
||||
If you choose the `transformer_int4_loadlowbit_gpu_win` or `transformer_int4_fp16_loadlowbit_gpu_win` test API, you will need to save the model in low bit first.
|
||||
|
||||
Run `python save.py` will save all models declared in `repo_id` list into low bit models under `local_model_hub` folder.
|
||||
Running `python save.py` will save all models declared in `repo_id` list into low bit models under `local_model_hub` folder.
|
||||
|
||||
## (Optional) Save model for OpenVINO
|
||||
If you choose the `transformers_openvino` test API, you will need to convert the model with OpenVINO first.
|
||||
|
||||
Follow commands below to set up the environment for testing OpenVINO on Intel GPU, in which `requirements.txt` should be downloaded from [here](Download the requirements txt from https://github.com/openvino-dev-samples/Qwen2.openvino/blob/main/requirements.txt):
|
||||
|
||||
```bash
|
||||
conda create -n test-ov python=3.11
|
||||
pip install -r requirements.txt
|
||||
pip install --pre --upgrade ipex-llm # only for IPEX-LLM BenchmarkWrapper
|
||||
pip install accelerate
|
||||
```
|
||||
|
||||
Then, running `python save_openvino.py` will save all models declared in `repo_id` list into OpenVINO models with `low_bit` precision under `local_model_hub` folder.
|
||||
|
||||
## Run
|
||||
|
||||
run `python run.py`, this will output results to `results.csv`.
|
||||
|
||||
For SPR performance, run `bash run-spr.sh`.
|
||||
For IPEX-LLM SPR performance, run `bash run-spr.sh`.
|
||||
|
||||
> **Note**
|
||||
>
|
||||
|
|
@ -86,6 +101,6 @@ For SPR performance, run `bash run-spr.sh`.
|
|||
>
|
||||
> Please install torch nightly version to avoid `Illegal instruction (core dumped)` issue, you can follow the following command to install: `pip install --pre --upgrade torch --index-url https://download.pytorch.org/whl/nightly/cpu`
|
||||
|
||||
For ARC performance, run `bash run-arc.sh`.
|
||||
For IPEX-LLM ARC performance, run `bash run-arc.sh`.
|
||||
|
||||
For MAX GPU performance, run `bash run-max-gpu.sh`.
|
||||
For IPEX-LLM MAX GPU performance, run `bash run-max-gpu.sh`.
|
||||
|
|
|
|||
|
|
@ -37,9 +37,11 @@ test_api:
|
|||
# - "deepspeed_transformer_int4_cpu" # on Intel CPU, deepspeed autotp inference
|
||||
# - "transformers_int4_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4)
|
||||
# - "transformers_int4_loadlowbit_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4), use load_low_bit API. Please make sure you have used the save_npu.py to save the converted low bit model
|
||||
# - "transformers_openvino" # on Intel GPU, use OpenVINO. Please make sure you have used the save_openvino.py to save the converted OpenVINO model
|
||||
cpu_embedding: False # whether put embedding to CPU
|
||||
streaming: False # whether output in streaming way (only available now for gpu win related test_api)
|
||||
optimize_model: False # whether apply further optimization on NPU (only available now for transformers_int4_npu_win test_api)
|
||||
use_fp16_torch_dtype: True # whether use fp16 for non-linear layer (only available now for "pipeline_parallel_gpu" test_api)
|
||||
task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
|
||||
transpose_value_cache: True # whether apply transposed v_cache optimization on NPU (only available now for transformers_int4_npu_win test_api)
|
||||
transpose_value_cache: True # whether apply transposed v_cache optimization on NPU (only available now for transformers_int4_npu_win test_api)
|
||||
group_size: 64 # group_size when converting OpenVINO model (only available or "transformers_openvino" test_api)
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ def preprocess_prompt(tokenizer, in_len, task):
|
|||
input_ids = tokenizer.encode(input_str, return_tensors="pt")
|
||||
return input_ids
|
||||
|
||||
def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1, num_trials=3, num_beams=1, low_bit='sym_int4', cpu_embedding=False, batch_size=1, streaming=False, use_fp16_torch_dtype=False, lookahead=False, task='continuation', optimize_model=False, transpose_value_cache=True):
|
||||
def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1, num_trials=3, num_beams=1, low_bit='sym_int4', cpu_embedding=False, batch_size=1, streaming=False, use_fp16_torch_dtype=False, lookahead=False, task='continuation', optimize_model=False, transpose_value_cache=True, group_size=64):
|
||||
# TODO: make a parameter
|
||||
result= {}
|
||||
if test_api == 'transformer_int4':
|
||||
|
|
@ -193,6 +193,8 @@ def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1,
|
|||
result = transformers_int4_npu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, optimize_model, transpose_value_cache)
|
||||
elif test_api == 'transformers_int4_loadlowbit_npu_win':
|
||||
result = run_transformer_int4_loadlowbit_npu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, optimize_model, transpose_value_cache)
|
||||
elif test_api == 'transformers_openvino':
|
||||
result = run_transformers_openvino(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, group_size)
|
||||
else:
|
||||
invalidInputError(False, "Unknown test_api " + test_api + ", please check your config.yaml.")
|
||||
|
||||
|
|
@ -746,6 +748,78 @@ def run_transformer_int4_loadlowbit_npu_win(repo_id,
|
|||
gc.collect()
|
||||
return result
|
||||
|
||||
def run_transformers_openvino(repo_id,
|
||||
local_model_hub,
|
||||
in_out_pairs,
|
||||
warm_up,
|
||||
num_trials,
|
||||
num_beams,
|
||||
low_bit,
|
||||
batch_size,
|
||||
group_size):
|
||||
from optimum.intel import OVModelForCausalLM
|
||||
from transformers import AutoTokenizer, LlamaTokenizer, PretrainedConfig
|
||||
|
||||
ir_repo_id = (repo_id + '-ov-' + low_bit + '-' +str(group_size))
|
||||
model_path = get_model_path(ir_repo_id, local_model_hub)
|
||||
|
||||
ov_config = {"PERFORMANCE_HINT": "LATENCY",
|
||||
"NUM_STREAMS": "1", "CACHE_DIR": ""}
|
||||
config_dict = dict(pretrained_model_name_or_path=model_path,
|
||||
trust_remote_code=True,
|
||||
use_cache=True, low_cpu_mem_usage=True)
|
||||
|
||||
config = PretrainedConfig(**config_dict)
|
||||
|
||||
# Load model converted by OpenVINO
|
||||
st = time.perf_counter()
|
||||
if repo_id in LLAMA_IDS:
|
||||
model = OVModelForCausalLM.from_pretrained(model_path, device="GPU",
|
||||
ov_config=ov_config, config=config).eval()
|
||||
tokenizer = LlamaTokenizer.from_pretrained(model_path, trust_remote_code=True)
|
||||
else:
|
||||
model = OVModelForCausalLM.from_pretrained(model_path, device="GPU",
|
||||
ov_config=ov_config, config=config).eval()
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
|
||||
end = time.perf_counter()
|
||||
load_time = end - st
|
||||
print(">> loading of model costs {}s".format(load_time))
|
||||
|
||||
model = BenchmarkWrapper(model)
|
||||
|
||||
result = {}
|
||||
with torch.inference_mode():
|
||||
for in_out in in_out_pairs:
|
||||
in_out_len = in_out.split("-")
|
||||
in_len = int(in_out_len[0])
|
||||
out_len = int(in_out_len[1])
|
||||
input_str = get_continuation_input_str(in_len, tokenizer)
|
||||
# As different tokenizer has different encodings,
|
||||
# slice the input_ids to ensure the prompt length is required length.
|
||||
input_ids = tokenizer.encode(input_str, return_tensors="pt")
|
||||
input_ids = input_ids[:, :in_len]
|
||||
true_str = tokenizer.batch_decode(input_ids)[0]
|
||||
input_list = [true_str] * batch_size
|
||||
input_ids = tokenizer(input_list, return_tensors="pt").input_ids
|
||||
input_ids = input_ids[:, :in_len]
|
||||
actual_in_len = input_ids.shape[1]
|
||||
result[in_out] = []
|
||||
for i in range(num_trials + warm_up):
|
||||
st = time.perf_counter()
|
||||
output_ids = model.generate(input_ids, do_sample=False, max_new_tokens=out_len,
|
||||
min_new_tokens=out_len, num_beams=num_beams)
|
||||
end = time.perf_counter()
|
||||
print("model generate cost: " + str(end - st))
|
||||
output = tokenizer.batch_decode(output_ids)
|
||||
print(output[0])
|
||||
actual_out_len = output_ids.shape[1] - actual_in_len
|
||||
if i >= warm_up:
|
||||
result[in_out].append([model.first_cost, model.rest_cost_mean, model.encoder_time,
|
||||
actual_in_len, actual_out_len, load_time])
|
||||
del model
|
||||
gc.collect()
|
||||
return result
|
||||
|
||||
def run_optimize_model_gpu(repo_id,
|
||||
local_model_hub,
|
||||
in_out_pairs,
|
||||
|
|
@ -2108,6 +2182,7 @@ if __name__ == '__main__':
|
|||
use_fp16_torch_dtype = False
|
||||
task = 'continuation'
|
||||
optimize_model = False # only for transformers_int4_npu_win
|
||||
group_size = 64
|
||||
if 'streaming' in conf:
|
||||
streaming = conf['streaming']
|
||||
if 'use_fp16_torch_dtype' in conf:
|
||||
|
|
@ -2116,6 +2191,8 @@ if __name__ == '__main__':
|
|||
task = conf['task']
|
||||
if 'optimize_model' in conf:
|
||||
optimize_model = conf['optimize_model']
|
||||
if 'group_size' in conf:
|
||||
group_size = conf['group_size']
|
||||
lookahead = False
|
||||
transpose_value_cache = True
|
||||
if 'transpose_value_cache' in conf:
|
||||
|
|
@ -2145,7 +2222,7 @@ if __name__ == '__main__':
|
|||
if task in ['QA', 'summarize'] and conf['num_beams'] == 1 and batch_size == 1:
|
||||
lookahead = True
|
||||
run_model(model, api, in_out_pairs, conf['local_model_hub'], conf['warm_up'], conf['num_trials'], conf['num_beams'],
|
||||
conf['low_bit'], conf['cpu_embedding'], batch_size, streaming, use_fp16_torch_dtype, lookahead, task, optimize_model, transpose_value_cache)
|
||||
conf['low_bit'], conf['cpu_embedding'], batch_size, streaming, use_fp16_torch_dtype, lookahead, task, optimize_model, transpose_value_cache, group_size)
|
||||
df = pd.DataFrame(results, columns=['model', '1st token avg latency (ms)', '2+ avg latency (ms/token)', 'encoder time (ms)',
|
||||
'input/output tokens', 'batch_size', 'actual input/output tokens', 'num_beams', 'low_bit', 'cpu_embedding',
|
||||
'model loading time (s)', 'peak mem (GB)', 'streaming', 'use_fp16_torch_dtype'])
|
||||
|
|
|
|||
107
python/llm/dev/benchmark/all-in-one/save_openvino.py
Normal file
107
python/llm/dev/benchmark/all-in-one/save_openvino.py
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
#
|
||||
# Copyright 2016 The BigDL Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Some parts of this file is adapted from
|
||||
# https://github.com/openvino-dev-samples/Qwen2.openvino/blob/main/convert.py
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import warnings
|
||||
|
||||
from transformers import AutoTokenizer, LlamaTokenizer
|
||||
from optimum.intel import OVWeightQuantizationConfig
|
||||
from optimum.intel.openvino import OVModelForCausalLM
|
||||
|
||||
from run import LLAMA_IDS, get_model_path
|
||||
|
||||
current_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
def save_model_to_openvino(repo_id,
|
||||
local_model_hub,
|
||||
low_bit,
|
||||
group_size,
|
||||
):
|
||||
model_path = get_model_path(repo_id, local_model_hub)
|
||||
|
||||
ir_repo_id = (repo_id.split(
|
||||
"/")[1] + '-ov-' + low_bit + '-' +str(group_size))
|
||||
|
||||
if local_model_hub:
|
||||
repo_model_name = repo_id.split(
|
||||
"/")[1] + '-ov-' + low_bit + '-' +str(group_size)
|
||||
ir_model_path = local_model_hub + os.path.sep + repo_model_name
|
||||
ir_model_path = Path(ir_model_path)
|
||||
else:
|
||||
ir_model_path = Path(ir_repo_id)
|
||||
|
||||
if not ir_model_path.exists():
|
||||
os.mkdir(ir_model_path)
|
||||
|
||||
compression_configs = {
|
||||
"sym": True,
|
||||
"group_size": group_size,
|
||||
"ratio": 1.0,
|
||||
}
|
||||
|
||||
print(">> Exporting IR")
|
||||
if low_bit == "sym_int4":
|
||||
compression_configs['sym'] = True
|
||||
ov_model = OVModelForCausalLM.from_pretrained(model_path, export=True,
|
||||
trust_remote_code=True,
|
||||
compile=False, quantization_config=OVWeightQuantizationConfig(
|
||||
bits=4, **compression_configs)).eval()
|
||||
elif low_bit == "asym_int4":
|
||||
compression_configs['sym'] = False
|
||||
ov_model = OVModelForCausalLM.from_pretrained(model_path, export=True,
|
||||
trust_remote_code=True,
|
||||
compile=False, quantization_config=OVWeightQuantizationConfig(
|
||||
bits=4, **compression_configs)).eval()
|
||||
|
||||
print(">> Saving IR")
|
||||
ov_model.save_pretrained(ir_model_path)
|
||||
|
||||
print(">> Exporting tokenizer")
|
||||
if repo_id in LLAMA_IDS:
|
||||
tokenizer = LlamaTokenizer.from_pretrained(model_path,
|
||||
trust_remote_code=True)
|
||||
else:
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_path,
|
||||
trust_remote_code=True)
|
||||
tokenizer.save_pretrained(ir_model_path)
|
||||
|
||||
print(">> Exporting IR tokenizer")
|
||||
from optimum.exporters.openvino.convert import export_tokenizer
|
||||
export_tokenizer(tokenizer, ir_model_path)
|
||||
print(f">> Finished saving OpenVINO IR for {repo_id} in {low_bit} with group size {group_size}")
|
||||
del ov_model
|
||||
del model_path
|
||||
|
||||
if __name__ == '__main__':
|
||||
supported_precision = ["sym_int4", "asym_int4"]
|
||||
|
||||
from omegaconf import OmegaConf
|
||||
conf = OmegaConf.load(f'{current_dir}/config.yaml')
|
||||
|
||||
if conf['low_bit'] in supported_precision:
|
||||
for model in conf.repo_id:
|
||||
save_model_to_openvino(repo_id=model,
|
||||
local_model_hub=conf['local_model_hub'],
|
||||
low_bit=conf['low_bit'],
|
||||
group_size=conf['group_size'],)
|
||||
else:
|
||||
warnings.warn(f"low_bit {conf['low_bit']} is not supported "
|
||||
"in all-in-one benchmark for OpenVINO tests. Only "
|
||||
'sym_int4 and asym_int4 is currently supported for "transformers_openvino" test api.')
|
||||
Loading…
Reference in a new issue