diff --git a/python/llm/dev/benchmark/all-in-one/README.md b/python/llm/dev/benchmark/all-in-one/README.md index d17332b4..09a5b783 100644 --- a/python/llm/dev/benchmark/all-in-one/README.md +++ b/python/llm/dev/benchmark/all-in-one/README.md @@ -60,23 +60,38 @@ test_api: # - "speculative_cpu" # on Intel CPU, inference with self-speculative decoding # - "deepspeed_transformer_int4_cpu" # on Intel CPU, deepspeed autotp inference # - "transformers_int4_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4) + # - "transformers_openvino" # on Intel GPU, use OpenVINO. Please make sure you have used the save_openvino.py to save the converted OpenVINO model cpu_embedding: False # whether put embedding to CPU streaming: False # whether output in streaming way (only available now for gpu win related test_api) use_fp16_torch_dtype: True # whether use fp16 for non-linear layer (only available now for "pipeline_parallel_gpu" test_api) task: 'continuation' # task can be 'continuation', 'QA' and 'summarize' - +group_size: 64 # group_size when converting OpenVINO model (only available or "transformers_openvino" test_api) ``` ## (Optional) Save model in low bit If you choose the `transformer_int4_loadlowbit_gpu_win` or `transformer_int4_fp16_loadlowbit_gpu_win` test API, you will need to save the model in low bit first. -Run `python save.py` will save all models declared in `repo_id` list into low bit models under `local_model_hub` folder. +Running `python save.py` will save all models declared in `repo_id` list into low bit models under `local_model_hub` folder. + +## (Optional) Save model for OpenVINO +If you choose the `transformers_openvino` test API, you will need to convert the model with OpenVINO first. + +Follow commands below to set up the environment for testing OpenVINO on Intel GPU, in which `requirements.txt` should be downloaded from [here](Download the requirements txt from https://github.com/openvino-dev-samples/Qwen2.openvino/blob/main/requirements.txt): + +```bash +conda create -n test-ov python=3.11 +pip install -r requirements.txt +pip install --pre --upgrade ipex-llm # only for IPEX-LLM BenchmarkWrapper +pip install accelerate +``` + +Then, running `python save_openvino.py` will save all models declared in `repo_id` list into OpenVINO models with `low_bit` precision under `local_model_hub` folder. ## Run run `python run.py`, this will output results to `results.csv`. -For SPR performance, run `bash run-spr.sh`. +For IPEX-LLM SPR performance, run `bash run-spr.sh`. > **Note** > @@ -86,6 +101,6 @@ For SPR performance, run `bash run-spr.sh`. > > Please install torch nightly version to avoid `Illegal instruction (core dumped)` issue, you can follow the following command to install: `pip install --pre --upgrade torch --index-url https://download.pytorch.org/whl/nightly/cpu` -For ARC performance, run `bash run-arc.sh`. +For IPEX-LLM ARC performance, run `bash run-arc.sh`. -For MAX GPU performance, run `bash run-max-gpu.sh`. +For IPEX-LLM MAX GPU performance, run `bash run-max-gpu.sh`. diff --git a/python/llm/dev/benchmark/all-in-one/config.yaml b/python/llm/dev/benchmark/all-in-one/config.yaml index dd7a876d..ddccd4d9 100644 --- a/python/llm/dev/benchmark/all-in-one/config.yaml +++ b/python/llm/dev/benchmark/all-in-one/config.yaml @@ -37,9 +37,11 @@ test_api: # - "deepspeed_transformer_int4_cpu" # on Intel CPU, deepspeed autotp inference # - "transformers_int4_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4) # - "transformers_int4_loadlowbit_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4), use load_low_bit API. Please make sure you have used the save_npu.py to save the converted low bit model + # - "transformers_openvino" # on Intel GPU, use OpenVINO. Please make sure you have used the save_openvino.py to save the converted OpenVINO model cpu_embedding: False # whether put embedding to CPU streaming: False # whether output in streaming way (only available now for gpu win related test_api) optimize_model: False # whether apply further optimization on NPU (only available now for transformers_int4_npu_win test_api) use_fp16_torch_dtype: True # whether use fp16 for non-linear layer (only available now for "pipeline_parallel_gpu" test_api) task: 'continuation' # task can be 'continuation', 'QA' and 'summarize' -transpose_value_cache: True # whether apply transposed v_cache optimization on NPU (only available now for transformers_int4_npu_win test_api) \ No newline at end of file +transpose_value_cache: True # whether apply transposed v_cache optimization on NPU (only available now for transformers_int4_npu_win test_api) +group_size: 64 # group_size when converting OpenVINO model (only available or "transformers_openvino" test_api) diff --git a/python/llm/dev/benchmark/all-in-one/run.py b/python/llm/dev/benchmark/all-in-one/run.py index 9420609d..e9e4e740 100644 --- a/python/llm/dev/benchmark/all-in-one/run.py +++ b/python/llm/dev/benchmark/all-in-one/run.py @@ -138,7 +138,7 @@ def preprocess_prompt(tokenizer, in_len, task): input_ids = tokenizer.encode(input_str, return_tensors="pt") return input_ids -def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1, num_trials=3, num_beams=1, low_bit='sym_int4', cpu_embedding=False, batch_size=1, streaming=False, use_fp16_torch_dtype=False, lookahead=False, task='continuation', optimize_model=False, transpose_value_cache=True): +def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1, num_trials=3, num_beams=1, low_bit='sym_int4', cpu_embedding=False, batch_size=1, streaming=False, use_fp16_torch_dtype=False, lookahead=False, task='continuation', optimize_model=False, transpose_value_cache=True, group_size=64): # TODO: make a parameter result= {} if test_api == 'transformer_int4': @@ -193,6 +193,8 @@ def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1, result = transformers_int4_npu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, optimize_model, transpose_value_cache) elif test_api == 'transformers_int4_loadlowbit_npu_win': result = run_transformer_int4_loadlowbit_npu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, optimize_model, transpose_value_cache) + elif test_api == 'transformers_openvino': + result = run_transformers_openvino(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, group_size) else: invalidInputError(False, "Unknown test_api " + test_api + ", please check your config.yaml.") @@ -746,6 +748,78 @@ def run_transformer_int4_loadlowbit_npu_win(repo_id, gc.collect() return result +def run_transformers_openvino(repo_id, + local_model_hub, + in_out_pairs, + warm_up, + num_trials, + num_beams, + low_bit, + batch_size, + group_size): + from optimum.intel import OVModelForCausalLM + from transformers import AutoTokenizer, LlamaTokenizer, PretrainedConfig + + ir_repo_id = (repo_id + '-ov-' + low_bit + '-' +str(group_size)) + model_path = get_model_path(ir_repo_id, local_model_hub) + + ov_config = {"PERFORMANCE_HINT": "LATENCY", + "NUM_STREAMS": "1", "CACHE_DIR": ""} + config_dict = dict(pretrained_model_name_or_path=model_path, + trust_remote_code=True, + use_cache=True, low_cpu_mem_usage=True) + + config = PretrainedConfig(**config_dict) + + # Load model converted by OpenVINO + st = time.perf_counter() + if repo_id in LLAMA_IDS: + model = OVModelForCausalLM.from_pretrained(model_path, device="GPU", + ov_config=ov_config, config=config).eval() + tokenizer = LlamaTokenizer.from_pretrained(model_path, trust_remote_code=True) + else: + model = OVModelForCausalLM.from_pretrained(model_path, device="GPU", + ov_config=ov_config, config=config).eval() + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + end = time.perf_counter() + load_time = end - st + print(">> loading of model costs {}s".format(load_time)) + + model = BenchmarkWrapper(model) + + result = {} + with torch.inference_mode(): + for in_out in in_out_pairs: + in_out_len = in_out.split("-") + in_len = int(in_out_len[0]) + out_len = int(in_out_len[1]) + input_str = get_continuation_input_str(in_len, tokenizer) + # As different tokenizer has different encodings, + # slice the input_ids to ensure the prompt length is required length. + input_ids = tokenizer.encode(input_str, return_tensors="pt") + input_ids = input_ids[:, :in_len] + true_str = tokenizer.batch_decode(input_ids)[0] + input_list = [true_str] * batch_size + input_ids = tokenizer(input_list, return_tensors="pt").input_ids + input_ids = input_ids[:, :in_len] + actual_in_len = input_ids.shape[1] + result[in_out] = [] + for i in range(num_trials + warm_up): + st = time.perf_counter() + output_ids = model.generate(input_ids, do_sample=False, max_new_tokens=out_len, + min_new_tokens=out_len, num_beams=num_beams) + end = time.perf_counter() + print("model generate cost: " + str(end - st)) + output = tokenizer.batch_decode(output_ids) + print(output[0]) + actual_out_len = output_ids.shape[1] - actual_in_len + if i >= warm_up: + result[in_out].append([model.first_cost, model.rest_cost_mean, model.encoder_time, + actual_in_len, actual_out_len, load_time]) + del model + gc.collect() + return result + def run_optimize_model_gpu(repo_id, local_model_hub, in_out_pairs, @@ -2108,6 +2182,7 @@ if __name__ == '__main__': use_fp16_torch_dtype = False task = 'continuation' optimize_model = False # only for transformers_int4_npu_win + group_size = 64 if 'streaming' in conf: streaming = conf['streaming'] if 'use_fp16_torch_dtype' in conf: @@ -2116,6 +2191,8 @@ if __name__ == '__main__': task = conf['task'] if 'optimize_model' in conf: optimize_model = conf['optimize_model'] + if 'group_size' in conf: + group_size = conf['group_size'] lookahead = False transpose_value_cache = True if 'transpose_value_cache' in conf: @@ -2145,7 +2222,7 @@ if __name__ == '__main__': if task in ['QA', 'summarize'] and conf['num_beams'] == 1 and batch_size == 1: lookahead = True run_model(model, api, in_out_pairs, conf['local_model_hub'], conf['warm_up'], conf['num_trials'], conf['num_beams'], - conf['low_bit'], conf['cpu_embedding'], batch_size, streaming, use_fp16_torch_dtype, lookahead, task, optimize_model, transpose_value_cache) + conf['low_bit'], conf['cpu_embedding'], batch_size, streaming, use_fp16_torch_dtype, lookahead, task, optimize_model, transpose_value_cache, group_size) df = pd.DataFrame(results, columns=['model', '1st token avg latency (ms)', '2+ avg latency (ms/token)', 'encoder time (ms)', 'input/output tokens', 'batch_size', 'actual input/output tokens', 'num_beams', 'low_bit', 'cpu_embedding', 'model loading time (s)', 'peak mem (GB)', 'streaming', 'use_fp16_torch_dtype']) diff --git a/python/llm/dev/benchmark/all-in-one/save_openvino.py b/python/llm/dev/benchmark/all-in-one/save_openvino.py new file mode 100644 index 00000000..816aa8a2 --- /dev/null +++ b/python/llm/dev/benchmark/all-in-one/save_openvino.py @@ -0,0 +1,107 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Some parts of this file is adapted from +# https://github.com/openvino-dev-samples/Qwen2.openvino/blob/main/convert.py + +import os +from pathlib import Path +import warnings + +from transformers import AutoTokenizer, LlamaTokenizer +from optimum.intel import OVWeightQuantizationConfig +from optimum.intel.openvino import OVModelForCausalLM + +from run import LLAMA_IDS, get_model_path + +current_dir = os.path.dirname(os.path.realpath(__file__)) + +def save_model_to_openvino(repo_id, + local_model_hub, + low_bit, + group_size, + ): + model_path = get_model_path(repo_id, local_model_hub) + + ir_repo_id = (repo_id.split( + "/")[1] + '-ov-' + low_bit + '-' +str(group_size)) + + if local_model_hub: + repo_model_name = repo_id.split( + "/")[1] + '-ov-' + low_bit + '-' +str(group_size) + ir_model_path = local_model_hub + os.path.sep + repo_model_name + ir_model_path = Path(ir_model_path) + else: + ir_model_path = Path(ir_repo_id) + + if not ir_model_path.exists(): + os.mkdir(ir_model_path) + + compression_configs = { + "sym": True, + "group_size": group_size, + "ratio": 1.0, + } + + print(">> Exporting IR") + if low_bit == "sym_int4": + compression_configs['sym'] = True + ov_model = OVModelForCausalLM.from_pretrained(model_path, export=True, + trust_remote_code=True, + compile=False, quantization_config=OVWeightQuantizationConfig( + bits=4, **compression_configs)).eval() + elif low_bit == "asym_int4": + compression_configs['sym'] = False + ov_model = OVModelForCausalLM.from_pretrained(model_path, export=True, + trust_remote_code=True, + compile=False, quantization_config=OVWeightQuantizationConfig( + bits=4, **compression_configs)).eval() + + print(">> Saving IR") + ov_model.save_pretrained(ir_model_path) + + print(">> Exporting tokenizer") + if repo_id in LLAMA_IDS: + tokenizer = LlamaTokenizer.from_pretrained(model_path, + trust_remote_code=True) + else: + tokenizer = AutoTokenizer.from_pretrained(model_path, + trust_remote_code=True) + tokenizer.save_pretrained(ir_model_path) + + print(">> Exporting IR tokenizer") + from optimum.exporters.openvino.convert import export_tokenizer + export_tokenizer(tokenizer, ir_model_path) + print(f">> Finished saving OpenVINO IR for {repo_id} in {low_bit} with group size {group_size}") + del ov_model + del model_path + +if __name__ == '__main__': + supported_precision = ["sym_int4", "asym_int4"] + + from omegaconf import OmegaConf + conf = OmegaConf.load(f'{current_dir}/config.yaml') + + if conf['low_bit'] in supported_precision: + for model in conf.repo_id: + save_model_to_openvino(repo_id=model, + local_model_hub=conf['local_model_hub'], + low_bit=conf['low_bit'], + group_size=conf['group_size'],) + else: + warnings.warn(f"low_bit {conf['low_bit']} is not supported " + "in all-in-one benchmark for OpenVINO tests. Only " + 'sym_int4 and asym_int4 is currently supported for "transformers_openvino" test api.')