Add OpenVINO performance tests to all-in-one benchmark (#12238)

* add-openvino-to-all-in-one

* update on openvino API

* Update save_openvino.py

* Update save_openvino.py

* Update save_openvino.py

* update on run.py and save_openvino

* update references

* Create openvino-requirements.txt

* fix on comments

* Small updates

* Small fix

* Fix

---------

Co-authored-by: Yuwen Hu <yuwen.hu@intel.com>

python/llm/dev/benchmark/all-in-one/README.md

@@ -60,23 +60,38 @@ test_api:
   # - "speculative_cpu"                     # on Intel CPU, inference with self-speculative decoding
   # - "deepspeed_transformer_int4_cpu"      # on Intel CPU, deepspeed autotp inference
   # - "transformers_int4_npu_win"           # on Intel NPU for Windows,  transformer-like API, (qtype=int4)
+  # - "transformers_openvino"               # on Intel GPU, use OpenVINO. Please make sure you have used the save_openvino.py to save the converted OpenVINO model
 cpu_embedding: False # whether put embedding to CPU
 streaming: False # whether output in streaming way (only available now for gpu win related test_api)
 use_fp16_torch_dtype: True # whether use fp16 for non-linear layer (only available now for "pipeline_parallel_gpu" test_api)
 task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
-
+group_size: 64 # group_size when converting OpenVINO model (only available or "transformers_openvino" test_api)
 ```
 
 ## (Optional) Save model in low bit
 If you choose the `transformer_int4_loadlowbit_gpu_win` or `transformer_int4_fp16_loadlowbit_gpu_win` test API, you will need to save the model in low bit first.
 
-Run `python save.py` will save all models declared in `repo_id` list into low bit models under `local_model_hub` folder.
+Running `python save.py` will save all models declared in `repo_id` list into low bit models under `local_model_hub` folder.
+
+## (Optional) Save model for OpenVINO
+If you choose the `transformers_openvino` test API, you will need to convert the model with OpenVINO first.
+
+Follow commands below to set up the environment for testing OpenVINO on Intel GPU, in which `requirements.txt` should be downloaded from [here](Download the requirements txt from https://github.com/openvino-dev-samples/Qwen2.openvino/blob/main/requirements.txt):
+
+```bash
+conda create -n test-ov python=3.11
+pip install -r requirements.txt
+pip install --pre --upgrade ipex-llm # only for IPEX-LLM BenchmarkWrapper
+pip install accelerate
+```
+
+Then, running `python save_openvino.py` will save all models declared in `repo_id` list into OpenVINO models with `low_bit` precision under `local_model_hub` folder.
 
 ## Run
 
 run `python run.py`, this will output results to `results.csv`.
 
-For SPR performance, run `bash run-spr.sh`.
+For IPEX-LLM SPR performance, run `bash run-spr.sh`.
 
 > **Note**
 >
@@ -86,6 +101,6 @@ For SPR performance, run `bash run-spr.sh`.
 >
 > Please install torch nightly version to avoid `Illegal instruction (core dumped)` issue, you can follow the following command to install: `pip install --pre --upgrade torch --index-url https://download.pytorch.org/whl/nightly/cpu`
 
-For ARC performance, run `bash run-arc.sh`.
+For IPEX-LLM ARC performance, run `bash run-arc.sh`.
 
-For MAX GPU performance, run `bash run-max-gpu.sh`.
+For IPEX-LLM MAX GPU performance, run `bash run-max-gpu.sh`.

python/llm/dev/benchmark/all-in-one/config.yaml

@@ -37,9 +37,11 @@ test_api:
   # - "deepspeed_transformer_int4_cpu"      # on Intel CPU, deepspeed autotp inference
   # - "transformers_int4_npu_win"           # on Intel NPU for Windows,  transformer-like API, (qtype=int4)
   # - "transformers_int4_loadlowbit_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4), use load_low_bit API. Please make sure you have used the save_npu.py to save the converted low bit model
+  # - "transformers_openvino"               # on Intel GPU, use OpenVINO. Please make sure you have used the save_openvino.py to save the converted OpenVINO model
 cpu_embedding: False # whether put embedding to CPU
 streaming: False # whether output in streaming way (only available now for gpu win related test_api)
 optimize_model: False # whether apply further optimization on NPU (only available now for transformers_int4_npu_win test_api)
 use_fp16_torch_dtype: True # whether use fp16 for non-linear layer (only available now for "pipeline_parallel_gpu" test_api)
 task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
-transpose_value_cache: True # whether apply transposed v_cache optimization on NPU (only available now for transformers_int4_npu_win test_api)
+transpose_value_cache: True # whether apply transposed v_cache optimization on NPU (only available now for transformers_int4_npu_win test_api)
+group_size: 64 # group_size when converting OpenVINO model (only available or "transformers_openvino" test_api)

python/llm/dev/benchmark/all-in-one/run.py

@@ -138,7 +138,7 @@ def preprocess_prompt(tokenizer, in_len, task):
         input_ids = tokenizer.encode(input_str, return_tensors="pt")    
     return input_ids
 
-def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1, num_trials=3, num_beams=1, low_bit='sym_int4', cpu_embedding=False, batch_size=1, streaming=False, use_fp16_torch_dtype=False, lookahead=False, task='continuation', optimize_model=False, transpose_value_cache=True):
+def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1, num_trials=3, num_beams=1, low_bit='sym_int4', cpu_embedding=False, batch_size=1, streaming=False, use_fp16_torch_dtype=False, lookahead=False, task='continuation', optimize_model=False, transpose_value_cache=True, group_size=64):
     # TODO: make a parameter
     result= {}
     if test_api == 'transformer_int4':
@@ -193,6 +193,8 @@ def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1,
         result = transformers_int4_npu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, optimize_model, transpose_value_cache)
     elif test_api == 'transformers_int4_loadlowbit_npu_win':
         result = run_transformer_int4_loadlowbit_npu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, optimize_model, transpose_value_cache)
+    elif test_api == 'transformers_openvino':
+        result = run_transformers_openvino(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, group_size)
     else:
         invalidInputError(False, "Unknown test_api " + test_api + ", please check your config.yaml.")
 
@@ -746,6 +748,78 @@ def run_transformer_int4_loadlowbit_npu_win(repo_id,
     gc.collect()
     return result
 
+def run_transformers_openvino(repo_id,
+                              local_model_hub,
+                              in_out_pairs,
+                              warm_up,
+                              num_trials,
+                              num_beams,
+                              low_bit,
+                              batch_size,
+                              group_size):
+    from optimum.intel import OVModelForCausalLM
+    from transformers import AutoTokenizer, LlamaTokenizer, PretrainedConfig
+
+    ir_repo_id = (repo_id + '-ov-' + low_bit + '-' +str(group_size))
+    model_path = get_model_path(ir_repo_id, local_model_hub)
+
+    ov_config = {"PERFORMANCE_HINT": "LATENCY",
+                 "NUM_STREAMS": "1", "CACHE_DIR": ""}
+    config_dict = dict(pretrained_model_name_or_path=model_path, 
+                       trust_remote_code=True,
+                       use_cache=True, low_cpu_mem_usage=True)
+
+    config = PretrainedConfig(**config_dict)
+
+    # Load model converted by OpenVINO
+    st = time.perf_counter()
+    if repo_id in LLAMA_IDS:
+        model = OVModelForCausalLM.from_pretrained(model_path, device="GPU",
+                                                   ov_config=ov_config, config=config).eval()
+        tokenizer = LlamaTokenizer.from_pretrained(model_path, trust_remote_code=True)
+    else:
+        model = OVModelForCausalLM.from_pretrained(model_path, device="GPU",
+                                                     ov_config=ov_config, config=config).eval()
+        tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+    end = time.perf_counter()
+    load_time = end - st
+    print(">> loading of model costs {}s".format(load_time))
+
+    model = BenchmarkWrapper(model)
+
+    result = {}
+    with torch.inference_mode():
+        for in_out in in_out_pairs:
+            in_out_len = in_out.split("-")
+            in_len = int(in_out_len[0])
+            out_len = int(in_out_len[1])
+            input_str = get_continuation_input_str(in_len, tokenizer)
+            # As different tokenizer has different encodings,
+            # slice the input_ids to ensure the prompt length is required length.
+            input_ids = tokenizer.encode(input_str, return_tensors="pt")
+            input_ids = input_ids[:, :in_len]
+            true_str = tokenizer.batch_decode(input_ids)[0]
+            input_list = [true_str] * batch_size
+            input_ids = tokenizer(input_list, return_tensors="pt").input_ids
+            input_ids = input_ids[:, :in_len]
+            actual_in_len = input_ids.shape[1]
+            result[in_out] = []
+            for i in range(num_trials + warm_up):
+                st = time.perf_counter()
+                output_ids = model.generate(input_ids, do_sample=False, max_new_tokens=out_len,
+                                            min_new_tokens=out_len, num_beams=num_beams)
+                end = time.perf_counter()
+                print("model generate cost: " + str(end - st))
+                output = tokenizer.batch_decode(output_ids)
+                print(output[0])
+                actual_out_len = output_ids.shape[1] - actual_in_len
+                if i >= warm_up:
+                    result[in_out].append([model.first_cost, model.rest_cost_mean, model.encoder_time,
+                                           actual_in_len, actual_out_len, load_time])
+    del model
+    gc.collect()
+    return result
+
 def run_optimize_model_gpu(repo_id,
                            local_model_hub,
                            in_out_pairs,
@@ -2108,6 +2182,7 @@ if __name__ == '__main__':
     use_fp16_torch_dtype = False
     task = 'continuation'
     optimize_model = False # only for transformers_int4_npu_win
+    group_size = 64
     if 'streaming' in conf:
         streaming = conf['streaming']
     if 'use_fp16_torch_dtype' in conf:
@@ -2116,6 +2191,8 @@ if __name__ == '__main__':
         task = conf['task']
     if 'optimize_model' in conf:
         optimize_model = conf['optimize_model']
+    if 'group_size' in conf:
+        group_size = conf['group_size']
     lookahead = False
     transpose_value_cache = True
     if 'transpose_value_cache' in conf:
@@ -2145,7 +2222,7 @@ if __name__ == '__main__':
                 if task in ['QA', 'summarize'] and conf['num_beams'] == 1 and batch_size == 1:
                     lookahead = True
                 run_model(model, api, in_out_pairs, conf['local_model_hub'], conf['warm_up'], conf['num_trials'], conf['num_beams'],
-                      conf['low_bit'], conf['cpu_embedding'], batch_size, streaming, use_fp16_torch_dtype, lookahead, task, optimize_model, transpose_value_cache)
+                      conf['low_bit'], conf['cpu_embedding'], batch_size, streaming, use_fp16_torch_dtype, lookahead, task, optimize_model, transpose_value_cache, group_size)
         df = pd.DataFrame(results, columns=['model', '1st token avg latency (ms)', '2+ avg latency (ms/token)', 'encoder time (ms)',
                                             'input/output tokens', 'batch_size', 'actual input/output tokens', 'num_beams', 'low_bit', 'cpu_embedding',
                                             'model loading time (s)', 'peak mem (GB)', 'streaming', 'use_fp16_torch_dtype'])

python/llm/dev/benchmark/all-in-one/save_openvino.py

@@ -0,0 +1,107 @@
+#
+# Copyright 2016 The BigDL Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Some parts of this file is adapted from
+# https://github.com/openvino-dev-samples/Qwen2.openvino/blob/main/convert.py
+
+import os
+from pathlib import Path
+import warnings
+
+from transformers import AutoTokenizer, LlamaTokenizer
+from optimum.intel import OVWeightQuantizationConfig
+from optimum.intel.openvino import OVModelForCausalLM
+
+from run import LLAMA_IDS, get_model_path
+
+current_dir = os.path.dirname(os.path.realpath(__file__))
+
+def save_model_to_openvino(repo_id,
+                           local_model_hub,
+                           low_bit,
+                           group_size,
+                           ):
+    model_path = get_model_path(repo_id, local_model_hub)
+
+    ir_repo_id = (repo_id.split(
+        "/")[1] + '-ov-' + low_bit + '-' +str(group_size))
+
+    if local_model_hub:
+        repo_model_name = repo_id.split(
+        "/")[1] + '-ov-' + low_bit + '-' +str(group_size)
+        ir_model_path = local_model_hub + os.path.sep + repo_model_name
+        ir_model_path = Path(ir_model_path)
+    else:
+        ir_model_path = Path(ir_repo_id)
+
+    if not ir_model_path.exists():
+        os.mkdir(ir_model_path)
+
+    compression_configs = {
+        "sym": True,
+        "group_size": group_size,
+        "ratio": 1.0,
+    }
+
+    print(">> Exporting IR")
+    if low_bit == "sym_int4":
+        compression_configs['sym'] = True
+        ov_model = OVModelForCausalLM.from_pretrained(model_path, export=True, 
+                                                      trust_remote_code=True,
+                                                      compile=False, quantization_config=OVWeightQuantizationConfig(
+                                                      bits=4, **compression_configs)).eval()
+    elif low_bit == "asym_int4":
+        compression_configs['sym'] = False
+        ov_model = OVModelForCausalLM.from_pretrained(model_path, export=True, 
+                                                      trust_remote_code=True,
+                                                      compile=False, quantization_config=OVWeightQuantizationConfig(
+                                                      bits=4, **compression_configs)).eval()
+
+    print(">> Saving IR")
+    ov_model.save_pretrained(ir_model_path)
+
+    print(">> Exporting tokenizer")
+    if repo_id in LLAMA_IDS:
+        tokenizer = LlamaTokenizer.from_pretrained(model_path,
+                                                   trust_remote_code=True)
+    else:
+        tokenizer = AutoTokenizer.from_pretrained(model_path,
+                                                  trust_remote_code=True)
+    tokenizer.save_pretrained(ir_model_path)
+
+    print(">> Exporting IR tokenizer")
+    from optimum.exporters.openvino.convert import export_tokenizer
+    export_tokenizer(tokenizer, ir_model_path)
+    print(f">> Finished saving OpenVINO IR for {repo_id} in {low_bit} with group size {group_size}")
+    del ov_model
+    del model_path
+
+if __name__ == '__main__':
+    supported_precision = ["sym_int4", "asym_int4"]
+
+    from omegaconf import OmegaConf
+    conf = OmegaConf.load(f'{current_dir}/config.yaml')
+
+    if conf['low_bit'] in supported_precision:
+        for model in conf.repo_id:
+            save_model_to_openvino(repo_id=model,
+                                   local_model_hub=conf['local_model_hub'],
+                                   low_bit=conf['low_bit'],
+                                   group_size=conf['group_size'],)
+    else:
+        warnings.warn(f"low_bit {conf['low_bit']} is not supported "
+                      "in all-in-one benchmark for OpenVINO tests. Only "
+                      'sym_int4 and asym_int4 is currently supported for "transformers_openvino" test api.')