From e713296090e0761a5c1acf2c42235abf128b847d Mon Sep 17 00:00:00 2001 From: Yuwen Hu <54161268+Oscilloscope98@users.noreply.github.com> Date: Fri, 25 Oct 2024 16:52:59 +0800 Subject: [PATCH] Update all-in-one benchmark (#12272) * Update all-in-one benchmark * Small fix * Small fix * Small fix --- python/llm/dev/benchmark/all-in-one/README.md | 29 +---- .../llm/dev/benchmark/all-in-one/config.yaml | 2 - .../dev/benchmark/all-in-one/save_openvino.py | 107 ------------------ 3 files changed, 6 insertions(+), 132 deletions(-) delete mode 100644 python/llm/dev/benchmark/all-in-one/save_openvino.py diff --git a/python/llm/dev/benchmark/all-in-one/README.md b/python/llm/dev/benchmark/all-in-one/README.md index b536256f..8610a852 100644 --- a/python/llm/dev/benchmark/all-in-one/README.md +++ b/python/llm/dev/benchmark/all-in-one/README.md @@ -2,9 +2,7 @@ All in one benchmark test allows users to test all the benchmarks and record them in a result CSV. Users can provide models and related information in `config.yaml`. -Before running, make sure you have [ipex-llm](../../../../../README.md) installed. - -If you would like to use all-in-one benchmark for testing OpenVINO, please directly refer to [this section](#optional-save-model-for-openvino) for environment setup. +Before running, make sure to have [ipex-llm](../../../../../README.md) installed. > The prompts for benchmarking are from datasets [abisee/cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail), [Open-Orca/OpenOrca](https://huggingface.co/datasets/Open-Orca/OpenOrca), [THUDM/LongBench](https://huggingface.co/datasets/THUDM/LongBench), etc. @@ -61,13 +59,12 @@ test_api: # - "bigdl_ipex_int8" # on Intel CPU, (qtype=int8) # - "speculative_cpu" # on Intel CPU, inference with self-speculative decoding # - "deepspeed_transformer_int4_cpu" # on Intel CPU, deepspeed autotp inference - # - "transformers_int4_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4) - # - "transformers_openvino" # on Intel GPU, use OpenVINO. Please make sure you have used the save_openvino.py to save the converted OpenVINO model + # - "transformers_int4_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4) cpu_embedding: False # whether put embedding to CPU streaming: False # whether output in streaming way (only available now for gpu win related test_api) use_fp16_torch_dtype: True # whether use fp16 for non-linear layer (only available now for "pipeline_parallel_gpu" test_api) task: 'continuation' # task can be 'continuation', 'QA' and 'summarize' -group_size: 64 # group_size when converting OpenVINO model (only available or "transformers_openvino" test_api) + ``` ## (Optional) Save model in low bit @@ -75,25 +72,11 @@ If you choose the `transformer_int4_loadlowbit_gpu_win` or `transformer_int4_fp1 Running `python save.py` will save all models declared in `repo_id` list into low bit models under `local_model_hub` folder. -## (Optional) Save model for OpenVINO -If you choose the `transformers_openvino` test API, you will need to convert the model with OpenVINO first. - -Follow commands below to set up the environment for testing OpenVINO on Intel GPU, in which `requirements.txt` should be downloaded from [here](Download the requirements txt from https://github.com/openvino-dev-samples/Qwen2.openvino/blob/main/requirements.txt): - -```bash -conda create -n test-ov python=3.11 -pip install -r requirements.txt -pip install --pre --upgrade ipex-llm # only for IPEX-LLM BenchmarkWrapper -pip install accelerate omegaconf pandas -``` - -Then, running `python save_openvino.py` will save all models declared in `repo_id` list into OpenVINO models with `low_bit` precision under `local_model_hub` folder. - ## Run run `python run.py`, this will output results to `results.csv`. -For IPEX-LLM SPR performance, run `bash run-spr.sh`. +For SPR performance, run `bash run-spr.sh`. > **Note** > @@ -103,6 +86,6 @@ For IPEX-LLM SPR performance, run `bash run-spr.sh`. > > Please install torch nightly version to avoid `Illegal instruction (core dumped)` issue, you can follow the following command to install: `pip install --pre --upgrade torch --index-url https://download.pytorch.org/whl/nightly/cpu` -For IPEX-LLM ARC performance, run `bash run-arc.sh`. +For ARC performance, run `bash run-arc.sh`. -For IPEX-LLM MAX GPU performance, run `bash run-max-gpu.sh`. +For MAX GPU performance, run `bash run-max-gpu.sh`. diff --git a/python/llm/dev/benchmark/all-in-one/config.yaml b/python/llm/dev/benchmark/all-in-one/config.yaml index ddccd4d9..377b2516 100644 --- a/python/llm/dev/benchmark/all-in-one/config.yaml +++ b/python/llm/dev/benchmark/all-in-one/config.yaml @@ -37,11 +37,9 @@ test_api: # - "deepspeed_transformer_int4_cpu" # on Intel CPU, deepspeed autotp inference # - "transformers_int4_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4) # - "transformers_int4_loadlowbit_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4), use load_low_bit API. Please make sure you have used the save_npu.py to save the converted low bit model - # - "transformers_openvino" # on Intel GPU, use OpenVINO. Please make sure you have used the save_openvino.py to save the converted OpenVINO model cpu_embedding: False # whether put embedding to CPU streaming: False # whether output in streaming way (only available now for gpu win related test_api) optimize_model: False # whether apply further optimization on NPU (only available now for transformers_int4_npu_win test_api) use_fp16_torch_dtype: True # whether use fp16 for non-linear layer (only available now for "pipeline_parallel_gpu" test_api) task: 'continuation' # task can be 'continuation', 'QA' and 'summarize' transpose_value_cache: True # whether apply transposed v_cache optimization on NPU (only available now for transformers_int4_npu_win test_api) -group_size: 64 # group_size when converting OpenVINO model (only available or "transformers_openvino" test_api) diff --git a/python/llm/dev/benchmark/all-in-one/save_openvino.py b/python/llm/dev/benchmark/all-in-one/save_openvino.py deleted file mode 100644 index 816aa8a2..00000000 --- a/python/llm/dev/benchmark/all-in-one/save_openvino.py +++ /dev/null @@ -1,107 +0,0 @@ -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Some parts of this file is adapted from -# https://github.com/openvino-dev-samples/Qwen2.openvino/blob/main/convert.py - -import os -from pathlib import Path -import warnings - -from transformers import AutoTokenizer, LlamaTokenizer -from optimum.intel import OVWeightQuantizationConfig -from optimum.intel.openvino import OVModelForCausalLM - -from run import LLAMA_IDS, get_model_path - -current_dir = os.path.dirname(os.path.realpath(__file__)) - -def save_model_to_openvino(repo_id, - local_model_hub, - low_bit, - group_size, - ): - model_path = get_model_path(repo_id, local_model_hub) - - ir_repo_id = (repo_id.split( - "/")[1] + '-ov-' + low_bit + '-' +str(group_size)) - - if local_model_hub: - repo_model_name = repo_id.split( - "/")[1] + '-ov-' + low_bit + '-' +str(group_size) - ir_model_path = local_model_hub + os.path.sep + repo_model_name - ir_model_path = Path(ir_model_path) - else: - ir_model_path = Path(ir_repo_id) - - if not ir_model_path.exists(): - os.mkdir(ir_model_path) - - compression_configs = { - "sym": True, - "group_size": group_size, - "ratio": 1.0, - } - - print(">> Exporting IR") - if low_bit == "sym_int4": - compression_configs['sym'] = True - ov_model = OVModelForCausalLM.from_pretrained(model_path, export=True, - trust_remote_code=True, - compile=False, quantization_config=OVWeightQuantizationConfig( - bits=4, **compression_configs)).eval() - elif low_bit == "asym_int4": - compression_configs['sym'] = False - ov_model = OVModelForCausalLM.from_pretrained(model_path, export=True, - trust_remote_code=True, - compile=False, quantization_config=OVWeightQuantizationConfig( - bits=4, **compression_configs)).eval() - - print(">> Saving IR") - ov_model.save_pretrained(ir_model_path) - - print(">> Exporting tokenizer") - if repo_id in LLAMA_IDS: - tokenizer = LlamaTokenizer.from_pretrained(model_path, - trust_remote_code=True) - else: - tokenizer = AutoTokenizer.from_pretrained(model_path, - trust_remote_code=True) - tokenizer.save_pretrained(ir_model_path) - - print(">> Exporting IR tokenizer") - from optimum.exporters.openvino.convert import export_tokenizer - export_tokenizer(tokenizer, ir_model_path) - print(f">> Finished saving OpenVINO IR for {repo_id} in {low_bit} with group size {group_size}") - del ov_model - del model_path - -if __name__ == '__main__': - supported_precision = ["sym_int4", "asym_int4"] - - from omegaconf import OmegaConf - conf = OmegaConf.load(f'{current_dir}/config.yaml') - - if conf['low_bit'] in supported_precision: - for model in conf.repo_id: - save_model_to_openvino(repo_id=model, - local_model_hub=conf['local_model_hub'], - low_bit=conf['low_bit'], - group_size=conf['group_size'],) - else: - warnings.warn(f"low_bit {conf['low_bit']} is not supported " - "in all-in-one benchmark for OpenVINO tests. Only " - 'sym_int4 and asym_int4 is currently supported for "transformers_openvino" test api.')