From e54af44ed60af724e864dc740622f2863f4d748b Mon Sep 17 00:00:00 2001 From: Ch1y0q Date: Mon, 4 Nov 2024 16:00:20 +0800 Subject: [PATCH] Add `transformers_int4_npu_pipeline_win` in all-in-one benchmark (#12325) * add transformers_int4_npu_pipeline_win * bugfix * bugfix: wrong actual_output_len * fix format * bugfix & update `README.md` --- python/llm/dev/benchmark/all-in-one/README.md | 9 ++- .../llm/dev/benchmark/all-in-one/config.yaml | 3 +- python/llm/dev/benchmark/all-in-one/run.py | 72 ++++++++++++++++++- .../npu_pipeline_model/convert_pipeline.py | 4 ++ 4 files changed, 83 insertions(+), 5 deletions(-) diff --git a/python/llm/dev/benchmark/all-in-one/README.md b/python/llm/dev/benchmark/all-in-one/README.md index 8610a852..f46c65d6 100644 --- a/python/llm/dev/benchmark/all-in-one/README.md +++ b/python/llm/dev/benchmark/all-in-one/README.md @@ -27,6 +27,7 @@ Config YAML file has following format repo_id: # - 'THUDM/chatglm2-6b' - 'meta-llama/Llama-2-7b-chat-hf' + # - 'meta-llama/Meta-Llama-3.1-8B-Instruct' # - 'liuhaotian/llava-v1.5-7b' # requires a LLAVA_REPO_DIR env variables pointing to the llava dir; added only for gpu win related test_api now local_model_hub: 'path to your local model hub' warm_up: 1 # must set >=2 when run "pipeline_parallel_gpu" test_api @@ -36,6 +37,7 @@ low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) batch_size: 1 # default to 1 in_out_pairs: - '32-32' + - '960-64' - '1024-128' test_api: - "transformer_int4_fp16_gpu" # on Intel GPU, transformer-like API, (qtype=int4), (dtype=fp16) @@ -59,11 +61,16 @@ test_api: # - "bigdl_ipex_int8" # on Intel CPU, (qtype=int8) # - "speculative_cpu" # on Intel CPU, inference with self-speculative decoding # - "deepspeed_transformer_int4_cpu" # on Intel CPU, deepspeed autotp inference - # - "transformers_int4_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4) + # - "transformers_int4_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4) + # - "transformers_int4_loadlowbit_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4), use load_low_bit API. Please make sure you have used the save_npu.py to save the converted low bit model + # - "transformers_int4_npu_pipeline_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4) cpu_embedding: False # whether put embedding to CPU streaming: False # whether output in streaming way (only available now for gpu win related test_api) +optimize_model: False # whether apply further optimization on NPU (only available now for transformers_int4_npu_win test_api) use_fp16_torch_dtype: True # whether use fp16 for non-linear layer (only available now for "pipeline_parallel_gpu" test_api) task: 'continuation' # task can be 'continuation', 'QA' and 'summarize' +transpose_value_cache: True # whether apply transposed v_cache optimization on NPU (only available now for transformers_int4_npu_win test_api) +npu_group_size: 0 # this can only be either 0 or 128, and only works for `transformers_int4_npu_win` / `transformers_int4_npu_pipeline_win` ``` diff --git a/python/llm/dev/benchmark/all-in-one/config.yaml b/python/llm/dev/benchmark/all-in-one/config.yaml index 7c098cea..c371bb65 100644 --- a/python/llm/dev/benchmark/all-in-one/config.yaml +++ b/python/llm/dev/benchmark/all-in-one/config.yaml @@ -37,10 +37,11 @@ test_api: # - "deepspeed_transformer_int4_cpu" # on Intel CPU, deepspeed autotp inference # - "transformers_int4_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4) # - "transformers_int4_loadlowbit_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4), use load_low_bit API. Please make sure you have used the save_npu.py to save the converted low bit model + # - "transformers_int4_npu_pipeline_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4) cpu_embedding: False # whether put embedding to CPU streaming: False # whether output in streaming way (only available now for gpu win related test_api) optimize_model: False # whether apply further optimization on NPU (only available now for transformers_int4_npu_win test_api) use_fp16_torch_dtype: True # whether use fp16 for non-linear layer (only available now for "pipeline_parallel_gpu" test_api) task: 'continuation' # task can be 'continuation', 'QA' and 'summarize' transpose_value_cache: True # whether apply transposed v_cache optimization on NPU (only available now for transformers_int4_npu_win test_api) -npu_group_size: 128 # This can only be either 0 or 128, and only works for `transformers_int4_npu_win` / `transformers_int4_npu_pipline_win` +npu_group_size: 0 # this can only be either 0 or 128, and only works for `transformers_int4_npu_win` / `transformers_int4_npu_pipeline_win` diff --git a/python/llm/dev/benchmark/all-in-one/run.py b/python/llm/dev/benchmark/all-in-one/run.py index 9f417c23..01f6a7c8 100644 --- a/python/llm/dev/benchmark/all-in-one/run.py +++ b/python/llm/dev/benchmark/all-in-one/run.py @@ -191,6 +191,8 @@ def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1, result = run_pipeline_parallel_gpu(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, cpu_embedding, fp16=use_fp16_torch_dtype) elif test_api == 'transformers_int4_npu_win': result = transformers_int4_npu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, optimize_model, transpose_value_cache, group_size) + elif test_api == 'transformers_int4_npu_pipeline_win': + result = transformers_int4_npu_pipeline_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, optimize_model, transpose_value_cache, group_size) elif test_api == 'transformers_int4_loadlowbit_npu_win': result = run_transformer_int4_loadlowbit_npu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, optimize_model, transpose_value_cache) elif test_api == 'transformers_openvino': @@ -215,7 +217,7 @@ def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1, result[in_out_pair][-1][6] if any(keyword in test_api for keyword in ['int4_gpu', 'int4_fp16_gpu_win', 'int4_loadlowbit_gpu', 'int4_fp16_loadlowbit_gpu', 'fp16_gpu', 'deepspeed_optimize_model_gpu']) and not lookahead else 'N/A', streaming if 'win' in test_api else 'N/A', use_fp16_torch_dtype if 'pipeline_parallel_gpu' in test_api else 'N/A', - group_size if 'transformers_int4_npu_win' in test_api else 'N/A'], + group_size if any(keyword in test_api for keyword in ['transformers_int4_npu_win', 'transformers_int4_npu_pipeline_win']) else 'N/A'], ) @@ -680,6 +682,70 @@ def transformers_int4_npu_win(repo_id, gc.collect() return result +def transformers_int4_npu_pipeline_win(repo_id, + local_model_hub, + in_out_pairs, + warm_up, + num_trials, + num_beams, + low_bit, + batch_size, + optimize_model, + transpose_value_cache, + npu_group_size): + from ipex_llm.transformers.npu_model import AutoModel, AutoModelForCausalLM + from transformers import AutoTokenizer, LlamaTokenizer + + model_path = get_model_path(repo_id, local_model_hub) + in_out_len = in_out_pairs[0].split("-") + max_context_len = max(int(in_out_len[0]) + int(in_out_len[1]), 1024) + # Load model in 4 bit, + # which convert the relevant layers in the model into INT4 format + st = time.perf_counter() + + model = AutoModelForCausalLM.from_pretrained(model_path, load_in_low_bit=low_bit, trust_remote_code=True, pipeline=True, torch_dtype=torch.float16, + optimize_model=optimize_model, max_context_len=max_context_len, max_prompt_len=int(in_out_len[0]), + quantization_group_size=npu_group_size, transpose_value_cache=transpose_value_cache, + use_cache=True, attn_implementation="eager").eval() + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + + end = time.perf_counter() + load_time = end - st + print(">> loading of model costs {}s".format(load_time)) + + result = {} + with torch.inference_mode(): + for in_out in in_out_pairs: + in_out_len = in_out.split("-") + in_len = int(in_out_len[0]) + out_len = int(in_out_len[1]) + input_str = get_continuation_input_str(in_len, tokenizer) + # As different tokenizer has different encodings, + # slice the input_ids to ensure the prompt length is required length. + input_ids = tokenizer.encode(input_str, return_tensors="pt") + input_ids = input_ids[:, :in_len] + true_str = tokenizer.batch_decode(input_ids)[0] + input_list = [true_str] * batch_size + input_ids = tokenizer(input_list, return_tensors="pt").input_ids + input_ids = input_ids[:, :in_len] + actual_in_len = input_ids.shape[1] + result[in_out] = [] + for i in range(num_trials + warm_up): + st = time.perf_counter() + output_ids = model.generate(input_ids, do_sample=False, max_new_tokens=out_len, + min_new_tokens=out_len, num_beams=num_beams) + end = time.perf_counter() + print("model generate cost: " + str(end - st)) + output = tokenizer.batch_decode(output_ids) + print(output[0]) + actual_out_len = output_ids.shape[1] - actual_in_len + if i >= warm_up: + result[in_out].append([model.first_cost, model.rest_cost_mean, model.encoder_time, + actual_in_len, actual_out_len, load_time]) + del model + gc.collect() + return result + def run_transformer_int4_loadlowbit_npu_win(repo_id, local_model_hub, in_out_pairs, @@ -2186,7 +2252,7 @@ if __name__ == '__main__': streaming = False use_fp16_torch_dtype = False task = 'continuation' - optimize_model = False # only for transformers_int4_npu_win + optimize_model = False # only for transformers_int4_npu_win, transformers_int4_npu_pipeline_win group_size = 64 if 'streaming' in conf: streaming = conf['streaming'] @@ -2233,7 +2299,7 @@ if __name__ == '__main__': df = pd.DataFrame(results, columns=['model', '1st token avg latency (ms)', '2+ avg latency (ms/token)', 'encoder time (ms)', 'input/output tokens', 'batch_size', 'actual input/output tokens', 'num_beams', 'low_bit', 'cpu_embedding', 'model loading time (s)', 'peak mem (GB)', 'streaming', 'use_fp16_torch_dtype', 'npu_group_size']) - if "pipeline" in api or "deepspeed" in api: + if ("pipeline" in api or "deepspeed" in api) and api != 'transformers_int4_npu_pipeline_win': if torch.distributed.get_rank() == 0: df.index += max(line_counter - 1, 0) if line_counter == 0: diff --git a/python/llm/src/ipex_llm/transformers/npu_pipeline_model/convert_pipeline.py b/python/llm/src/ipex_llm/transformers/npu_pipeline_model/convert_pipeline.py index 4e7394d3..4c49781c 100644 --- a/python/llm/src/ipex_llm/transformers/npu_pipeline_model/convert_pipeline.py +++ b/python/llm/src/ipex_llm/transformers/npu_pipeline_model/convert_pipeline.py @@ -172,6 +172,10 @@ def generate( thread.join() time_end = time.perf_counter() + self.first_cost = (time_t3 - time_start_all - (time_t2 - time_t1)) # seconds + self.rest_cost_mean = (time_end - time_t3) / (idx - 1) # seconds + self.encoder_time = 0.0 + if do_print: print(f" Start the thread and connect the pipe time: {(time_t2 - time_t1):.2f} s") print(f" Number of input tokens: {input_length}")