Add transformers_int4_npu_pipeline_win in all-in-one benchmark (#12325)

* add transformers_int4_npu_pipeline_win

* bugfix

* bugfix: wrong actual_output_len

* fix format

* bugfix & update `README.md`

python/llm/dev/benchmark/all-in-one/README.md

@@ -27,6 +27,7 @@ Config YAML file has following format
 repo_id:
   # - 'THUDM/chatglm2-6b'
   - 'meta-llama/Llama-2-7b-chat-hf'
+  # - 'meta-llama/Meta-Llama-3.1-8B-Instruct'
   # - 'liuhaotian/llava-v1.5-7b' # requires a LLAVA_REPO_DIR env variables pointing to the llava dir; added only for gpu win related test_api now
 local_model_hub: 'path to your local model hub'
 warm_up: 1 # must set >=2 when run "pipeline_parallel_gpu" test_api
@@ -36,6 +37,7 @@ low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
 batch_size: 1 # default to 1
 in_out_pairs:
   - '32-32'
+  - '960-64'
   - '1024-128'
 test_api:
   - "transformer_int4_fp16_gpu"             # on Intel GPU, transformer-like API, (qtype=int4), (dtype=fp16)
@@ -59,11 +61,16 @@ test_api:
   # - "bigdl_ipex_int8"                     # on Intel CPU, (qtype=int8)
   # - "speculative_cpu"                     # on Intel CPU, inference with self-speculative decoding
   # - "deepspeed_transformer_int4_cpu"      # on Intel CPU, deepspeed autotp inference
-  # - "transformers_int4_npu_win"           # on Intel NPU for Windows, transformer-like API, (qtype=int4)
+  # - "transformers_int4_npu_win"           # on Intel NPU for Windows,  transformer-like API, (qtype=int4)
+  # - "transformers_int4_loadlowbit_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4), use load_low_bit API. Please make sure you have used the save_npu.py to save the converted low bit model
+  # - "transformers_int4_npu_pipeline_win"  # on Intel NPU for Windows,  transformer-like API, (qtype=int4)
 cpu_embedding: False # whether put embedding to CPU
 streaming: False # whether output in streaming way (only available now for gpu win related test_api)
+optimize_model: False # whether apply further optimization on NPU (only available now for transformers_int4_npu_win test_api)
 use_fp16_torch_dtype: True # whether use fp16 for non-linear layer (only available now for "pipeline_parallel_gpu" test_api)
 task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
+transpose_value_cache: True # whether apply transposed v_cache optimization on NPU (only available now for transformers_int4_npu_win test_api)
+npu_group_size: 0 # this can only be either 0 or 128, and only works for `transformers_int4_npu_win` / `transformers_int4_npu_pipeline_win`
 
 ```
 

python/llm/dev/benchmark/all-in-one/config.yaml

@@ -37,10 +37,11 @@ test_api:
   # - "deepspeed_transformer_int4_cpu"      # on Intel CPU, deepspeed autotp inference
   # - "transformers_int4_npu_win"           # on Intel NPU for Windows,  transformer-like API, (qtype=int4)
   # - "transformers_int4_loadlowbit_npu_win" # on Intel NPU for Windows, transformer-like API, (qtype=int4), use load_low_bit API. Please make sure you have used the save_npu.py to save the converted low bit model
+  # - "transformers_int4_npu_pipeline_win"  # on Intel NPU for Windows,  transformer-like API, (qtype=int4)
 cpu_embedding: False # whether put embedding to CPU
 streaming: False # whether output in streaming way (only available now for gpu win related test_api)
 optimize_model: False # whether apply further optimization on NPU (only available now for transformers_int4_npu_win test_api)
 use_fp16_torch_dtype: True # whether use fp16 for non-linear layer (only available now for "pipeline_parallel_gpu" test_api)
 task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
 transpose_value_cache: True # whether apply transposed v_cache optimization on NPU (only available now for transformers_int4_npu_win test_api)
-npu_group_size: 128 # This can only be either 0 or 128, and only works for `transformers_int4_npu_win` / `transformers_int4_npu_pipline_win`
+npu_group_size: 0 # this can only be either 0 or 128, and only works for `transformers_int4_npu_win` / `transformers_int4_npu_pipeline_win`

python/llm/dev/benchmark/all-in-one/run.py

@@ -191,6 +191,8 @@ def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1,
         result = run_pipeline_parallel_gpu(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, cpu_embedding, fp16=use_fp16_torch_dtype)
     elif test_api == 'transformers_int4_npu_win':
         result = transformers_int4_npu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, optimize_model, transpose_value_cache, group_size)
+    elif test_api == 'transformers_int4_npu_pipeline_win':
+        result = transformers_int4_npu_pipeline_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, optimize_model, transpose_value_cache, group_size)
     elif test_api == 'transformers_int4_loadlowbit_npu_win':
         result = run_transformer_int4_loadlowbit_npu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size, optimize_model, transpose_value_cache)
     elif test_api == 'transformers_openvino':
@@ -215,7 +217,7 @@ def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1,
                             result[in_out_pair][-1][6] if any(keyword in test_api for keyword in ['int4_gpu', 'int4_fp16_gpu_win', 'int4_loadlowbit_gpu', 'int4_fp16_loadlowbit_gpu', 'fp16_gpu', 'deepspeed_optimize_model_gpu']) and not lookahead else 'N/A',
                             streaming if 'win' in test_api else 'N/A',
                             use_fp16_torch_dtype if 'pipeline_parallel_gpu' in test_api else 'N/A',
-                            group_size if 'transformers_int4_npu_win' in test_api else 'N/A'],
+                            group_size if any(keyword in test_api for keyword in ['transformers_int4_npu_win', 'transformers_int4_npu_pipeline_win']) else 'N/A'],
                             ) 
 
 
@@ -680,6 +682,70 @@ def transformers_int4_npu_win(repo_id,
     gc.collect()
     return result
 
+def transformers_int4_npu_pipeline_win(repo_id,
+                                       local_model_hub,
+                                       in_out_pairs,
+                                       warm_up,
+                                       num_trials,
+                                       num_beams,
+                                       low_bit,
+                                       batch_size,
+                                       optimize_model,
+                                       transpose_value_cache,
+                                       npu_group_size):
+    from ipex_llm.transformers.npu_model import AutoModel, AutoModelForCausalLM
+    from transformers import AutoTokenizer, LlamaTokenizer
+
+    model_path = get_model_path(repo_id, local_model_hub)
+    in_out_len = in_out_pairs[0].split("-")
+    max_context_len = max(int(in_out_len[0]) + int(in_out_len[1]), 1024)
+    # Load model in 4 bit,
+    # which convert the relevant layers in the model into INT4 format
+    st = time.perf_counter()
+
+    model = AutoModelForCausalLM.from_pretrained(model_path, load_in_low_bit=low_bit, trust_remote_code=True, pipeline=True, torch_dtype=torch.float16,
+                                                 optimize_model=optimize_model, max_context_len=max_context_len, max_prompt_len=int(in_out_len[0]), 
+                                                 quantization_group_size=npu_group_size, transpose_value_cache=transpose_value_cache,
+                                                 use_cache=True, attn_implementation="eager").eval()
+    tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+
+    end = time.perf_counter()
+    load_time = end - st
+    print(">> loading of model costs {}s".format(load_time))
+
+    result = {}
+    with torch.inference_mode():
+        for in_out in in_out_pairs:
+            in_out_len = in_out.split("-")
+            in_len = int(in_out_len[0])
+            out_len = int(in_out_len[1])
+            input_str = get_continuation_input_str(in_len, tokenizer)
+            # As different tokenizer has different encodings,
+            # slice the input_ids to ensure the prompt length is required length.
+            input_ids = tokenizer.encode(input_str, return_tensors="pt")
+            input_ids = input_ids[:, :in_len]
+            true_str = tokenizer.batch_decode(input_ids)[0]
+            input_list = [true_str] * batch_size
+            input_ids = tokenizer(input_list, return_tensors="pt").input_ids
+            input_ids = input_ids[:, :in_len]
+            actual_in_len = input_ids.shape[1]
+            result[in_out] = []
+            for i in range(num_trials + warm_up):
+                st = time.perf_counter()
+                output_ids = model.generate(input_ids, do_sample=False, max_new_tokens=out_len,
+                                            min_new_tokens=out_len, num_beams=num_beams)
+                end = time.perf_counter()
+                print("model generate cost: " + str(end - st))
+                output = tokenizer.batch_decode(output_ids)
+                print(output[0])
+                actual_out_len = output_ids.shape[1] - actual_in_len
+                if i >= warm_up:
+                    result[in_out].append([model.first_cost, model.rest_cost_mean, model.encoder_time,
+                                           actual_in_len, actual_out_len, load_time])
+    del model
+    gc.collect()
+    return result
+
 def run_transformer_int4_loadlowbit_npu_win(repo_id,
                                             local_model_hub,
                                             in_out_pairs,
@@ -2186,7 +2252,7 @@ if __name__ == '__main__':
     streaming = False
     use_fp16_torch_dtype = False
     task = 'continuation'
-    optimize_model = False # only for transformers_int4_npu_win
+    optimize_model = False # only for transformers_int4_npu_win, transformers_int4_npu_pipeline_win
     group_size = 64
     if 'streaming' in conf:
         streaming = conf['streaming']
@@ -2233,7 +2299,7 @@ if __name__ == '__main__':
         df = pd.DataFrame(results, columns=['model', '1st token avg latency (ms)', '2+ avg latency (ms/token)', 'encoder time (ms)',
                                             'input/output tokens', 'batch_size', 'actual input/output tokens', 'num_beams', 'low_bit', 'cpu_embedding',
                                             'model loading time (s)', 'peak mem (GB)', 'streaming', 'use_fp16_torch_dtype', 'npu_group_size'])
-        if "pipeline" in api or "deepspeed" in api:
+        if ("pipeline" in api or "deepspeed" in api) and api != 'transformers_int4_npu_pipeline_win':
             if torch.distributed.get_rank() == 0:
                 df.index += max(line_counter - 1, 0)
                 if line_counter == 0:

python/llm/src/ipex_llm/transformers/npu_pipeline_model/convert_pipeline.py

@@ -172,6 +172,10 @@ def generate(
     thread.join()
     time_end = time.perf_counter()
 
+    self.first_cost = (time_t3 - time_start_all - (time_t2 - time_t1))  # seconds
+    self.rest_cost_mean = (time_end - time_t3) / (idx - 1)  # seconds
+    self.encoder_time = 0.0
+
     if do_print:
         print(f" Start the thread and connect the pipe time: {(time_t2 - time_t1):.2f} s")
         print(f" Number of input tokens: {input_length}")