LLM: add benchmark script for Max gpu and ipex fp16 gpu (#9112)

* add pvc bash

* meet code review

* rename to run-max-gpu.sh

python/llm/dev/benchmark/all-in-one/README.md

@@ -27,8 +27,9 @@ test_api:
   - "native_int4"
   - "optimize_model"
   - "pytorch_autocast_bf16"
-  # - "transformer_int4_gpu"  # on arc
+  # - "ipex_fp16_gpu" # on Intel GPU
-  # - "optimize_model_gpu"  # on arc
+  # - "transformer_int4_gpu"  # on Intel GPU
+  # - "optimize_model_gpu"  # on Intel GPU
 ```
 
 ## Run

python/llm/dev/benchmark/all-in-one/config.yaml

@@ -13,5 +13,6 @@ test_api:
   - "native_int4"
   - "optimize_model"
   - "pytorch_autocast_bf16"
-  # - "transformer_int4_gpu"  # on arc
+  # - "ipex_fp16_gpu" # on Intel GPU
-  # - "optimize_model_gpu"  # on arc
+  # - "transformer_int4_gpu"  # on Intel GPU
+  # - "optimize_model_gpu"  # on Intel GPU

python/llm/dev/benchmark/all-in-one/run-max-gpu.sh

@@ -0,0 +1,7 @@
+source /opt/intel/oneapi/setvars.sh
+export LD_PRELOAD=${LD_PRELOAD}:${CONDA_PREFIX}/lib/libtcmalloc.so
+export USE_XETLA=OFF
+export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
+export ENABLE_SDP_FUSION=1
+
+python run.py # make sure config YAML file

python/llm/dev/benchmark/all-in-one/run.py

@@ -47,6 +47,8 @@ def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1,
         result = run_optimize_model_gpu(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials)
     elif test_api == 'pytorch_autocast_bf16':
         result = run_pytorch_autocast_bf16(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials)
+    elif test_api == 'ipex_fp16_gpu':
+        result = run_ipex_fp16_gpu(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials)
 
     for in_out_pair in in_out_pairs:
         results.append([repo_id,
@@ -388,6 +390,61 @@ def run_optimize_model_gpu(repo_id,
     return result
 
 
+def run_ipex_fp16_gpu(repo_id,
+                      local_model_hub,
+                      in_out_pairs,
+                      warm_up,
+                      num_trials):
+    from transformers import AutoModel, AutoModelForCausalLM
+    from transformers import AutoTokenizer, GPTJForCausalLM
+    import intel_extension_for_pytorch as ipex
+    model_path = get_model_path(repo_id, local_model_hub)
+    st = time.perf_counter()
+    if repo_id in ['THUDM/chatglm-6b', 'THUDM/chatglm2-6b']:
+        model = AutoModel.from_pretrained(model_path, trust_remote_code=True, use_cache=True)
+        tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+        model = model.half().to('xpu')
+    else:
+        model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True, use_cache=True)
+        tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+        model = model.half().to('xpu')
+        if isinstance(model, GPTJForCausalLM):
+            # For gpt-j model family, this optimization can provide a better performance.
+            model = ipex.optimize(model.eval(), inplace=True)
+    end = time.perf_counter()
+    print(">> loading of model costs {}s".format(end - st))
+
+    model = BenchmarkWrapper(model)
+
+    result = {}
+    with torch.inference_mode():
+        for in_out in in_out_pairs:
+            in_out_len = in_out.split("-")
+            in_len = int(in_out_len[0])
+            out_len = int(in_out_len[1])
+            input_str = open(f"prompt/{in_len}.txt", 'r').read()
+            # As different tokenizer has different encodings,
+            # slice the input_ids to ensure the prompt length is required length.
+            input_ids = tokenizer.encode(input_str, return_tensors="pt")
+            input_ids = input_ids[:, :in_len]
+            true_str = tokenizer.batch_decode(input_ids)[0]
+            input_ids = tokenizer.encode(true_str, return_tensors="pt").to('xpu')
+            result[in_out] = []
+            for i in range(num_trials + warm_up):
+                st = time.perf_counter()
+                output_ids = model.generate(input_ids, do_sample=False, max_new_tokens=out_len)
+                torch.xpu.synchronize()
+                end = time.perf_counter()
+                output_ids = output_ids.cpu()
+                print("model generate cost: " + str(end - st))
+                output = tokenizer.batch_decode(output_ids)
+                print(output[0])
+                if i >= warm_up:
+                    result[in_out].append([model.first_cost, model.rest_cost_mean, model.encoder_time])
+    torch.xpu.empty_cache()
+    return result
+
+
 if __name__ == '__main__':
     from omegaconf import OmegaConf
     conf = OmegaConf.load(f'{current_dir}/config.yaml')