From ad7d9231f53f4a1f2e9458cf361ab600e12cf48b Mon Sep 17 00:00:00 2001 From: Ruonan Wang <105281011+rnwang04@users.noreply.github.com> Date: Tue, 10 Oct 2023 10:18:41 +0800 Subject: [PATCH] LLM: add benchmark script for Max gpu and ipex fp16 gpu (#9112) * add pvc bash * meet code review * rename to run-max-gpu.sh --- python/llm/dev/benchmark/all-in-one/README.md | 5 +- .../llm/dev/benchmark/all-in-one/config.yaml | 5 +- .../dev/benchmark/all-in-one/run-max-gpu.sh | 7 +++ python/llm/dev/benchmark/all-in-one/run.py | 57 +++++++++++++++++++ 4 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 python/llm/dev/benchmark/all-in-one/run-max-gpu.sh diff --git a/python/llm/dev/benchmark/all-in-one/README.md b/python/llm/dev/benchmark/all-in-one/README.md index d5b5457b..f4614a56 100644 --- a/python/llm/dev/benchmark/all-in-one/README.md +++ b/python/llm/dev/benchmark/all-in-one/README.md @@ -27,8 +27,9 @@ test_api: - "native_int4" - "optimize_model" - "pytorch_autocast_bf16" - # - "transformer_int4_gpu" # on arc - # - "optimize_model_gpu" # on arc + # - "ipex_fp16_gpu" # on Intel GPU + # - "transformer_int4_gpu" # on Intel GPU + # - "optimize_model_gpu" # on Intel GPU ``` ## Run diff --git a/python/llm/dev/benchmark/all-in-one/config.yaml b/python/llm/dev/benchmark/all-in-one/config.yaml index b89132ba..47b5347e 100644 --- a/python/llm/dev/benchmark/all-in-one/config.yaml +++ b/python/llm/dev/benchmark/all-in-one/config.yaml @@ -13,5 +13,6 @@ test_api: - "native_int4" - "optimize_model" - "pytorch_autocast_bf16" - # - "transformer_int4_gpu" # on arc - # - "optimize_model_gpu" # on arc \ No newline at end of file + # - "ipex_fp16_gpu" # on Intel GPU + # - "transformer_int4_gpu" # on Intel GPU + # - "optimize_model_gpu" # on Intel GPU \ No newline at end of file diff --git a/python/llm/dev/benchmark/all-in-one/run-max-gpu.sh b/python/llm/dev/benchmark/all-in-one/run-max-gpu.sh new file mode 100644 index 00000000..9129a4b9 --- /dev/null +++ b/python/llm/dev/benchmark/all-in-one/run-max-gpu.sh @@ -0,0 +1,7 @@ +source /opt/intel/oneapi/setvars.sh +export LD_PRELOAD=${LD_PRELOAD}:${CONDA_PREFIX}/lib/libtcmalloc.so +export USE_XETLA=OFF +export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 +export ENABLE_SDP_FUSION=1 + +python run.py # make sure config YAML file diff --git a/python/llm/dev/benchmark/all-in-one/run.py b/python/llm/dev/benchmark/all-in-one/run.py index 9d3115e8..088c21ff 100644 --- a/python/llm/dev/benchmark/all-in-one/run.py +++ b/python/llm/dev/benchmark/all-in-one/run.py @@ -47,6 +47,8 @@ def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1, result = run_optimize_model_gpu(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials) elif test_api == 'pytorch_autocast_bf16': result = run_pytorch_autocast_bf16(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials) + elif test_api == 'ipex_fp16_gpu': + result = run_ipex_fp16_gpu(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials) for in_out_pair in in_out_pairs: results.append([repo_id, @@ -388,6 +390,61 @@ def run_optimize_model_gpu(repo_id, return result +def run_ipex_fp16_gpu(repo_id, + local_model_hub, + in_out_pairs, + warm_up, + num_trials): + from transformers import AutoModel, AutoModelForCausalLM + from transformers import AutoTokenizer, GPTJForCausalLM + import intel_extension_for_pytorch as ipex + model_path = get_model_path(repo_id, local_model_hub) + st = time.perf_counter() + if repo_id in ['THUDM/chatglm-6b', 'THUDM/chatglm2-6b']: + model = AutoModel.from_pretrained(model_path, trust_remote_code=True, use_cache=True) + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + model = model.half().to('xpu') + else: + model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True, use_cache=True) + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + model = model.half().to('xpu') + if isinstance(model, GPTJForCausalLM): + # For gpt-j model family, this optimization can provide a better performance. + model = ipex.optimize(model.eval(), inplace=True) + end = time.perf_counter() + print(">> loading of model costs {}s".format(end - st)) + + model = BenchmarkWrapper(model) + + result = {} + with torch.inference_mode(): + for in_out in in_out_pairs: + in_out_len = in_out.split("-") + in_len = int(in_out_len[0]) + out_len = int(in_out_len[1]) + input_str = open(f"prompt/{in_len}.txt", 'r').read() + # As different tokenizer has different encodings, + # slice the input_ids to ensure the prompt length is required length. + input_ids = tokenizer.encode(input_str, return_tensors="pt") + input_ids = input_ids[:, :in_len] + true_str = tokenizer.batch_decode(input_ids)[0] + input_ids = tokenizer.encode(true_str, return_tensors="pt").to('xpu') + result[in_out] = [] + for i in range(num_trials + warm_up): + st = time.perf_counter() + output_ids = model.generate(input_ids, do_sample=False, max_new_tokens=out_len) + torch.xpu.synchronize() + end = time.perf_counter() + output_ids = output_ids.cpu() + print("model generate cost: " + str(end - st)) + output = tokenizer.batch_decode(output_ids) + print(output[0]) + if i >= warm_up: + result[in_out].append([model.first_cost, model.rest_cost_mean, model.encoder_time]) + torch.xpu.empty_cache() + return result + + if __name__ == '__main__': from omegaconf import OmegaConf conf = OmegaConf.load(f'{current_dir}/config.yaml')