From cbe24cc7e680d055838cd62afacd3d07349aa66a Mon Sep 17 00:00:00 2001 From: Xiangyu Tian <109123695+xiangyuT@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:59:54 +0800 Subject: [PATCH] LLM: Enable BigDL IPEX Int8 (#10480) Enable BigDL IPEX Int8 --- .../llm/dev/benchmark/all-in-one/config.yaml | 1 + python/llm/dev/benchmark/all-in-one/run.py | 72 +++++++++++++++++++ .../bigdl/llm/transformers/convert_ipex.py | 16 +++++ 3 files changed, 89 insertions(+) diff --git a/python/llm/dev/benchmark/all-in-one/config.yaml b/python/llm/dev/benchmark/all-in-one/config.yaml index d788427e..501e4e80 100644 --- a/python/llm/dev/benchmark/all-in-one/config.yaml +++ b/python/llm/dev/benchmark/all-in-one/config.yaml @@ -20,6 +20,7 @@ test_api: # - "transformer_autocast_bf16" # - "bigdl_ipex_bf16" # - "bigdl_ipex_int4" + # - "bigdl_ipex_int8" # - "ipex_fp16_gpu" # on Intel GPU # - "bigdl_fp16_gpu" # on Intel GPU # - "transformer_int4_gpu" # on Intel GPU diff --git a/python/llm/dev/benchmark/all-in-one/run.py b/python/llm/dev/benchmark/all-in-one/run.py index dfd2a20b..59554495 100644 --- a/python/llm/dev/benchmark/all-in-one/run.py +++ b/python/llm/dev/benchmark/all-in-one/run.py @@ -98,6 +98,8 @@ def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1, result = run_bigdl_ipex_bf16(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, batch_size) elif test_api == 'bigdl_ipex_int4': result = run_bigdl_ipex_int4(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, batch_size) + elif test_api == 'bigdl_ipex_int8': + result = run_bigdl_ipex_int8(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, batch_size) elif test_api == 'deepspeed_optimize_model_gpu': result = run_deepspeed_optimize_model_gpu(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size) @@ -1337,6 +1339,76 @@ def run_bigdl_ipex_int4(repo_id, return result +def run_bigdl_ipex_int8(repo_id, + local_model_hub, + in_out_pairs, + warm_up, + num_trials, + num_beams, + batch_size): + from bigdl.llm.transformers import AutoModel, AutoModelForCausalLM + from transformers import AutoTokenizer, LlamaTokenizer + + os.environ["BIGDL_OPT_IPEX"] = "true" + + model_path = get_model_path(repo_id, local_model_hub) + + st = time.perf_counter() + if repo_id in CHATGLM_IDS: + model = AutoModel.from_pretrained(model_path, load_in_low_bit='sym_int8', trust_remote_code=True, torch_dtype='auto', + use_cache=True, torchscript=True) + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + elif repo_id in LLAMA_IDS: + model = AutoModelForCausalLM.from_pretrained(model_path, load_in_low_bit='sym_int8', trust_remote_code=True, torch_dtype='auto', + use_cache=True, torchscript=True) + tokenizer = LlamaTokenizer.from_pretrained(model_path, trust_remote_code=True) + else: + model = AutoModelForCausalLM.from_pretrained(model_path, load_in_low_bit='sym_int8', trust_remote_code=True, torch_dtype='auto', + use_cache=True, torchscript=True) + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + if not hasattr(model.config, "token_latency"): + model.config.token_latency = True + end = time.perf_counter() + load_time = end - st + print(">> loading of model costs {}s".format(load_time)) + + result = {} + with torch.inference_mode(), torch.autocast("cpu"): + for in_out in in_out_pairs: + in_out_len = in_out.split("-") + in_len = int(in_out_len[0]) + out_len = int(in_out_len[1]) + # As different tokenizer has different encodings, + # in_len.txt maybe shorter than we need, + # use much longer context to make sure input length + test_length = min(in_len*2, 8192) + while test_length not in [32, 256, 1024, 2048, 8192]: + test_length = test_length * 2 + input_str = open(f"prompt/{test_length}.txt", 'r').read() + # As different tokenizer has different encodings, + # slice the input_ids to ensure the prompt length is required length. + input_ids = tokenizer.encode(input_str, return_tensors="pt") + input_ids = input_ids[:, :in_len] + true_str = tokenizer.batch_decode(input_ids)[0] + input_list = [true_str] * batch_size + input_ids = tokenizer(input_list, return_tensors="pt").input_ids + actual_in_len = input_ids.shape[1] + result[in_out] = [] + for i in range(num_trials + warm_up): + st = time.perf_counter() + output_ids, total_list = model.generate(input_ids, do_sample=False, max_new_tokens=out_len, + num_beams=num_beams) + end = time.perf_counter() + print("model generate cost: " + str(end - st)) + output = tokenizer.batch_decode(output_ids) + print(output[0]) + actual_out_len = output_ids.shape[1] - actual_in_len + if i >= warm_up: + result[in_out].append([total_list[0], np.mean(total_list[1:]), 0, + actual_in_len, actual_out_len, load_time]) + return result + + def run_deepspeed_optimize_model_gpu(repo_id, local_model_hub, in_out_pairs, diff --git a/python/llm/src/bigdl/llm/transformers/convert_ipex.py b/python/llm/src/bigdl/llm/transformers/convert_ipex.py index 5e5421d1..b50f6c04 100644 --- a/python/llm/src/bigdl/llm/transformers/convert_ipex.py +++ b/python/llm/src/bigdl/llm/transformers/convert_ipex.py @@ -127,6 +127,22 @@ def _ipex_optimize_model(model, rms_classes, qtype): group_size=-1, ) model = ipex_quantization_flow(model, torch.bfloat16, None, qconfig, None) + elif qtype == ggml_tensor_qtype["sym_int8"]: + is_quantization = True + is_woq = True + act_quant_mode_dict = { + "PER_TENSOR": ipex.quantization.WoqActQuantMode.PER_TENSOR, + "PER_IC_BLOCK": ipex.quantization.WoqActQuantMode.PER_IC_BLOCK, + "PER_BATCH": ipex.quantization.WoqActQuantMode.PER_BATCH, + "PER_BATCH_IC_BLOCK": ipex.quantization.WoqActQuantMode.PER_BATCH_IC_BLOCK, + } + qconfig = ipex.quantization.get_weight_only_quant_qconfig_mapping( + weight_dtype=torch.qint8, # INT8 + lowp_mode=ipex.quantization.WoqLowpMode.INT8, + act_quant_mode=act_quant_mode_dict["PER_IC_BLOCK"], + group_size=-1, + ) + model = ipex_quantization_flow(model, torch.bfloat16, None, qconfig, None) is_tpp = _using_tpp()