LLM: Enable BigDL IPEX Int8 (#10480)

Enable BigDL IPEX Int8

python/llm/dev/benchmark/all-in-one/config.yaml

@@ -20,6 +20,7 @@ test_api:
   # - "transformer_autocast_bf16"
   # - "bigdl_ipex_bf16"
   # - "bigdl_ipex_int4"
+  # - "bigdl_ipex_int8"
   # - "ipex_fp16_gpu" # on Intel GPU
   # - "bigdl_fp16_gpu" # on Intel GPU
   # - "transformer_int4_gpu"  # on Intel GPU

python/llm/dev/benchmark/all-in-one/run.py

@@ -98,6 +98,8 @@ def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1,
         result = run_bigdl_ipex_bf16(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, batch_size)
     elif test_api == 'bigdl_ipex_int4':
         result = run_bigdl_ipex_int4(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, batch_size)
+    elif test_api == 'bigdl_ipex_int8':
+        result = run_bigdl_ipex_int8(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, batch_size)
     elif test_api == 'deepspeed_optimize_model_gpu':
         result = run_deepspeed_optimize_model_gpu(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, batch_size)
 
@@ -1337,6 +1339,76 @@ def run_bigdl_ipex_int4(repo_id,
     return result
 
 
+def run_bigdl_ipex_int8(repo_id,
+                    local_model_hub,
+                    in_out_pairs,
+                    warm_up,
+                    num_trials,
+                    num_beams,
+                    batch_size):
+    from bigdl.llm.transformers import AutoModel, AutoModelForCausalLM
+    from transformers import AutoTokenizer, LlamaTokenizer
+
+    os.environ["BIGDL_OPT_IPEX"] = "true"
+
+    model_path = get_model_path(repo_id, local_model_hub)
+
+    st = time.perf_counter()
+    if repo_id in CHATGLM_IDS:
+        model = AutoModel.from_pretrained(model_path, load_in_low_bit='sym_int8', trust_remote_code=True, torch_dtype='auto',
+                                          use_cache=True, torchscript=True)
+        tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+    elif repo_id in LLAMA_IDS:
+        model = AutoModelForCausalLM.from_pretrained(model_path, load_in_low_bit='sym_int8', trust_remote_code=True, torch_dtype='auto',
+                                                     use_cache=True, torchscript=True)
+        tokenizer = LlamaTokenizer.from_pretrained(model_path, trust_remote_code=True)
+    else:
+        model = AutoModelForCausalLM.from_pretrained(model_path, load_in_low_bit='sym_int8', trust_remote_code=True, torch_dtype='auto',
+                                                     use_cache=True, torchscript=True)
+        tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+    if not hasattr(model.config, "token_latency"):
+        model.config.token_latency = True
+    end = time.perf_counter()
+    load_time = end - st
+    print(">> loading of model costs {}s".format(load_time))
+
+    result = {}
+    with torch.inference_mode(), torch.autocast("cpu"):
+        for in_out in in_out_pairs:
+            in_out_len = in_out.split("-")
+            in_len = int(in_out_len[0])
+            out_len = int(in_out_len[1])
+            # As different tokenizer has different encodings,
+            # in_len.txt maybe shorter than we need,
+            # use much longer context to make sure input length
+            test_length = min(in_len*2, 8192)
+            while test_length not in [32, 256, 1024, 2048, 8192]:
+                test_length = test_length * 2
+            input_str = open(f"prompt/{test_length}.txt", 'r').read()
+            # As different tokenizer has different encodings,
+            # slice the input_ids to ensure the prompt length is required length.
+            input_ids = tokenizer.encode(input_str, return_tensors="pt")
+            input_ids = input_ids[:, :in_len]
+            true_str = tokenizer.batch_decode(input_ids)[0]
+            input_list = [true_str] * batch_size
+            input_ids = tokenizer(input_list, return_tensors="pt").input_ids
+            actual_in_len = input_ids.shape[1]
+            result[in_out] = []
+            for i in range(num_trials + warm_up):
+                st = time.perf_counter()
+                output_ids, total_list = model.generate(input_ids, do_sample=False, max_new_tokens=out_len,
+                                            num_beams=num_beams)
+                end = time.perf_counter()
+                print("model generate cost: " + str(end - st))
+                output = tokenizer.batch_decode(output_ids)
+                print(output[0])
+                actual_out_len = output_ids.shape[1] - actual_in_len
+                if i >= warm_up:
+                    result[in_out].append([total_list[0], np.mean(total_list[1:]), 0,
+                                          actual_in_len, actual_out_len, load_time])
+    return result
+
+
 def run_deepspeed_optimize_model_gpu(repo_id,
                                      local_model_hub,
                                      in_out_pairs,

python/llm/src/bigdl/llm/transformers/convert_ipex.py

@@ -127,6 +127,22 @@ def _ipex_optimize_model(model, rms_classes, qtype):
             group_size=-1,
         )
         model = ipex_quantization_flow(model, torch.bfloat16, None, qconfig, None)
+    elif qtype == ggml_tensor_qtype["sym_int8"]:
+        is_quantization = True
+        is_woq = True
+        act_quant_mode_dict = {
+            "PER_TENSOR": ipex.quantization.WoqActQuantMode.PER_TENSOR,
+            "PER_IC_BLOCK": ipex.quantization.WoqActQuantMode.PER_IC_BLOCK,
+            "PER_BATCH": ipex.quantization.WoqActQuantMode.PER_BATCH,
+            "PER_BATCH_IC_BLOCK": ipex.quantization.WoqActQuantMode.PER_BATCH_IC_BLOCK,
+        }
+        qconfig = ipex.quantization.get_weight_only_quant_qconfig_mapping(
+            weight_dtype=torch.qint8,  # INT8
+            lowp_mode=ipex.quantization.WoqLowpMode.INT8,
+            act_quant_mode=act_quant_mode_dict["PER_IC_BLOCK"],
+            group_size=-1,
+        )
+        model = ipex_quantization_flow(model, torch.bfloat16, None, qconfig, None)
 
     is_tpp = _using_tpp()