From 8a3247ac713080ac5378c7645a341cfb2d11e93b Mon Sep 17 00:00:00 2001 From: Ruonan Wang Date: Fri, 14 Jun 2024 18:25:50 +0800 Subject: [PATCH] support batch forward for q4_k, q6_k (#11325) --- python/llm/src/ipex_llm/transformers/low_bit_linear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/llm/src/ipex_llm/transformers/low_bit_linear.py b/python/llm/src/ipex_llm/transformers/low_bit_linear.py index 86a689ee..37a9780f 100644 --- a/python/llm/src/ipex_llm/transformers/low_bit_linear.py +++ b/python/llm/src/ipex_llm/transformers/low_bit_linear.py @@ -332,7 +332,7 @@ def use_batch_forward(x: torch.Tensor, qtype: int, output_len: int): and output_len % 32 == 0 and device in ["arc", "flex", "pvc", "mtl"] and qtype in [SYM_INT4, ASYM_INT4, SYM_INT8, FP4, - FP8E5, FP6, FP8E4] + FP8E5, FP6, FP8E4, Q4_K, Q6_K] and batch_size <= 64 ) if hard_condition: