ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

use new q4_0 batch kernel (#12396)

Browse source
This commit is contained in:
Yishuo Wang 2024-11-13 18:37:34 +08:00 committed by GitHub
parent d6d63d6b84
commit 00fce5c940
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 10 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
python/llm/src/ipex_llm/transformers/low_bit_linear.py
View file

@ -362,12 +362,16 @@ def use_batch_forward(x: torch.Tensor, qtype: int, output_len: int):
batch_size = x.shape[0] batch_size = x.shape[0]
hard_condition = ( hard_condition = (
x.dtype in [torch.float, torch.half] x.dtype in [torch.float, torch.half]
and x.shape[1] % 256 == 0 and (
and output_len % 32 == 0 x.shape[1] % 128 == 0 and qtype in [SYM_INT4]
and device in ["arc", "flex", "pvc", "mtl"] or (
and qtype in [SYM_INT4, ASYM_INT4, SYM_INT8, FP4, x.shape[1] % 256 == 0
FP8E5, FP6, FP8E4, Q4_K, Q6_K] and output_len % 32 == 0
and batch_size <= 64 and device in ["arc", "flex", "pvc", "mtl"]
and qtype in [ASYM_INT4, SYM_INT8, FP4, FP8E5, FP6, FP8E4, Q4_K, Q6_K]
)
)
and batch_size <= 48
) )
if hard_condition: if hard_condition:
return ( return (