ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

woq int4 fwd (#12711)

Browse source
This commit is contained in:
Zhao Changmin 2025-01-16 15:48:05 +08:00 committed by GitHub
parent b62734748f
commit 54d6328b3c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 4 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
python/llm/src/ipex_llm/transformers/low_bit_linear.py
View file

@ -273,7 +273,7 @@ def use_batch_forward(x: torch.Tensor, qtype: int, output_len: int):
and x.shape[1] % 128 == 0
and (
(
qtype in [SYM_INT4, ASYM_INT4, FP8E5, FP8E4]
qtype in [SYM_INT4, ASYM_INT4, FP8E5, FP8E4, WOQ_INT4]
and (
batch_size <= 48
or (
@ -284,7 +284,7 @@ def use_batch_forward(x: torch.Tensor, qtype: int, output_len: int):
)
)
or (
qtype in [SYM_INT8, FP4, FP6, Q4_K, Q6_K]
qtype in [SYM_INT8, FP4, FP6, Q4_K, Q6_K, WOQ_INT4]
and batch_size <= 48
and device_name in ["arc", "pvc", "mtl", "arl"]
and x.shape[1] % 256 == 0
@ -297,8 +297,8 @@ def use_batch_forward(x: torch.Tensor, qtype: int, output_len: int):
batch_size > 1
or (device_name in ["arc"] and qtype in [SYM_INT8, FP4])
or (device_name in ["arc", "mtl"] and qtype in [FP8E4])
or (device_name in ["lnl"] and qtype in [SYM_INT4] and x.shape[1] % 512 == 0)
or (device_name in ["bmg"] and qtype in [SYM_INT4, FP8E5])
or (device_name in ["lnl"] and qtype in [SYM_INT4, WOQ_INT4] and x.shape[1] % 512 == 0)
or (device_name in ["bmg"] and qtype in [SYM_INT4, WOQ_INT4, FP8E5])
)
return False