ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

support q8_0 (#8765)

Browse source
This commit is contained in:
Yina Chen 2023-08-17 15:06:36 +08:00 committed by GitHub
parent 394304b918
commit 4afea496ab
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
python/llm/src/bigdl/llm/transformers/linear_quant.py
View file

@ -247,7 +247,7 @@ class LinearQuant(nn.Linear):
if x_2d.shape[0] > 1 and x_2d.dtype == torch.float32: if x_2d.shape[0] > 1 and x_2d.dtype == torch.float32:
x_2d = x_2d.half() x_2d = x_2d.half()
# input format of linear_q4.forward is 1: input, 2: weight # input format of linear_q4.forward is 1: input, 2: weight
result = linear_q4_0.forward(x_2d, x0) result = linear_q4_0.forward(x_2d, x0, self.qtype)
new_shape = x_shape[:-1] + (self.out_len,) new_shape = x_shape[:-1] + (self.out_len,)
result = result.view(new_shape) result = result.view(new_shape)
if self.bias is not None: if self.bias is not None: