ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

LLM: optimize QLoRA by reducing convert time (#9370)

Browse source
This commit is contained in:
Ruonan Wang 2023-11-08 13:14:34 +08:00 committed by GitHub
parent 298b64217e
commit 7e8fb29b7c
1 changed files with 2 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
python/llm/src/bigdl/llm/transformers/low_bit_linear.py
View file

@ -448,6 +448,8 @@ class LowBitLinear(nn.Linear):
input_seq_size) input_seq_size)
result = result.to(x.dtype) result = result.to(x.dtype)
else: else:
if torch.xpu.is_autocast_xpu_enabled():
x_2d = x_2d.to(torch.xpu.get_autocast_xpu_dtype())
result = linear_q4_0.forward_new(x_2d, self.weight.data, self.weight.qtype, result = linear_q4_0.forward_new(x_2d, self.weight.data, self.weight.qtype,
input_seq_size) input_seq_size)
new_shape = x_shape[:-1] + (self.out_len,) new_shape = x_shape[:-1] + (self.out_len,)