From 7e8fb29b7c5b4544bceb3552feb86d53bce547b2 Mon Sep 17 00:00:00 2001 From: Ruonan Wang <105281011+rnwang04@users.noreply.github.com> Date: Wed, 8 Nov 2023 13:14:34 +0800 Subject: [PATCH] LLM: optimize QLoRA by reducing convert time (#9370) --- python/llm/src/bigdl/llm/transformers/low_bit_linear.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/llm/src/bigdl/llm/transformers/low_bit_linear.py b/python/llm/src/bigdl/llm/transformers/low_bit_linear.py index 16c8d334..6715dbce 100644 --- a/python/llm/src/bigdl/llm/transformers/low_bit_linear.py +++ b/python/llm/src/bigdl/llm/transformers/low_bit_linear.py @@ -448,6 +448,8 @@ class LowBitLinear(nn.Linear): input_seq_size) result = result.to(x.dtype) else: + if torch.xpu.is_autocast_xpu_enabled(): + x_2d = x_2d.to(torch.xpu.get_autocast_xpu_dtype()) result = linear_q4_0.forward_new(x_2d, self.weight.data, self.weight.qtype, input_seq_size) new_shape = x_shape[:-1] + (self.out_len,)