ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

[LLM]Support q5_0 on arc (#8926)

Browse source 
* support q5_0

* delete

* fix style
This commit is contained in:
Yina Chen 2023-09-08 15:52:36 +08:00 committed by GitHub
parent ca35c93825
commit 33d75adadf
1 changed files with 12 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
python/llm/src/bigdl/llm/transformers/low_bit_linear.py
View file

@ -100,6 +100,12 @@ def ggml_q_format_convet_cpu2xpu(tensor: torch.Tensor, num_elem: int, qtype: int
if qtype == ggml_tensor_qtype["sym_int4"]: if qtype == ggml_tensor_qtype["sym_int4"]:
dst_tensor = torch.empty_like(tensor) dst_tensor = torch.empty_like(tensor)
elif qtype == ggml_tensor_qtype["sym_int5"]:
QK = ggml.ggml_qk_size(qtype)
block_size_in_bytes = ggml.ggml_type_size(ggml_tensor_qtype["asym_int5"])
dst_size = (num_elem // QK) * block_size_in_bytes
dst_tensor = torch.empty(dst_size, dtype=torch.uint8,
device=torch.device('cpu'))
else: else:
return tensor return tensor
dst = ctypes.c_void_p(dst_tensor.data.data_ptr()) dst = ctypes.c_void_p(dst_tensor.data.data_ptr())
@ -119,6 +125,12 @@ def ggml_q_format_convet_xpu2cpu(tensor: torch.Tensor, num_elem: int, qtype: int
if qtype == ggml_tensor_qtype["sym_int4"]: if qtype == ggml_tensor_qtype["sym_int4"]:
dst_tensor = torch.empty_like(tensor) dst_tensor = torch.empty_like(tensor)
elif qtype == ggml_tensor_qtype["sym_int5"]:
QK = ggml.ggml_qk_size(ggml_tensor_qtype["asym_int5"])
block_size_in_bytes = ggml.ggml_type_size(qtype)
dst_size = (num_elem // QK) * block_size_in_bytes
dst_tensor = torch.empty(dst_size, dtype=torch.uint8,
device=torch.device('cpu'))
else: else:
return tensor return tensor
dst = ctypes.c_void_p(dst_tensor.data.data_ptr()) dst = ctypes.c_void_p(dst_tensor.data.data_ptr())