ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

[LLM] Fix arc qtype != q4_0 generate issue (#8920)

Browse source 
* Fix arc precision!=q4_0 generate issue

* meet comments
This commit is contained in:
Yina Chen 2023-09-07 23:56:36 +08:00 committed by GitHub
parent 3d2efe9608
commit b209b8f7b6
1 changed files with 8 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
python/llm/src/bigdl/llm/transformers/low_bit_linear.py
View file

@ -98,7 +98,10 @@ def ggml_q_format_convet_cpu2xpu(tensor: torch.Tensor, num_elem: int, qtype: int
src = ctypes.c_void_p(tensor.data.data_ptr()) src = ctypes.c_void_p(tensor.data.data_ptr())
dst_tensor = torch.empty_like(tensor) if qtype == ggml_tensor_qtype["sym_int4"]:
dst_tensor = torch.empty_like(tensor)
else:
return tensor
dst = ctypes.c_void_p(dst_tensor.data.data_ptr()) dst = ctypes.c_void_p(dst_tensor.data.data_ptr())
ggml.ggml_q_format_convet_cpu2xpu(src, dst, num_elem, qtype) ggml.ggml_q_format_convet_cpu2xpu(src, dst, num_elem, qtype)
return dst_tensor return dst_tensor
@ -114,7 +117,10 @@ def ggml_q_format_convet_xpu2cpu(tensor: torch.Tensor, num_elem: int, qtype: int
src = ctypes.c_void_p(tensor.data.data_ptr()) src = ctypes.c_void_p(tensor.data.data_ptr())
dst_tensor = torch.empty_like(tensor) if qtype == ggml_tensor_qtype["sym_int4"]:
dst_tensor = torch.empty_like(tensor)
else:
return tensor
dst = ctypes.c_void_p(dst_tensor.data.data_ptr()) dst = ctypes.c_void_p(dst_tensor.data.data_ptr())
ggml.ggml_q_format_convet_xpu2cpu(src, dst, num_elem, qtype) ggml.ggml_q_format_convet_xpu2cpu(src, dst, num_elem, qtype)
return dst_tensor return dst_tensor