ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

fp8 convert use ggml code (#9277)

Browse source
This commit is contained in:
Yina Chen 2023-10-26 17:03:29 +08:00 committed by GitHub
parent 4ed7f066d3
commit f879c48f98
1 changed files with 5 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
python/llm/src/bigdl/llm/transformers/low_bit_linear.py
View file

@ -89,13 +89,9 @@ def ggml_convert_qtype(tensor: torch.Tensor, qtype: int,
device=device)
if not convert_shape_only and device != 'meta':
if qtype == FP8:
import linear_q4_0
linear_q4_0.cvt_fp32_e4m3_rne(tensor, dst_tensor, n, k)
else:
dst = ctypes.c_void_p(dst_tensor.data.data_ptr())
hist = (ctypes.c_int64 * 16)()
ggml.ggml_quantize_tensor(src, dst, qtype, n, k, hist)
dst = ctypes.c_void_p(dst_tensor.data.data_ptr())
hist = (ctypes.c_int64 * 16)()
ggml.ggml_quantize_tensor(src, dst, qtype, n, k, hist)
return dst_tensor
@ -109,7 +105,7 @@ def ggml_q_format_convet_cpu2xpu(tensor: torch.Tensor, num_elem: int, qtype: int
src = ctypes.c_void_p(tensor.data.data_ptr())
if qtype in [SYM_INT4, SYM_INT8, NF4, NF3, FP4]:
if qtype in [SYM_INT4, SYM_INT8, NF4, NF3, FP4, FP8]:
dst_tensor = torch.empty_like(tensor)
elif qtype == ggml_tensor_qtype["sym_int5"]:
QK = ggml.ggml_qk_size(qtype)
@ -134,7 +130,7 @@ def ggml_q_format_convet_xpu2cpu(tensor: torch.Tensor, num_elem: int, qtype: int
src = ctypes.c_void_p(tensor.data.data_ptr())
if qtype in [SYM_INT4, SYM_INT8, NF4, NF3, FP4]:
if qtype in [SYM_INT4, SYM_INT8, NF4, NF3, FP4, FP8]:
dst_tensor = torch.empty_like(tensor)
elif qtype == ggml_tensor_qtype["sym_int5"]:
QK = ggml.ggml_qk_size(ggml_tensor_qtype["asym_int5"])