ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

set woq_int4 as default int4 (#13021)

Browse source
This commit is contained in:
Yishuo Wang 2025-04-14 14:10:59 +08:00 committed by GitHub
parent 6693e8ab04
commit 10c30cdba9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 9 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
python/llm/src/ipex_llm/transformers/convert.py
View file

@ -1089,6 +1089,15 @@ def ggml_convert_low_bit(model, qtype, optimize_model=True,
embedding_qtype=None,
mixed_precision=False,
disable_optimize_pre=False):
if qtype == ggml_tensor_qtype["sym_int4"] and torch.__version__ >= "2.6":
logger.warning("sym_int4 is deprecated, use woq_int4 instead, "
"if you are loading saved sym_int4 low bit model, "
"please resaved it with woq_int4")
qtype = ggml_tensor_qtype["woq_int4"]
elif qtype == ggml_tensor_qtype["woq_int4"] and torch.__version__ < "2.6":
logger.warning("woq_int4 is not supported with pytorch<2.6, "
"use sym_int4 instead or use ipex-llm with pytorch>=2.6")
qtype = ggml_tensor_qtype["sym_int4"]
if qtype in ggml_tensor_qtype.values():
index = list(ggml_tensor_qtype.values()).index(qtype)
logger.info(f"Converting the current model to "