ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

LLM: fix mpt load_low_bit issue (#10075)

Browse source 
* fix

* retry

* retry
This commit is contained in:
Jin Qiao 2024-02-05 10:17:07 +08:00 committed by GitHub
parent f8dcaff7f4
commit ad050107b3
1 changed files with 5 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
python/llm/src/bigdl/llm/transformers/model.py
View file

@ -361,6 +361,10 @@ class _BaseAutoModelClass:
cpu_embedding=cpu_embedding, lightweight_bmm=lightweight_bmm, cpu_embedding=cpu_embedding, lightweight_bmm=lightweight_bmm,
torch_dtype=kwargs.get("torch_dtype", 'auto')) torch_dtype=kwargs.get("torch_dtype", 'auto'))
model.config.update({"bigdl_transformers_low_bit": q_k}) model.config.update({"bigdl_transformers_low_bit": q_k})
# enable tie_word_embeddings for MPT
# refer to https://huggingface.co/mosaicml/mpt-7b-chat/blob/main/modeling_mpt.py#L232
if model.config.architectures[0] != 'MPTForCausalLM':
model.config.update({"tie_word_embeddings": False}) model.config.update({"tie_word_embeddings": False})
# add save_low_bit to pretrained model dynamically # add save_low_bit to pretrained model dynamically