ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Optimize transformer int4 memory footprint (#8579)

Browse source
This commit is contained in:
Yang Wang 2023-07-21 11:22:13 +08:00 committed by GitHub
parent 57e880f63a
commit feb3af0567
1 changed files with 1 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
python/llm/src/bigdl/llm/transformers/convert.py
View file

@ -61,6 +61,7 @@ def _replace_with_quant_linear(model, qtype, modules_to_not_convert=None,
name, name,
"cpu", "cpu",
torch.empty(*param.size(), dtype=torch.float32)) torch.empty(*param.size(), dtype=torch.float32))
del model_state_dict
for name, module in model.named_children(): for name, module in model.named_children():
if current_key_name is None: if current_key_name is None: