ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

[NPU] Fix generate example (#12541)

Browse source
This commit is contained in:
binbin Deng 2024-12-13 14:07:24 +08:00 committed by GitHub
parent 15219944b8
commit d20a968ce2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
python/llm/example/NPU/HF-Transformers-AutoModels/LLM/generate.py
View file

@ -52,7 +52,6 @@ if __name__ == '__main__':
attn_implementation="eager" attn_implementation="eager"
) )
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
tokenizer.save_pretrained(args.lowbit_path)
else: else:
model = AutoModelForCausalLM.load_low_bit( model = AutoModelForCausalLM.load_low_bit(
args.lowbit_path, args.lowbit_path,
@ -66,6 +65,7 @@ if __name__ == '__main__':
if args.lowbit_path and not os.path.exists(args.lowbit_path): if args.lowbit_path and not os.path.exists(args.lowbit_path):
model.save_low_bit(args.lowbit_path) model.save_low_bit(args.lowbit_path)
tokenizer.save_pretrained(args.lowbit_path)
with torch.inference_mode(): with torch.inference_mode():
input_ids = tokenizer.encode(args.prompt, return_tensors="pt") input_ids = tokenizer.encode(args.prompt, return_tensors="pt")