Update optimize_model=True in llama2 chatglm2 arc examples (#8878)

* add optimize_model=True in llama2 chatglm2 examples

* add ipex optimize in gpt-j example
This commit is contained in:
Yina Chen 2023-09-05 10:35:37 +08:00 committed by GitHub
parent 5e58f698cd
commit 74a2c2ddf5
4 changed files with 3 additions and 2 deletions

View file

@ -44,7 +44,7 @@ if __name__ == '__main__':
# which convert the relevant layers in the model into INT4 format # which convert the relevant layers in the model into INT4 format
model = AutoModel.from_pretrained(model_path, model = AutoModel.from_pretrained(model_path,
load_in_4bit=True, load_in_4bit=True,
optimize_model=False, optimize_model=True,
trust_remote_code=True) trust_remote_code=True)
model = model.to('xpu') model = model.to('xpu')

View file

@ -45,6 +45,7 @@ if __name__ == '__main__':
optimize_model=False, optimize_model=False,
trust_remote_code=True) trust_remote_code=True)
model = model.to('xpu') model = model.to('xpu')
model = ipex.optimize(model.eval(), dtype="float16", inplace=True)
# Load tokenizer # Load tokenizer
tokenizer = AutoTokenizer.from_pretrained(model_path, tokenizer = AutoTokenizer.from_pretrained(model_path,

View file

@ -57,7 +57,7 @@ if __name__ == '__main__':
# which convert the relevant layers in the model into INT4 format # which convert the relevant layers in the model into INT4 format
model = AutoModelForCausalLM.from_pretrained(model_path, model = AutoModelForCausalLM.from_pretrained(model_path,
load_in_4bit=True, load_in_4bit=True,
optimize_model=False, optimize_model=True,
trust_remote_code=True) trust_remote_code=True)
model = model.to('xpu') model = model.to('xpu')