ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

LLM: add use_cache=True for all gpu examples (#8971)

Browse source
This commit is contained in:
JinBridge 2023-09-15 09:54:38 +08:00 committed by GitHub
parent d1b62ef2f2
commit c12b8f24b6
11 changed files with 22 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
python/llm/example/gpu/hf-transformers-models/baichuan/generate.py
View file

@ -43,7 +43,8 @@ if __name__ == '__main__':
model = AutoModelForCausalLM.from_pretrained(model_path, model = AutoModelForCausalLM.from_pretrained(model_path,
load_in_4bit=True, load_in_4bit=True,
optimize_model=False, optimize_model=False,
trust_remote_code=True) trust_remote_code=True,
use_cache=True)
model = model.to('xpu') model = model.to('xpu')
# Load tokenizer # Load tokenizer

3
python/llm/example/gpu/hf-transformers-models/chatglm2/generate.py
View file

@ -45,7 +45,8 @@ if __name__ == '__main__':
model = AutoModel.from_pretrained(model_path, model = AutoModel.from_pretrained(model_path,
load_in_4bit=True, load_in_4bit=True,
optimize_model=True, optimize_model=True,
trust_remote_code=True) trust_remote_code=True,
use_cache=True)
model = model.to('xpu') model = model.to('xpu')
# Load tokenizer # Load tokenizer

3
python/llm/example/gpu/hf-transformers-models/falcon/generate.py
View file

@ -45,7 +45,8 @@ if __name__ == '__main__':
model = AutoModelForCausalLM.from_pretrained(model_path, model = AutoModelForCausalLM.from_pretrained(model_path,
load_in_4bit=True, load_in_4bit=True,
optimize_model=False, optimize_model=False,
trust_remote_code=True) trust_remote_code=True,
use_cache=True)
model = model.to('xpu') model = model.to('xpu')
# Load tokenizer # Load tokenizer

3
python/llm/example/gpu/hf-transformers-models/gpt-j/generate.py
View file

@ -43,7 +43,8 @@ if __name__ == '__main__':
model = AutoModelForCausalLM.from_pretrained(model_path, model = AutoModelForCausalLM.from_pretrained(model_path,
load_in_4bit=True, load_in_4bit=True,
optimize_model=False, optimize_model=False,
trust_remote_code=True) trust_remote_code=True,
use_cache=True)
model = model.to('xpu') model = model.to('xpu')
model = ipex.optimize(model.eval(), dtype="float16", inplace=True) model = ipex.optimize(model.eval(), dtype="float16", inplace=True)

3
python/llm/example/gpu/hf-transformers-models/internlm/generate.py
View file

@ -44,7 +44,8 @@ if __name__ == '__main__':
model = AutoModelForCausalLM.from_pretrained(model_path, model = AutoModelForCausalLM.from_pretrained(model_path,
load_in_4bit=True, load_in_4bit=True,
optimize_model=False, optimize_model=False,
trust_remote_code=True) trust_remote_code=True,
use_cache=True)
model = model.to('xpu') model = model.to('xpu')
# Load tokenizer # Load tokenizer

3
python/llm/example/gpu/hf-transformers-models/llama2/generate.py
View file

@ -58,7 +58,8 @@ if __name__ == '__main__':
model = AutoModelForCausalLM.from_pretrained(model_path, model = AutoModelForCausalLM.from_pretrained(model_path,
load_in_4bit=True, load_in_4bit=True,
optimize_model=True, optimize_model=True,
trust_remote_code=True) trust_remote_code=True,
use_cache=True)
model = model.to('xpu') model = model.to('xpu')
# Load tokenizer # Load tokenizer

3
python/llm/example/gpu/hf-transformers-models/mpt/generate.py
View file

@ -45,7 +45,8 @@ if __name__ == '__main__':
model = AutoModelForCausalLM.from_pretrained(model_path, model = AutoModelForCausalLM.from_pretrained(model_path,
load_in_4bit=True, load_in_4bit=True,
optimize_model=False, optimize_model=False,
trust_remote_code=True) trust_remote_code=True,
use_cache=True)
model = model.to('xpu') model = model.to('xpu')
# Load tokenizer # Load tokenizer

3
python/llm/example/gpu/hf-transformers-models/qwen/generate.py
View file

@ -43,7 +43,8 @@ if __name__ == '__main__':
model = AutoModelForCausalLM.from_pretrained(model_path, model = AutoModelForCausalLM.from_pretrained(model_path,
load_in_4bit=True, load_in_4bit=True,
optimize_model=False, optimize_model=False,
trust_remote_code=True) trust_remote_code=True,
use_cache=True)
model = model.to('xpu') model = model.to('xpu')
# Load tokenizer # Load tokenizer

3
python/llm/example/gpu/hf-transformers-models/starcoder/generate.py
View file

@ -43,7 +43,8 @@ if __name__ == '__main__':
model = AutoModelForCausalLM.from_pretrained(model_path, model = AutoModelForCausalLM.from_pretrained(model_path,
load_in_4bit=True, load_in_4bit=True,
optimize_model=False, optimize_model=False,
trust_remote_code=True) trust_remote_code=True,
use_cache=True)
model = model.to('xpu') model = model.to('xpu')
# Load tokenizer # Load tokenizer

4
python/llm/example/gpu/hf-transformers-models/voiceassistant/generate.py
View file

@ -89,11 +89,11 @@ if __name__ == '__main__':
processor = WhisperProcessor.from_pretrained(whisper_model_path) processor = WhisperProcessor.from_pretrained(whisper_model_path)
# generate token ids # generate token ids
whisper = AutoModelForSpeechSeq2Seq.from_pretrained(whisper_model_path, load_in_4bit=True, optimize_model=False) whisper = AutoModelForSpeechSeq2Seq.from_pretrained(whisper_model_path, load_in_4bit=True, optimize_model=False, use_cache=True)
whisper.config.forced_decoder_ids = None whisper.config.forced_decoder_ids = None
whisper = whisper.to('xpu') whisper = whisper.to('xpu')
llama_model = AutoModelForCausalLM.from_pretrained(llama_model_path, load_in_4bit=True, trust_remote_code=True, optimize_model=False) llama_model = AutoModelForCausalLM.from_pretrained(llama_model_path, load_in_4bit=True, trust_remote_code=True, optimize_model=False, use_cache=True)
llama_model = llama_model.to('xpu') llama_model = llama_model.to('xpu')
tokenizer = LlamaTokenizer.from_pretrained(llama_model_path) tokenizer = LlamaTokenizer.from_pretrained(llama_model_path)

3
python/llm/example/gpu/hf-transformers-models/whisper/recognize.py
View file

@ -45,7 +45,8 @@ if __name__ == '__main__':
# which convert the relevant layers in the model into INT4 format # which convert the relevant layers in the model into INT4 format
model = AutoModelForSpeechSeq2Seq.from_pretrained(model_path, model = AutoModelForSpeechSeq2Seq.from_pretrained(model_path,
load_in_4bit=True, load_in_4bit=True,
optimize_model=False) optimize_model=False,
use_cache=True)
model.to('xpu') model.to('xpu')
model.config.forced_decoder_ids = None model.config.forced_decoder_ids = None