ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

optimize lookahead init time (#11769)

Browse source
This commit is contained in:
Ruonan Wang 2024-08-12 12:19:12 +03:00 committed by GitHub
parent 05989ad0f9
commit 8db34057b4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 3 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
python/llm/example/GPU/HuggingFace/LLM/qwen2/generate.py
View file

@ -19,7 +19,6 @@ import time
import argparse
from transformers import AutoTokenizer
from ipex_llm import optimize_model
import numpy as np
@ -36,7 +35,7 @@ if __name__ == '__main__':
args = parser.parse_args()
model_path = args.repo_id_or_model_path
from ipex_llm.transformers import AutoModelForCausalLM
# Load model in 4 bit,
# which convert the relevant layers in the model into INT4 format
@ -45,7 +44,7 @@ if __name__ == '__main__':
optimize_model=True,
trust_remote_code=True,
use_cache=True)
model = model.to("xpu")
model = model.half().to("xpu")
# Load tokenizer
tokenizer = AutoTokenizer.from_pretrained(model_path,

2
python/llm/src/ipex_llm/transformers/lookup.py
View file

@ -149,7 +149,7 @@ class PromptLookupCandidateGenerator():
input_ids: torch.LongTensor):
for ngram_size in range(self.max_matching_ngram_size, 0, -1):
# Create sliding windows of size ngram_size
windows = input_ids.unfold(dimension=1, size=ngram_size, step=1)
windows = input_ids.cpu().unfold(dimension=1, size=ngram_size, step=1)
for idx in range(windows.size(1)):
window = tensor2key(windows[0, idx])
if window not in self.lookup_table: