ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Change run.py in benchmark to support phi-3-vision in arc-perf (#11638)

Browse source 
Co-authored-by: ATMxsp01 <shou.xu@intel.com>
This commit is contained in:
Xu, Shuo 2024-07-23 09:51:36 +08:00 committed by GitHub
parent 060792a648
commit 7f80db95eb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 7 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
python/llm/dev/benchmark/all-in-one/run.py
View file

@ -480,6 +480,13 @@ def run_transformer_int4_gpu(repo_id,
use_cache=True, cpu_embedding=cpu_embedding, use_cache=True, cpu_embedding=cpu_embedding,
torch_dtype=torch_dtype).eval() torch_dtype=torch_dtype).eval()
tokenizer = LlamaTokenizer.from_pretrained(model_path, trust_remote_code=True) tokenizer = LlamaTokenizer.from_pretrained(model_path, trust_remote_code=True)
elif origin_repo_id in PHI3VISION_IDS:
model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_low_bit=low_bit,
_attn_implementation="eager",
modules_to_not_convert=["vision_embed_tokens"],
trust_remote_code=True, use_cache=True,
cpu_embedding=cpu_embedding, torch_dtype=torch_dtype).eval()
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
else: else:
if "4bit" in repo_id: if "4bit" in repo_id:
model = AutoModelForCausalLM.load_low_bit(model_path, optimize_model=True, model = AutoModelForCausalLM.load_low_bit(model_path, optimize_model=True,