ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

update run_transformer_int4_gpu (#8983)

Browse source 
* xpuperf

* update run.py

* clean upo

* uodate

* update

* meet code review
This commit is contained in:
Xin Qiu 2023-09-15 15:10:04 +08:00 committed by GitHub
parent aeef73a182
commit 64ee1d7689
1 changed files with 20 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
python/llm/dev/benchmark/all-in-one/run.py
View file

@ -58,7 +58,10 @@ def run_model(repo_id, test_api, in_out_pairs, local_model_hub=None, warm_up=1,
def get_model_path(repo_id, local_model_hub):
if local_model_hub:
repo_model_name = repo_id.split("/")[1]
return local_model_hub + os.path.sep + repo_model_name
local_model_path = local_model_hub + os.path.sep + repo_model_name
invalidInputError(os.path.isdir(local_model_path),
local_model_path + " not exists!, Please check your models' folder.")
return local_model_path
else:
return repo_id
@ -275,25 +278,20 @@ def run_transformer_int4_gpu(repo_id,
from bigdl.llm.transformers import AutoModel, AutoModelForCausalLM
from transformers import AutoTokenizer
import intel_extension_for_pytorch as ipex
if local_model_hub:
repo_model_name = repo_id.split("/")[1]
model_path = local_model_hub + "/" + repo_model_name
else:
model_path = repo_id
model_path = get_model_path(repo_id, local_model_hub)
# Load model in 4 bit,
# which convert the relevant layers in the model into INT4 format
st = time.perf_counter()
if repo_id in ['THUDM/chatglm-6b', 'THUDM/chatglm2-6b']:
model = AutoModel.from_pretrained(model_path, load_in_4bit=True, optimize_model=True, trust_remote_code=True)
model = model.to('xpu')
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
else:
model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_4bit=True)
model = model.to('xpu')
tokenizer = AutoTokenizer.from_pretrained(model_path)
model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_4bit=True, trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
end = time.perf_counter()
print(">> loading of model costs {}s".format(end - st))
model = model.to('xpu')
model = BenchmarkWrapper(model)
result = {}
@ -305,8 +303,10 @@ def run_transformer_int4_gpu(repo_id,
input_str = open(f"prompt/{in_len}.txt", 'r').read()
# As different tokenizer has different encodings,
# slice the input_ids to ensure the prompt length is required length.
input_ids = tokenizer.encode(input_str, return_tensors="pt").to('xpu')
input_ids = tokenizer.encode(input_str, return_tensors="pt")
input_ids = input_ids[:, :in_len]
true_str = tokenizer.batch_decode(input_ids)[0]
input_ids = tokenizer.encode(true_str, return_tensors="pt").to('xpu')
result[in_out] = []
for i in range(num_trials + warm_up):
st = time.perf_counter()
@ -319,6 +319,7 @@ def run_transformer_int4_gpu(repo_id,
print(output[0])
if i >= warm_up:
result[in_out].append([model.first_cost, model.rest_cost_mean, model.encoder_time])
torch.xpu.empty_cache()
return result
@ -330,27 +331,22 @@ def run_optimize_model_gpu(repo_id,
from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer
from bigdl.llm import optimize_model
import intel_extension_for_pytorch as ipex
if local_model_hub:
repo_model_name = repo_id.split("/")[1]
model_path = local_model_hub + "/" + repo_model_name
else:
model_path = repo_id
model_path = get_model_path(repo_id, local_model_hub)
# Load model in 4 bit,
# which convert the relevant layers in the model into INT4 format
st = time.perf_counter()
if repo_id in ['THUDM/chatglm-6b', 'THUDM/chatglm2-6b']:
model = AutoModel.from_pretrained(model_path, torch_dtype='auto', low_cpu_mem_usage=True, trust_remote_code=True)
model = optimize_model(model)
model = model.to('xpu')
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
else:
model = AutoModelForCausalLM.from_pretrained(model_path, torch_dtype='auto', low_cpu_mem_usage=True)
model = AutoModelForCausalLM.from_pretrained(model_path, torch_dtype='auto', low_cpu_mem_usage=True, trust_remote_code=True)
model = optimize_model(model)
model = model.to('xpu')
tokenizer = AutoTokenizer.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
end = time.perf_counter()
print(">> loading of model costs {}s".format(end - st))
model = model.to('xpu')
model = BenchmarkWrapper(model)
result = {}
@ -362,8 +358,10 @@ def run_optimize_model_gpu(repo_id,
input_str = open(f"prompt/{in_len}.txt", 'r').read()
# As different tokenizer has different encodings,
# slice the input_ids to ensure the prompt length is required length.
input_ids = tokenizer.encode(input_str, return_tensors="pt").to('xpu')
input_ids = tokenizer.encode(input_str, return_tensors="pt")
input_ids = input_ids[:, :in_len]
true_str = tokenizer.batch_decode(input_ids)[0]
input_ids = tokenizer.encode(true_str, return_tensors="pt").to('xpu')
result[in_out] = []
for i in range(num_trials + warm_up):
st = time.perf_counter()
@ -376,6 +374,7 @@ def run_optimize_model_gpu(repo_id,
print(output[0])
if i >= warm_up:
result[in_out].append([model.first_cost, model.rest_cost_mean, model.encoder_time])
torch.xpu.empty_cache()
return result