ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

LLM: update multi gpu write csv in all-in-one benchmark. (#11538)

Browse source
This commit is contained in:
Cengguang Zhang 2024-07-09 11:14:17 +08:00 committed by GitHub
parent 69701b3ec8
commit fa81dbefd3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 16 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
python/llm/dev/benchmark/all-in-one/run.py
View file

@ -1980,11 +1980,20 @@ if __name__ == '__main__':
df = pd.DataFrame(results, columns=['model', '1st token avg latency (ms)', '2+ avg latency (ms/token)', 'encoder time (ms)',
'input/output tokens', 'batch_size', 'actual input/output tokens', 'num_beams', 'low_bit', 'cpu_embedding',
'model loading time (s)', 'peak mem (GB)', 'streaming', 'use_fp16_torch_dtype'])
df.index += max(line_counter-1, 0)
if api not in ["transformer_int4_gpu", "transformer_int4_fp16_gpu"]:
if line_counter == 0:
df.to_csv(csv_name, mode='a')
else:
df.to_csv(csv_name, mode='a', header=None)
line_counter += len(df.index)
if "pipeline" in api or "deepspeed" in api:
if torch.distributed.get_rank() == 0:
df.index += max(line_counter - 1, 0)
if line_counter == 0:
df.to_csv(csv_name, mode='a')
else:
df.to_csv(csv_name, mode='a', header=None)
line_counter += len(df.index)
else:
df.index += max(line_counter - 1, 0)
if api not in ["transformer_int4_gpu", "transformer_int4_fp16_gpu"]:
if line_counter == 0:
df.to_csv(csv_name, mode='a')
else:
df.to_csv(csv_name, mode='a', header=None)
line_counter += len(df.index)
results = []