ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

In harness-evaluation workflow, add statistical tables (#10118)

Browse source 
* chnage storage

* fix typo

* change label

* change label to arc03

* change needs in the last step

* add generate csv in harness/make_table_results.py

* modify needs in the last job

* add csv to html

* mfix path issue in llm-harness-summary-nightly

* modify output_path

* modify args in make_table_results.py

* modify make table command in summary

* change pr env label

* remove irrelevant code in summary; add set output path step; add limit in harness run

* re-organize code structure

* modify limit in run harness

* modify csv_to_html input path

* modify needs in summary-nightly
This commit is contained in:
yb-peng 2024-02-08 19:01:05 +08:00 committed by GitHub
parent c2378a9546
commit b4dc33def6
4 changed files with 451 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

81
.github/workflows/llm-harness-evaluation.yml vendored
View file

@ -65,10 +65,11 @@ jobs:
- name: set-pr-env
if: ${{github.event_name == 'pull_request'}}
env:
PR_MATRIX_MODEL_NAME: '["Llama2-7b-guanaco-dolphin-500"]'
PR_MATRIX_TASK: '["truthfulqa"]'
PR_MATRIX_PRECISION: '["sys_int4"]'
PR_LABELS: '["self-hosted", "llm", "temp-arc01"]'
PR_MATRIX_MODEL_NAME: '["Mistral-7B-v0.1"]'
PR_MATRIX_TASK: '["arc", "truthfulqa", "winogrande"]'
PR_MATRIX_PRECISION: '["fp8"]'
PR_LABELS: '["self-hosted", "llm", "accuracy-nightly"]'
run: |
echo "model_name=$PR_MATRIX_MODEL_NAME" >> $GITHUB_ENV
echo "precision=$PR_MATRIX_PRECISION" >> $GITHUB_ENV
@ -168,6 +169,7 @@ jobs:
env:
USE_XETLA: OFF
# SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS: 1
# TODO: limit just for debug, remove it later
run: |
export HF_HOME=${HARNESS_HF_HOME}
export HF_DATASETS=$HARNESS_HF_HOME/datasets
@ -180,7 +182,8 @@ jobs:
--precision ${{ matrix.precision }} \
--device ${{ matrix.device }} \
--tasks ${{ matrix.task }} \
--batch_size 1 --no_cache --output_path results
--batch_size 1 --no_cache --output_path results \
--limit 3
- uses: actions/upload-artifact@v3
@ -211,18 +214,70 @@ jobs:
run: |
pip install --upgrade pip
pip install jsonlines pytablewriter regex
DATE=$(date +%Y-%m-%d)
OUTPUT_PATH="results_$DATE"
echo "OUTPUT_PATH=$OUTPUT_PATH" >> $GITHUB_ENV
- name: Download all results
uses: actions/download-artifact@v3
with:
name: harness_results
path: ${{ env.OUTPUT_PATH }}
path: results
- name: Summarize the results
shell: bash
run: |
echo ${{ env.OUTPUT_PATH }}
ls ${{ env.OUTPUT_PATH }}
python ${{ github.workspace }}/python/llm/dev/benchmark/harness/make_table_results.py ${{ env.OUTPUT_PATH }}
ls results
python ${{ github.workspace }}/python/llm/dev/benchmark/harness/make_table_and_csv.py results
llm-harness-summary-nightly:
if: ${{github.event_name == 'schedule' || github.event_name == 'pull_request'}}
needs: [set-matrix, llm-harness-evalution]
runs-on: ["self-hosted", "llm", "accuracy1", "accuracy-nightly"]
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.9
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Install dependencies
shell: bash
run: |
pip install --upgrade pip
pip install jsonlines pytablewriter regex
- name: Set output path
shell: bash
run: |
DATE=$(date +%Y-%m-%d)
OUTPUT_PATH="results_$DATE"
echo "OUTPUT_PATH=$OUTPUT_PATH" >> $GITHUB_ENV
NIGHTLY_FOLDER="/home/arda/harness-action-runners/nightly-accuracy-data"
echo "NIGHTLY_FOLDER=$NIGHTLY_FOLDER" >> $GITHUB_ENV
PR_FOLDER="/home/arda/harness-action-runners/pr-accuracy-data"
echo "PR_FOLDER=$PR_FOLDER" >> $GITHUB_ENV
- name: Download all results for nightly run
if: github.event_name == 'schedule'
uses: actions/download-artifact@v3
with:
name: harness_results
path: ${{ env.NIGHTLY_FOLDER}}/${{ env.OUTPUT_PATH }}
- name: Download all results for pr run
if: github.event_name == 'pull_request'
uses: actions/download-artifact@v3
with:
name: harness_results
path: ${{ env.PR_FOLDER}}/${{ env.OUTPUT_PATH }}
- name: Summarize the results for nightly run
if: github.event_name == 'schedule'
shell: bash
run: |
ls /home/arda/harness-action-runners/nightly-accuracy-data/${{ env.OUTPUT_PATH }}
python ${{ github.workspace }}/python/llm/dev/benchmark/harness/make_table_and_csv.py ${{ env.NIGHTLY_FOLDER}}/${{ env.OUTPUT_PATH }} ${{ env.NIGHTLY_FOLDER}}
python ${{ github.workspace }}/python/llm/dev/benchmark/harness/harness_csv_to_html.py -f ${{ env.NIGHTLY_FOLDER}}
- name: Summarize the results for pull request
if: github.event_name == 'pull_request'
shell: bash
run: |
ls /home/arda/harness-action-runners/pr-accuracy-data/${{ env.OUTPUT_PATH }}
python ${{ github.workspace }}/python/llm/dev/benchmark/harness/make_table_and_csv.py ${{ env.PR_FOLDER}}/${{ env.OUTPUT_PATH }} ${{ env.PR_FOLDER}}
python ${{ github.workspace }}/python/llm/dev/benchmark/harness/harness_csv_to_html.py -f ${{ env.PR_FOLDER}}

204
python/llm/dev/benchmark/harness/harness_csv_to_html.py Normal file
View file

@ -0,0 +1,204 @@
#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Python program to convert CSV to HTML Table
import os
import sys
import argparse
import pandas as pd
def highlight_vals(val, max=3.0, color1='red', color2='green'):
if isinstance(val, float):
if val > max:
return 'background-color: %s' % color2
elif val <= -max:
return 'background-color: %s' % color1
else:
return ''
def nonzero_min(lst):
non_zero_lst = [num for num in lst if num > 0.0]
return min(non_zero_lst) if non_zero_lst else None
def is_diffs_within_normal_range(diff1, diff2, diff3, threshold=5.0):
return not any(diff < (-threshold) for diff in diff1 + diff2 + diff3 if isinstance(diff, float))
def add_to_dict(dict, key, value):
if key not in dict:
dict[key] = []
dict[key].append(value)
def best_in_dict(dict, key, value):
if key in dict:
best_value = nonzero_min(dict[key])
if best_value < value or value <= 0.0:
return best_value
return value
return value
def main():
parser = argparse.ArgumentParser(description="convert .csv file to .html file")
parser.add_argument("-f", "--folder_path", type=str, dest="folder_path",
help="The directory which stores the .csv file", default="/home/arda/yibo/BigDL/python/llm/dev/benchmark/harness")
parser.add_argument("-t", "--threshold", type=float, dest="threshold",
help="the threshold of highlight values", default=3.0)
parser.add_argument("-b", "--baseline_path", type=str, dest="baseline_path",
help="the baseline path which stores the baseline.csv file")
args = parser.parse_args()
csv_files = []
for file_name in os.listdir(args.folder_path):
file_path = os.path.join(args.folder_path, file_name)
if os.path.isfile(file_path) and file_name.endswith(".csv"):
csv_files.append(file_path)
csv_files.sort(reverse=True)
highlight_threshold=args.threshold
latest_csv = pd.read_csv(csv_files[0], index_col=0)
daily_html=csv_files[0].split(".")[0]+".html"
diffs_within_normal_range = True
if len(csv_files)>1:
if args.baseline_path:
previous_csv = pd.read_csv(args.baseline_path, index_col=0)
else:
previous_csv = pd.read_csv(csv_files[1], index_col=0)
last1=['']*len(latest_csv.index)
diff1=['']*len(latest_csv.index)
last2=['']*len(latest_csv.index)
diff2=['']*len(latest_csv.index)
last3=['']*len(latest_csv.index)
diff3=['']*len(latest_csv.index)
best_last1=['']*len(latest_csv.index)
best_diff1=['']*len(latest_csv.index)
best_last2=['']*len(latest_csv.index)
best_diff2=['']*len(latest_csv.index)
best_last3=['']*len(latest_csv.index)
best_diff3=['']*len(latest_csv.index)
Arc='Arc'
TruthfulQA='TruthfulQA'
Winogrande='Winogrande'
csv_dict = {}
for csv_file in csv_files:
current_csv = pd.read_csv(csv_file, index_col=0)
for current_csv_ind,current_csv_row in current_csv.iterrows():
current_csv_model=current_csv_row['Model'].strip()
current_csv_precision=current_csv_row['Precision'].strip()
current_csv_model_arc=current_csv_model+'-'+current_csv_precision+'-'+'Arc'
current_csv_model_truthfulqa=current_csv_model+'-'+current_csv_precision+'-'+'TruthfulQA'
current_csv_model_winogrande=current_csv_model+'-'+current_csv_precision+'-'+'Winogrande'
add_to_dict(csv_dict, current_csv_model_arc, current_csv_row[Arc])
add_to_dict(csv_dict, current_csv_model_truthfulqa, current_csv_row[TruthfulQA])
add_to_dict(csv_dict, current_csv_model_winogrande, current_csv_row[Winogrande])
for latest_csv_ind,latest_csv_row in latest_csv.iterrows():
latest_csv_model=latest_csv_row['Model'].strip()
latest_csv_precision=latest_csv_row['Precision'].strip()
latest_arc=latest_csv_row[Arc]
latest_truthfulqa=latest_csv_row[TruthfulQA]
latest_winogrande=latest_csv_row[Winogrande]
key1=latest_csv_model+'-'+latest_csv_precision+'-'+'Arc'
key2=latest_csv_model+'-'+latest_csv_precision+'-'+'TruthfulQA'
key3=latest_csv_model+'-'+latest_csv_precision+'-'+'Winogrande'
best_last1_value=best_in_dict(csv_dict, key1, latest_arc)
best_last2_value=best_in_dict(csv_dict, key2, latest_truthfulqa)
best_last3_value=best_in_dict(csv_dict, key3, latest_winogrande)
best_last1[latest_csv_ind]=best_last1_value
best_diff1[latest_csv_ind]=round((best_last1_value-latest_arc)*100/best_last1_value,2)
best_last2[latest_csv_ind]=best_last2_value
best_diff2[latest_csv_ind]=round((best_last2_value-latest_truthfulqa)*100/best_last2_value,2)
best_last3[latest_csv_ind]=best_last3_value
best_diff3[latest_csv_ind]=round((best_last3_value-latest_winogrande)*100/best_last3_value,2)
in_previous_flag=False
for previous_csv_ind,previous_csv_row in previous_csv.iterrows():
previous_csv_model=previous_csv_row['Model'].strip()
previous_csv_precision=previous_csv_row['Precision'].strip()
if latest_csv_model==previous_csv_model and latest_csv_precision==previous_csv_precision:
previous_arc=previous_csv_row[Arc]
previous_truthfulqa=previous_csv_row[TruthfulQA]
previous_winogrande=previous_csv_row[Winogrande]
if previous_arc > 0.0 and previous_truthfulqa > 0.0 and previous_winogrande > 0.0:
last1[latest_csv_ind]=previous_arc
diff1[latest_csv_ind]=round((previous_arc-latest_arc)*100/previous_arc,2)
last2[latest_csv_ind]=previous_truthfulqa
diff2[latest_csv_ind]=round((previous_truthfulqa-latest_truthfulqa)*100/previous_truthfulqa,2)
last3[latest_csv_ind]=previous_winogrande
diff3[latest_csv_ind]=round((previous_winogrande-latest_winogrande)*100/previous_winogrande,2)
in_previous_flag=True
if not in_previous_flag:
last1[latest_csv_ind]=pd.NA
diff1[latest_csv_ind]=pd.NA
last2[latest_csv_ind]=pd.NA
diff2[latest_csv_ind]=pd.NA
last3[latest_csv_ind]=pd.NA
diff3[latest_csv_ind]=pd.NA
latest_csv.insert(loc=5,column='last1',value=last1)
latest_csv.insert(loc=6,column='diff1(%)',value=diff1)
latest_csv.insert(loc=7,column='last2',value=last2)
latest_csv.insert(loc=8,column='diff2(%)',value=diff2)
latest_csv.insert(loc=9,column='last3',value=last3)
latest_csv.insert(loc=10,column='diff3(%)',value=diff3)
latest_csv.insert(loc=11,column='best 1',value=best_last1)
latest_csv.insert(loc=12,column='best diff1(%)',value=best_diff1)
latest_csv.insert(loc=13,column='best 2',value=best_last2)
latest_csv.insert(loc=14,column='best diff2(%)',value=best_diff2)
latest_csv.insert(loc=15,column='best 3',value=best_last3)
latest_csv.insert(loc=16,column='best diff3(%)',value=best_diff3)
diffs_within_normal_range = is_diffs_within_normal_range(diff1, diff2, diff3, threshold=highlight_threshold)
subset1=['diff1(%)','diff2(%)','diff3(%)' ]
subset2=['best diff1(%)','best diff2(%)','best diff3(%)']
columns={'Arc': '{:.2f}', 'TruthfulQA': '{:.2f}', 'Winogrande': '{:.2f}', 'last1': '{:.2f}', 'diff1(%)': '{:.2f}',
'last2': '{:.2f}', 'diff2(%)': '{:.2f}', 'last3': '{:.2f}', 'diff3(%)': '{:.2f}',
'best 1': '{:.2f}', 'best diff1(%)': '{:.2f}', 'best 2': '{:.2f}', 'best diff2(%)': '{:.2f}', 'best 3': '{:.2f}', 'best diff3(%)': '{:.2f}'}
styled_df = latest_csv.style.format(columns).applymap(lambda val: highlight_vals(val, max=3.0, color1='red', color2='green'), subset=subset1)
styled_df = styled_df.applymap(lambda val: highlight_vals(val, max=3.0, color1='yellow'), subset=subset2)
html_output = styled_df.set_table_attributes("border=1").render()
with open(daily_html, 'w') as f:
f.write(html_output)
else:
latest_csv.to_html(daily_html)
if args.baseline_path and not diffs_within_normal_range:
print("The diffs are outside the normal range: %" + str(highlight_threshold))
return 1
return 0
if __name__ == "__main__":
sys.exit(main())

141
python/llm/dev/benchmark/harness/make_table_and_csv.py Normal file
View file

@ -0,0 +1,141 @@
#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Usage:
python make_table_results.py <input_dir>
"""
import logging
from pytablewriter import MarkdownTableWriter, LatexTableWriter
import os
import json
import sys
import csv
import datetime
from harness_to_leaderboard import task_to_metric
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def make_table(result_dict):
"""Generate table of results."""
md_writer = MarkdownTableWriter()
latex_writer = LatexTableWriter()
md_writer.headers = ["Model", "Precision", "Arc", "Hellaswag", "MMLU", "TruthfulQA","Winogrande", "GSM8K"]
latex_writer.headers = ["Model", "Precision", "Arc", "Hellaswag", "MMLU", "TruthfulQA","Winogrande", "GSM8K"]
tasks = ["arc", "hellaswag", "mmlu", "truthfulqa", "winogrande", "gsm8k"]
values = []
for model, model_results in result_dict.items():
for precision, prec_results in model_results.items():
value = [model, precision]
for task in tasks:
task_results = prec_results.get(task, None)
if task_results is None:
value.append("")
else:
m = task_to_metric[task]
results = task_results["results"]
if len(results) > 1:
result = results[task]
else:
result = list(results.values())[0]
value.append("%.2f" % (result[m] * 100))
values.append(value)
model = ""
precision = ""
md_writer.value_matrix = values
latex_writer.value_matrix = values
# todo: make latex table look good
# print(latex_writer.dumps())
return md_writer.dumps()
def make_csv(result_dict, output_path=None):
current_date = datetime.datetime.now().strftime("%Y-%m-%d")
file_name = f'results_{current_date}.csv'
full_path = os.path.join(output_path, file_name) if output_path else file_name
print('Writing to', full_path)
file_name = full_path
headers = ["Index", "Model", "Precision", "Arc", "TruthfulQA", "Winogrande"]
with open(file_name, mode='w', newline='') as csv_file:
writer = csv.writer(csv_file)
writer.writerow(headers)
index = 0
for model, model_results in result_dict.items():
for precision, prec_results in model_results.items():
row = [index, model, precision]
for task in headers[3:]:
task_results = prec_results.get(task.lower(), None)
if task_results is None:
row.append("")
else:
m = task_to_metric[task.lower()]
results = task_results["results"]
result = list(results.values())[0] if len(results) == 1 else results[task.lower()]
row.append("%.2f" % (result[m] * 100))
writer.writerow(row)
index += 1
def merge_results(path):
# loop dirs and subdirs in results dir
# for each dir, load json files
print('Read from', path)
merged_results = dict()
for dirpath, dirnames, filenames in os.walk(sys.argv[1]):
# skip dirs without files
if not filenames:
continue
for filename in sorted([f for f in filenames if f.endswith("result.json")]):
path = os.path.join(dirpath, filename)
model, device, precision, task = dirpath.split('/')[-4:]
with open(path, "r") as f:
result_dict = json.load(f)
if model not in merged_results:
merged_results[model] = dict()
if precision not in merged_results[model]:
merged_results[model][precision] = dict()
merged_results[model][precision][task] = result_dict
return merged_results
def main(*args):
if len(args) > 1:
input_path = args[1]
else:
raise ValueError("Input path is required")
if len(args) > 2:
output_path = args[2] # use the third argument as the output path
else:
output_path = "./" # default to current directory
merged_results = merge_results(input_path)
make_csv(merged_results, output_path)
print(make_table(merged_results))
if __name__ == "__main__":
# when running from the harness, the first argument is the script name
# you must name the second argument and the third argument(optional) to be the input_dir and output_dir
main(*sys.argv)

41
python/llm/dev/benchmark/harness/make_table_results.py
View file

@ -23,6 +23,8 @@ from pytablewriter import MarkdownTableWriter, LatexTableWriter
import os
import json
import sys
import csv
import datetime
from harness_to_leaderboard import task_to_metric
@ -67,10 +69,39 @@ def make_table(result_dict):
return md_writer.dumps()
def make_csv(result_dict, output_path=None):
current_date = datetime.datetime.now().strftime("%Y-%m-%d")
file_name = f'results_{current_date}.csv'
full_path = os.path.join(output_path, file_name) if output_path else file_name
print('Writing to', full_path)
file_name = full_path
headers = ["Index", "Model", "Precision", "Arc", "TruthfulQA", "Winogrande"]
def merge_results(path):
with open(file_name, mode='w', newline='') as csv_file:
writer = csv.writer(csv_file)
writer.writerow(headers)
index = 0
for model, model_results in result_dict.items():
for precision, prec_results in model_results.items():
row = [index, model, precision]
for task in headers[3:]:
task_results = prec_results.get(task.lower(), None)
if task_results is None:
row.append("")
else:
m = task_to_metric[task.lower()]
results = task_results["results"]
result = list(results.values())[0] if len(results) == 1 else results[task.lower()]
row.append("%.2f" % (result[m] * 100))
writer.writerow(row)
index += 1
def merge_results(args):
# loop dirs and subdirs in results dir
# for each dir, load json files
path = args[1]
print('Read from', path)
merged_results = dict()
for dirpath, dirnames, filenames in os.walk(sys.argv[1]):
# skip dirs without files
@ -86,15 +117,17 @@ def merge_results(path):
if precision not in merged_results[model]:
merged_results[model][precision] = dict()
merged_results[model][precision][task] = result_dict
# args[2] is the output path
make_csv(merged_results, args[2])
return merged_results
def main(*args):
merged_results = merge_results(args[0])
merged_results = merge_results(args)
print(make_table(merged_results))
if __name__ == "__main__":
# when running from the harness, the first argument is the script name
# you must name the second argument and the third argument to be the input_dir and output_dir
main(*sys.argv)