ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Further update windows gpu perf test regarding results integrity check (#12232)

Browse source
This commit is contained in:
Yuwen Hu 2024-10-18 18:15:13 +08:00 committed by GitHub
parent ef659629f3
commit 5935b25622
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 36 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
.github/workflows/llm_performance_tests.yml vendored
View file

@ -783,7 +783,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -814,7 +814,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_436.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_436.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -845,7 +845,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_438.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_438.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1) REM if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -882,7 +882,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_443.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_443.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1) REM if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -916,7 +916,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_445.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_445.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1) REM if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -971,7 +971,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1002,7 +1002,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_436.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_436.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1033,7 +1033,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_438.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_438.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1070,7 +1070,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_443.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_443.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1104,7 +1104,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_445.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_445.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1158,7 +1158,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1189,7 +1189,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_436.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_436.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1220,7 +1220,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_438.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_438.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1257,7 +1257,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_443.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_443.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1291,7 +1291,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_445.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_445.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1345,7 +1345,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1379,7 +1379,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_436.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_436.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1415,7 +1415,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_438.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_438.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
if "${{ matrix.platform }}"=="perf-igpu" ( if "${{ matrix.platform }}"=="perf-igpu" (
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
) )
@ -1462,7 +1462,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_443.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_443.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
if "${{ matrix.platform }}"=="perf-igpu" ( if "${{ matrix.platform }}"=="perf-igpu" (
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
) )
@ -1506,7 +1506,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_445.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_445.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
if "${{ matrix.platform }}"=="perf-igpu" ( if "${{ matrix.platform }}"=="perf-igpu" (
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
) )
@ -1570,7 +1570,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1601,7 +1601,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_438.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_438.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1637,7 +1637,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_443.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_443.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1671,7 +1671,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_445.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_445.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1727,7 +1727,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1760,7 +1760,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_436.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_436.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1793,7 +1793,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_438.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_438.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1827,7 +1827,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_443.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_443.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1863,7 +1863,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_445.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_445.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1919,7 +1919,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1952,7 +1952,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_436.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128_436.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -1985,7 +1985,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_438.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128_438.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -2019,7 +2019,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_443.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128_443.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
@ -2055,7 +2055,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_445.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\1024-128_445.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)

2
python/llm/test/benchmark/igpu-perf/check_csv_results.py
View file

@ -65,6 +65,8 @@ def main():
if test_case not in actual_test_cases and test_case not in exclude_test_cases: if test_case not in actual_test_cases and test_case not in exclude_test_cases:
print(test_case) print(test_case)
raise ValueError("The above tests failed. Please check the errors in the log.") raise ValueError("The above tests failed. Please check the errors in the log.")
else:
print("Results integrity check completed")
if __name__ == "__main__": if __name__ == "__main__":
sys.exit(main()) sys.exit(main())