ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Change igpu perf to mainly test int4+fp16 (#11513)

Browse source
This commit is contained in:
Yuwen Hu 2024-07-05 17:12:33 +08:00 committed by GitHub
parent 1efb6ebe93
commit 8f376e5192
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 230 additions and 237 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

430
.github/workflows/llm_performance_tests.yml vendored
View file

@ -629,15 +629,16 @@ jobs:
run_transformer_int4_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming) run_transformer_int4_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)
' python/llm/dev/benchmark/all-in-one/run.py ' python/llm/dev/benchmark/all-in-one/run.py
- name: Prepare igpu perf test (32-32) # 32-32 int4+fp16
- name: Prepare igpu perf test (32-32 int4+fp16)
shell: bash shell: bash
run: | run: |
# hide time info # hide time info
# sed -i 's/str(end - st)/"xxxxxx"/g' python/llm/dev/benchmark/all-in-one/run.py # sed -i 's/str(end - st)/"xxxxxx"/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i 's/{api}-results-{today}.csv/32-32-{api}-results-{today}_test1.csv/g' python/llm/dev/benchmark/all-in-one/run.py sed -i 's/{api}-results-{today}.csv/32-32-{api}-results-{today}_test1.csv/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32.yaml sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16.yaml
- name: Test on igpu (32-32) - name: Test on igpu (32-32 int4+fp16)
shell: cmd shell: cmd
run: | run: |
call conda activate igpu-perf call conda activate igpu-perf
@ -647,20 +648,20 @@ jobs:
set TRANSFORMERS_OFFLINE=1 set TRANSFORMERS_OFFLINE=1
cd python\llm\dev\benchmark\all-in-one cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\32-32.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate call conda deactivate
- name: Prepare igpu perf test for transformers 4.37 (32-32) - name: Prepare igpu perf test for transformers 4.37 (32-32 int4+fp16)
shell: bash shell: bash
run: | run: |
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_437.yaml sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_437.yaml
- name: Test on igpu for transformers 4.37 (32-32) - name: Test on igpu for transformers 4.37 (32-32 int4+fp16)
shell: cmd shell: cmd
run: | run: |
call conda activate igpu-perf call conda activate igpu-perf
@ -670,14 +671,14 @@ jobs:
set BIGDL_LLM_XMX_DISABLED=1 set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\32-32_437.yaml config.yaml move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_437.yaml config.yaml
set PYTHONIOENCODING=utf-8 set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32\log\%LOG_FILE% 2>&1 python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate call conda deactivate
- name: Concat csv and generate html (32-32) - name: Concat csv and generate html (32-32 int4+fp16)
shell: cmd shell: cmd
run: | run: |
call conda activate html-gen call conda activate html-gen
@ -686,20 +687,220 @@ jobs:
python ..\..\..\test\benchmark\concat_csv.py python ..\..\..\test\benchmark\concat_csv.py
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
del /q *test*.csv del /q *test*.csv
move *.csv %CSV_SAVE_PATH%\32-32\ move *.csv %CSV_SAVE_PATH%\32-32_int4_fp16\
cd ..\..\..\test\benchmark cd ..\..\..\test\benchmark
python csv_to_html.py -f %CSV_SAVE_PATH%\32-32\ python csv_to_html.py -f %CSV_SAVE_PATH%\32-32_int4_fp16\
if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
move %CSV_SAVE_PATH%\32-32\*.html %CSV_SAVE_PATH% move %CSV_SAVE_PATH%\32-32_int4_fp16\*.html %CSV_SAVE_PATH%
call conda deactivate call conda deactivate
# TODO: create a action function here for different input # TODO: create a action function here for different input
# 1024-128 int4+fp16
- name: Prepare igpu perf test (1024-128 int4+fp16)
shell: bash
run: |
sed -i 's/32-32/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml
- name: Test on igpu (1024-128 int4+fp16)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.36.2
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
REM for llava
set TRANSFORMERS_OFFLINE=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for transformers 4.37 (1024-128 int4+fp16)
shell: bash
run: |
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_437.yaml
- name: Test on igpu for transformers 4.37 (1024-128 int4+fp16)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.37.0
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_437.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Concat csv and generate html (1024-128 int4+fp16)
shell: cmd
run: |
call conda activate html-gen
cd python\llm\dev\benchmark\all-in-one
python ..\..\..\test\benchmark\concat_csv.py
if %ERRORLEVEL% neq 0 (exit /b 1)
del /q *test*.csv
move *.csv %CSV_SAVE_PATH%\1024-128_int4_fp16\
cd ..\..\..\test\benchmark
python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_int4_fp16\
if %ERRORLEVEL% neq 0 (exit /b 1)
move %CSV_SAVE_PATH%\1024-128_int4_fp16\*.html %CSV_SAVE_PATH%
call conda deactivate
# 2048-256 int4+fp16
- name: Prepare igpu perf test (2048-256 int4+fp16)
shell: bash
run: |
sed -i 's/1024-128/2048-256/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16.yaml
- name: Test on igpu (2048-256 int4+fp16)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.36.2
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
REM for llava
set TRANSFORMERS_OFFLINE=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for transformers 4.37 (2048-256 int4+fp16)
shell: bash
run: |
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_437.yaml
- name: Test on igpu for transformers 4.37 (2048-256 int4+fp16)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.37.0
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_437.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Concat csv and generate html (2048-256 int4+fp16)
shell: cmd
run: |
call conda activate html-gen
cd python\llm\dev\benchmark\all-in-one
python ..\..\..\test\benchmark\concat_csv.py
if %ERRORLEVEL% neq 0 (exit /b 1)
del /q *test*.csv
move *.csv %CSV_SAVE_PATH%\2048-256_int4_fp16\
cd ..\..\..\test\benchmark
python csv_to_html.py -f %CSV_SAVE_PATH%\2048-256_int4_fp16\
if %ERRORLEVEL% neq 0 (exit /b 1)
move %CSV_SAVE_PATH%\2048-256_int4_fp16\*.html %CSV_SAVE_PATH%
call conda deactivate
# load_low_bit 1024-128 int4+fp16
- name: Prepare igpu perf test (load_low_bit 1024-128 int4+fp16)
shell: bash
run: |
sed -i 's/2048-256/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit.yaml
- name: Test on igpu (load_low_bit 1024-128 int4+fp16)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.36.2
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
REM for llava
set TRANSFORMERS_OFFLINE=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for transformers 4.37 (load_low_bit 1024-128 int4+fp16)
shell: bash
run: |
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_437.yaml
- name: Test on igpu for transformers 4.37 (load_low_bit 1024-128 int4+fp16)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.37.0
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_437.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Concat csv and generate html (load_low_bit 1024-128 int4+fp16)
shell: cmd
run: |
call conda activate html-gen
cd python\llm\dev\benchmark\all-in-one
python ..\..\..\test\benchmark\concat_csv.py
if %ERRORLEVEL% neq 0 (exit /b 1)
del /q *test*.csv
move *.csv %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\
cd ..\..\..\test\benchmark
python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\
if %ERRORLEVEL% neq 0 (exit /b 1)
move %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\*.html %CSV_SAVE_PATH%
call conda deactivate
# 1024-128 # 1024-128
- name: Prepare igpu perf test (1024-128) - name: Prepare igpu perf test (1024-128)
shell: bash shell: bash
run: | run: |
sed -i 's/32-32/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128.yaml sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128.yaml
@ -762,205 +963,6 @@ jobs:
call conda deactivate call conda deactivate
# 2048-256
- name: Prepare igpu perf test (2048-256)
shell: bash
run: |
sed -i 's/1024-128/2048-256/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256.yaml
- name: Test on igpu (2048-256)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.36.2
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
REM for llava
set TRANSFORMERS_OFFLINE=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\2048-256.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for transformers 4.37 (2048-256)
shell: bash
run: |
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_437.yaml
- name: Test on igpu for transformers 4.37 (2048-256)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.37.0
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\2048-256_437.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Concat csv and generate html (2048-256)
shell: cmd
run: |
call conda activate html-gen
cd python\llm\dev\benchmark\all-in-one
python ..\..\..\test\benchmark\concat_csv.py
if %ERRORLEVEL% neq 0 (exit /b 1)
del /q *test*.csv
move *.csv %CSV_SAVE_PATH%\2048-256\
cd ..\..\..\test\benchmark
python csv_to_html.py -f %CSV_SAVE_PATH%\2048-256\
if %ERRORLEVEL% neq 0 (exit /b 1)
move %CSV_SAVE_PATH%\2048-256\*.html %CSV_SAVE_PATH%
call conda deactivate
# load_low_bit 1024-128
- name: Prepare igpu perf test (load_low_bit 1024-128)
shell: bash
run: |
sed -i 's/2048-256/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit.yaml
- name: Test on igpu (load_low_bit 1024-128)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.36.2
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
REM for llava
set TRANSFORMERS_OFFLINE=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for transformers 4.37 (load_low_bit 1024-128)
shell: bash
run: |
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_437.yaml
- name: Test on igpu for transformers 4.37 (load_low_bit 1024-128)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.37.0
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit_437.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Concat csv and generate html (load_low_bit 1024-128)
shell: cmd
run: |
call conda activate html-gen
cd python\llm\dev\benchmark\all-in-one
python ..\..\..\test\benchmark\concat_csv.py
if %ERRORLEVEL% neq 0 (exit /b 1)
del /q *test*.csv
move *.csv %CSV_SAVE_PATH%\1024-128_loadlowbit\
cd ..\..\..\test\benchmark
python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_loadlowbit\
if %ERRORLEVEL% neq 0 (exit /b 1)
move %CSV_SAVE_PATH%\1024-128_loadlowbit\*.html %CSV_SAVE_PATH%
call conda deactivate
- name: Prepare igpu perf test (int4+fp16 1024-128)
shell: bash
run: |
sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml
- name: Test on igpu (int4+fp16 1024-128)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.36.2
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
REM for llava
set TRANSFORMERS_OFFLINE=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for transformers 4.37 (int4+fp16 1024-128)
shell: bash
run: |
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_437.yaml
- name: Test on igpu for transformers 4.37 (int4+fp16 1024-128)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.37.0
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_437.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Concat csv and generate html (int4+fp16 1024-128)
shell: cmd
run: |
call conda activate html-gen
cd python\llm\dev\benchmark\all-in-one
python ..\..\..\test\benchmark\concat_csv.py
if %ERRORLEVEL% neq 0 (exit /b 1)
del /q *test*.csv
move *.csv %CSV_SAVE_PATH%\1024-128_int4_fp16\
cd ..\..\..\test\benchmark
python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_int4_fp16\
if %ERRORLEVEL% neq 0 (exit /b 1)
move %CSV_SAVE_PATH%\1024-128_int4_fp16\*.html %CSV_SAVE_PATH%
call conda deactivate
# TODO: avoid duplicated code # TODO: avoid duplicated code
- name: Upload results to ftp - name: Upload results to ftp
if: ${{ always() }} if: ${{ always() }}

3
python/llm/test/benchmark/igpu-perf/1024-128.yaml
View file

@ -3,10 +3,9 @@ repo_id:
- 'THUDM/glm-4-9b-chat' - 'THUDM/glm-4-9b-chat'
- 'baichuan-inc/Baichuan2-7B-Chat' - 'baichuan-inc/Baichuan2-7B-Chat'
- 'baichuan-inc/Baichuan2-13B-Chat' - 'baichuan-inc/Baichuan2-13B-Chat'
# - '01-ai/Yi-6B'
- 'meta-llama/Llama-2-7b-chat-hf' - 'meta-llama/Llama-2-7b-chat-hf'
- 'meta-llama/Llama-2-13b-chat-hf' - 'meta-llama/Llama-2-13b-chat-hf'
# - 'liuhaotian/llava-v1.5-7b' # Cannot load using AutoModelForCausalLM in 4.36+ - 'meta-llama/Meta-Llama-3-8B-Instruct'
local_model_hub: 'path to your local model hub' local_model_hub: 'path to your local model hub'
warm_up: 1 warm_up: 1
num_trials: 3 num_trials: 3

1
python/llm/test/benchmark/igpu-perf/1024-128_437.yaml
View file

@ -1,7 +1,6 @@
repo_id: repo_id:
- 'Qwen/Qwen1.5-7B-Chat' - 'Qwen/Qwen1.5-7B-Chat'
- 'Qwen/Qwen2-7B-Instruct' - 'Qwen/Qwen2-7B-Instruct'
- 'meta-llama/Meta-Llama-3-8B-Instruct'
- 'microsoft/Phi-3-mini-4k-instruct' - 'microsoft/Phi-3-mini-4k-instruct'
local_model_hub: 'path to your local model hub' local_model_hub: 'path to your local model hub'
warm_up: 1 warm_up: 1

6
python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml
View file

@ -1,13 +1,11 @@
repo_id: repo_id:
- 'THUDM/chatglm3-6b' - 'THUDM/chatglm3-6b'
- 'THUDM/glm-4-9b-chat'
- 'baichuan-inc/Baichuan2-7B-Chat' - 'baichuan-inc/Baichuan2-7B-Chat'
- 'baichuan-inc/Baichuan2-13B-Chat' - 'baichuan-inc/Baichuan2-13B-Chat'
# - '01-ai/Yi-6B'
- 'meta-llama/Llama-2-7b-chat-hf' - 'meta-llama/Llama-2-7b-chat-hf'
- 'meta-llama/Llama-2-13b-chat-hf' - 'meta-llama/Llama-2-13b-chat-hf'
# - 'liuhaotian/llava-v1.5-7b' # Cannot load using AutoModelForCausalLM in 4.36+ - 'meta-llama/Meta-Llama-3-8B-Instruct'
# - 'RWKV/rwkv-4-world-7b'
# - 'RWKV/rwkv-5-world-7b'
local_model_hub: 'path to your local model hub' local_model_hub: 'path to your local model hub'
warm_up: 1 warm_up: 1
num_trials: 3 num_trials: 3

3
python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_437.yaml
View file

@ -1,6 +1,7 @@
repo_id: repo_id:
- 'Qwen/Qwen1.5-7B-Chat' - 'Qwen/Qwen1.5-7B-Chat'
- 'meta-llama/Meta-Llama-3-8B-Instruct' - 'Qwen/Qwen2-7B-Instruct'
- 'microsoft/Phi-3-mini-4k-instruct'
local_model_hub: 'path to your local model hub' local_model_hub: 'path to your local model hub'
warm_up: 1 warm_up: 1
num_trials: 3 num_trials: 3

5
python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit.yaml → python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit.yaml
View file

@ -3,10 +3,9 @@ repo_id:
- 'THUDM/glm-4-9b-chat' - 'THUDM/glm-4-9b-chat'
- 'baichuan-inc/Baichuan2-7B-Chat' - 'baichuan-inc/Baichuan2-7B-Chat'
- 'baichuan-inc/Baichuan2-13B-Chat' - 'baichuan-inc/Baichuan2-13B-Chat'
# - '01-ai/Yi-6B'
- 'meta-llama/Llama-2-7b-chat-hf' - 'meta-llama/Llama-2-7b-chat-hf'
- 'meta-llama/Llama-2-13b-chat-hf' - 'meta-llama/Llama-2-13b-chat-hf'
# - 'liuhaotian/llava-v1.5-7b' # Cannot load using AutoModelForCausalLM in 4.36+ - 'meta-llama/Meta-Llama-3-8B-Instruct'
local_model_hub: 'path to your local model hub' local_model_hub: 'path to your local model hub'
warm_up: 1 warm_up: 1
num_trials: 3 num_trials: 3
@ -16,5 +15,5 @@ batch_size: 1 # default to 1
in_out_pairs: in_out_pairs:
- '1024-128' - '1024-128'
test_api: test_api:
- "transformer_int4_loadlowbit_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) - "transformer_int4_fp16_loadlowbit_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)

3
python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_437.yaml → python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_437.yaml
View file

@ -1,7 +1,6 @@
repo_id: repo_id:
- 'Qwen/Qwen1.5-7B-Chat' - 'Qwen/Qwen1.5-7B-Chat'
- 'Qwen/Qwen2-7B-Instruct' - 'Qwen/Qwen2-7B-Instruct'
- 'meta-llama/Meta-Llama-3-8B-Instruct'
- 'microsoft/Phi-3-mini-4k-instruct' - 'microsoft/Phi-3-mini-4k-instruct'
local_model_hub: 'path to your local model hub' local_model_hub: 'path to your local model hub'
warm_up: 1 warm_up: 1
@ -12,5 +11,5 @@ batch_size: 1 # default to 1
in_out_pairs: in_out_pairs:
- '1024-128' - '1024-128'
test_api: test_api:
- "transformer_int4_loadlowbit_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) - "transformer_int4_fp16_loadlowbit_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)

5
python/llm/test/benchmark/igpu-perf/2048-256.yaml → python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16.yaml
View file

@ -3,10 +3,9 @@ repo_id:
- 'THUDM/glm-4-9b-chat' - 'THUDM/glm-4-9b-chat'
- 'baichuan-inc/Baichuan2-7B-Chat' - 'baichuan-inc/Baichuan2-7B-Chat'
- 'baichuan-inc/Baichuan2-13B-Chat' - 'baichuan-inc/Baichuan2-13B-Chat'
# - '01-ai/Yi-6B'
- 'meta-llama/Llama-2-7b-chat-hf' - 'meta-llama/Llama-2-7b-chat-hf'
- 'meta-llama/Llama-2-13b-chat-hf' - 'meta-llama/Llama-2-13b-chat-hf'
# - 'liuhaotian/llava-v1.5-7b' # Cannot load using AutoModelForCausalLM in 4.36+ - 'meta-llama/Meta-Llama-3-8B-Instruct'
local_model_hub: 'path to your local model hub' local_model_hub: 'path to your local model hub'
warm_up: 1 warm_up: 1
num_trials: 3 num_trials: 3
@ -16,5 +15,5 @@ batch_size: 1 # default to 1
in_out_pairs: in_out_pairs:
- '2048-256' - '2048-256'
test_api: test_api:
- "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)

3
python/llm/test/benchmark/igpu-perf/2048-256_437.yaml → python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_437.yaml
View file

@ -1,7 +1,6 @@
repo_id: repo_id:
- 'Qwen/Qwen1.5-7B-Chat' - 'Qwen/Qwen1.5-7B-Chat'
- 'Qwen/Qwen2-7B-Instruct' - 'Qwen/Qwen2-7B-Instruct'
- 'meta-llama/Meta-Llama-3-8B-Instruct'
- 'microsoft/Phi-3-mini-4k-instruct' - 'microsoft/Phi-3-mini-4k-instruct'
local_model_hub: 'path to your local model hub' local_model_hub: 'path to your local model hub'
warm_up: 1 warm_up: 1
@ -12,5 +11,5 @@ batch_size: 1 # default to 1
in_out_pairs: in_out_pairs:
- '2048-256' - '2048-256'
test_api: test_api:
- "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)

5
python/llm/test/benchmark/igpu-perf/32-32.yaml → python/llm/test/benchmark/igpu-perf/32-32_int4_fp16.yaml
View file

@ -3,10 +3,9 @@ repo_id:
- 'THUDM/glm-4-9b-chat' - 'THUDM/glm-4-9b-chat'
- 'baichuan-inc/Baichuan2-7B-Chat' - 'baichuan-inc/Baichuan2-7B-Chat'
- 'baichuan-inc/Baichuan2-13B-Chat' - 'baichuan-inc/Baichuan2-13B-Chat'
# - '01-ai/Yi-6B'
- 'meta-llama/Llama-2-7b-chat-hf' - 'meta-llama/Llama-2-7b-chat-hf'
- 'meta-llama/Llama-2-13b-chat-hf' - 'meta-llama/Llama-2-13b-chat-hf'
# - 'liuhaotian/llava-v1.5-7b' # Cannot load using AutoModelForCausalLM in 4.36+ - 'meta-llama/Meta-Llama-3-8B-Instruct'
local_model_hub: 'path to your local model hub' local_model_hub: 'path to your local model hub'
warm_up: 3 warm_up: 3
num_trials: 5 num_trials: 5
@ -16,5 +15,5 @@ batch_size: 1 # default to 1
in_out_pairs: in_out_pairs:
- '32-32' - '32-32'
test_api: test_api:
- "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)

3
python/llm/test/benchmark/igpu-perf/32-32_437.yaml → python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_437.yaml
View file

@ -1,7 +1,6 @@
repo_id: repo_id:
- 'Qwen/Qwen1.5-7B-Chat' - 'Qwen/Qwen1.5-7B-Chat'
- 'Qwen/Qwen2-7B-Instruct' - 'Qwen/Qwen2-7B-Instruct'
- 'meta-llama/Meta-Llama-3-8B-Instruct'
- 'microsoft/Phi-3-mini-4k-instruct' - 'microsoft/Phi-3-mini-4k-instruct'
local_model_hub: 'path to your local model hub' local_model_hub: 'path to your local model hub'
warm_up: 3 warm_up: 3
@ -12,5 +11,5 @@ batch_size: 1 # default to 1
in_out_pairs: in_out_pairs:
- '32-32' - '32-32'
test_api: test_api:
- "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)