From d45e577d8c472608beefe7e60ea210cca03d39f9 Mon Sep 17 00:00:00 2001 From: Yuwen Hu <54161268+Oscilloscope98@users.noreply.github.com> Date: Mon, 4 Mar 2024 18:03:57 +0800 Subject: [PATCH] [LLM] Test `load_low_bit` in iGPU perf test on Windows (#10313) --- .github/workflows/llm_performance_tests.yml | 43 +++++++++---------- .../{32-512.yaml => 1024-128_loadlowbit.yaml} | 4 +- ..._434.yaml => 1024-128_loadlowbit_434.yaml} | 4 +- ..._437.yaml => 1024-128_loadlowbit_437.yaml} | 4 +- 4 files changed, 27 insertions(+), 28 deletions(-) rename python/llm/test/benchmark/igpu-perf/{32-512.yaml => 1024-128_loadlowbit.yaml} (88%) rename python/llm/test/benchmark/igpu-perf/{32-512_434.yaml => 1024-128_loadlowbit_434.yaml} (78%) rename python/llm/test/benchmark/igpu-perf/{32-512_437.yaml => 1024-128_loadlowbit_437.yaml} (77%) diff --git a/.github/workflows/llm_performance_tests.yml b/.github/workflows/llm_performance_tests.yml index c914e43c..292bd52c 100644 --- a/.github/workflows/llm_performance_tests.yml +++ b/.github/workflows/llm_performance_tests.yml @@ -679,16 +679,15 @@ jobs: call conda deactivate - # 32-512 - - name: Prepare igpu perf test (32-512) + # load_low_bit 1024-128 + - name: Prepare igpu perf test (load_low_bit 1024-128) shell: bash run: | - # hide time info - sed -i 's/2048-256/32-512/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i 's/2048-256/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-512.yaml + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit.yaml - - name: Test on igpu (32-512) + - name: Test on igpu (load_low_bit 1024-128) shell: cmd run: | call conda activate igpu-perf @@ -701,20 +700,20 @@ jobs: set TRANSFORMERS_OFFLINE=1 cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\32-512.yaml config.yaml + move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit.yaml config.yaml set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\32-512\log\%LOG_FILE% 2>&1 + python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1 if %ERRORLEVEL% neq 0 (exit /b 1) call conda deactivate - - name: Prepare igpu perf test for Mistral (32-512) + - name: Prepare igpu perf test for Mistral (load_low_bit 1024-128) shell: bash run: | sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-512_434.yaml + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_434.yaml - - name: Test on igpu for Mistral (32-512) + - name: Test on igpu for Mistral (load_low_bit 1024-128) shell: cmd run: | call conda activate igpu-perf @@ -725,20 +724,20 @@ jobs: set BIGDL_LLM_XMX_DISABLED=1 cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\32-512_434.yaml config.yaml + move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit_434.yaml config.yaml set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\32-512\log\%LOG_FILE% 2>&1 + python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1 if %ERRORLEVEL% neq 0 (exit /b 1) call conda deactivate - - name: Prepare igpu perf test for Qwen 1.5 (32-512) + - name: Prepare igpu perf test for Qwen 1.5 (load_low_bit 1024-128) shell: bash run: | sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-512_437.yaml + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_437.yaml - - name: Test on igpu for Qwen 1.5 (32-512) + - name: Test on igpu for Qwen 1.5 (load_low_bit 1024-128) shell: cmd run: | call conda activate igpu-perf @@ -749,14 +748,14 @@ jobs: set BIGDL_LLM_XMX_DISABLED=1 cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\32-512_437.yaml config.yaml + move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit_437.yaml config.yaml set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\32-512\log\%LOG_FILE% 2>&1 + python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1 if %ERRORLEVEL% neq 0 (exit /b 1) call conda deactivate - - name: Concat csv and generate html (32-512) + - name: Concat csv and generate html (load_low_bit 1024-128) shell: cmd run: | call conda activate html-gen @@ -765,11 +764,11 @@ jobs: python ..\..\..\test\benchmark\concat_csv.py if %ERRORLEVEL% neq 0 (exit /b 1) del /q *test*.csv - move *.csv %CSV_SAVE_PATH%\32-512\ + move *.csv %CSV_SAVE_PATH%\1024-128_loadlowbit\ cd ..\..\..\test\benchmark - python csv_to_html.py -f %CSV_SAVE_PATH%\32-512\ + python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_loadlowbit\ if %ERRORLEVEL% neq 0 (exit /b 1) - move %CSV_SAVE_PATH%\32-512\*.html %CSV_SAVE_PATH% + move %CSV_SAVE_PATH%\1024-128_loadlowbit\*.html %CSV_SAVE_PATH% call conda deactivate diff --git a/python/llm/test/benchmark/igpu-perf/32-512.yaml b/python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit.yaml similarity index 88% rename from python/llm/test/benchmark/igpu-perf/32-512.yaml rename to python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit.yaml index d90b4082..ea869865 100644 --- a/python/llm/test/benchmark/igpu-perf/32-512.yaml +++ b/python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit.yaml @@ -23,7 +23,7 @@ num_beams: 1 # default to greedy search low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) batch_size: 1 # default to 1 in_out_pairs: - - '32-512' + - '1024-128' test_api: - - "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) + - "transformer_int4_loadlowbit_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) diff --git a/python/llm/test/benchmark/igpu-perf/32-512_434.yaml b/python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_434.yaml similarity index 78% rename from python/llm/test/benchmark/igpu-perf/32-512_434.yaml rename to python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_434.yaml index a9d6d62a..51453bd1 100644 --- a/python/llm/test/benchmark/igpu-perf/32-512_434.yaml +++ b/python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_434.yaml @@ -7,7 +7,7 @@ num_beams: 1 # default to greedy search low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) batch_size: 1 # default to 1 in_out_pairs: - - '32-512' + - '1024-128' test_api: - - "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) + - "transformer_int4_loadlowbit_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) diff --git a/python/llm/test/benchmark/igpu-perf/32-512_437.yaml b/python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_437.yaml similarity index 77% rename from python/llm/test/benchmark/igpu-perf/32-512_437.yaml rename to python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_437.yaml index 2650ffea..1252d1bb 100644 --- a/python/llm/test/benchmark/igpu-perf/32-512_437.yaml +++ b/python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_437.yaml @@ -7,7 +7,7 @@ num_beams: 1 # default to greedy search low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) batch_size: 1 # default to 1 in_out_pairs: - - '32-512' + - '1024-128' test_api: - - "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) + - "transformer_int4_loadlowbit_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)