From 8643b62521ca4258108cb8fde4b02f0b8c8cbd5c Mon Sep 17 00:00:00 2001 From: Yuwen Hu <54161268+Oscilloscope98@users.noreply.github.com> Date: Tue, 16 Jan 2024 17:48:37 +0800 Subject: [PATCH] [LLM] Support longer context in iGPU perf tests (2048-256) (#9910) --- .github/workflows/llm_performance_tests.yml | 103 +++--------------- .../igpu-perf/{32-256.yaml => 2048-256.yaml} | 2 +- .../{32-256_434.yaml => 2048-256_434.yaml} | 2 +- .../llm/test/benchmark/igpu-perf/512-64.yaml | 24 ---- .../test/benchmark/igpu-perf/512-64_434.yaml | 12 -- 5 files changed, 19 insertions(+), 124 deletions(-) rename python/llm/test/benchmark/igpu-perf/{32-256.yaml => 2048-256.yaml} (98%) rename python/llm/test/benchmark/igpu-perf/{32-256_434.yaml => 2048-256_434.yaml} (96%) delete mode 100644 python/llm/test/benchmark/igpu-perf/512-64.yaml delete mode 100644 python/llm/test/benchmark/igpu-perf/512-64_434.yaml diff --git a/.github/workflows/llm_performance_tests.yml b/.github/workflows/llm_performance_tests.yml index c387f619..c908847c 100644 --- a/.github/workflows/llm_performance_tests.yml +++ b/.github/workflows/llm_performance_tests.yml @@ -423,16 +423,15 @@ jobs: call conda deactivate - - name: Prepare igpu perf test (32-256) + # 2048-256 + - name: Prepare igpu perf test (2048-256) shell: bash run: | - # hide time info - sed -i 's/1024-128/32-256/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i 's/1024-128/2048-256/g' python/llm/dev/benchmark/all-in-one/run.py sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-256.yaml + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256.yaml - # 32-256 - - name: Test on igpu (32-256) + - name: Test on igpu (2048-256) shell: cmd run: | call conda activate igpu-perf @@ -445,20 +444,20 @@ jobs: set TRANSFORMERS_OFFLINE=1 cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\32-256.yaml config.yaml + move ..\..\..\test\benchmark\igpu-perf\2048-256.yaml config.yaml set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\32-256\log\%LOG_FILE% 2>&1 + python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1 if %ERRORLEVEL% neq 0 (exit /b 1) call conda deactivate - - name: Prepare igpu perf test for Mistral (32-256) + - name: Prepare igpu perf test for Mistral (2048-256) shell: bash run: | sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-256_434.yaml + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_434.yaml - - name: Test on igpu for Mistral (32-256) + - name: Test on igpu for Mistral (2048-256) shell: cmd run: | call conda activate igpu-perf @@ -469,14 +468,14 @@ jobs: set BIGDL_LLM_XMX_DISABLED=1 cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\32-256_434.yaml config.yaml + move ..\..\..\test\benchmark\igpu-perf\2048-256_434.yaml config.yaml set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\32-256\log\%LOG_FILE% 2>&1 + python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1 if %ERRORLEVEL% neq 0 (exit /b 1) call conda deactivate - - name: Concat csv and generate html (32-256) + - name: Concat csv and generate html (2048-256) shell: cmd run: | call conda activate html-gen @@ -484,11 +483,11 @@ jobs: cd python\llm\dev\benchmark\all-in-one python ..\..\..\test\benchmark\concat_csv.py del /q *test*.csv - move *.csv %CSV_SAVE_PATH%\32-256\ + move *.csv %CSV_SAVE_PATH%\2048-256\ cd ..\..\..\test\benchmark - python csv_to_html.py -f %CSV_SAVE_PATH%\32-256\ + python csv_to_html.py -f %CSV_SAVE_PATH%\2048-256\ if %ERRORLEVEL% neq 0 (exit /b 1) - move %CSV_SAVE_PATH%\32-256\*.html %CSV_SAVE_PATH% + move %CSV_SAVE_PATH%\2048-256\*.html %CSV_SAVE_PATH% call conda deactivate @@ -497,7 +496,7 @@ jobs: shell: bash run: | # hide time info - sed -i 's/32-256/32-512/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i 's/2048-256/32-512/g' python/llm/dev/benchmark/all-in-one/run.py sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-512.yaml @@ -561,74 +560,6 @@ jobs: call conda deactivate - # 512-64 - - name: Prepare igpu perf test (512-64) - shell: bash - run: | - sed -i 's/32-512/512-64/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/512-64.yaml - - - name: Test on igpu (512-64) - shell: cmd - run: | - call conda activate igpu-perf - pip install transformers==4.31.0 - - call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" - set SYCL_CACHE_PERSISTENT=1 - set BIGDL_LLM_XMX_DISABLED=1 - REM for llava - set TRANSFORMERS_OFFLINE=1 - - cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\512-64.yaml config.yaml - set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\512-64\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) - - call conda deactivate - - - name: Prepare igpu perf test for Mistral (512-64) - shell: bash - run: | - sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/512-64_434.yaml - - - name: Test on igpu for Mistral (512-64) - shell: cmd - run: | - call conda activate igpu-perf - pip install transformers==4.34.0 - - call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" - set SYCL_CACHE_PERSISTENT=1 - set BIGDL_LLM_XMX_DISABLED=1 - - cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\512-64_434.yaml config.yaml - set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\512-64\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) - - call conda deactivate - - - name: Concat csv and generate html (512-64) - shell: cmd - run: | - call conda activate html-gen - - cd python\llm\dev\benchmark\all-in-one - python ..\..\..\test\benchmark\concat_csv.py - del /q *test*.csv - move *.csv %CSV_SAVE_PATH%\512-64\ - cd ..\..\..\test\benchmark - python csv_to_html.py -f %CSV_SAVE_PATH%\512-64\ - if %ERRORLEVEL% neq 0 (exit /b 1) - move %CSV_SAVE_PATH%\512-64\*.html %CSV_SAVE_PATH% - - call conda deactivate - - name: Upload results to ftp if: ${{ always() }} shell: cmd diff --git a/python/llm/test/benchmark/igpu-perf/32-256.yaml b/python/llm/test/benchmark/igpu-perf/2048-256.yaml similarity index 98% rename from python/llm/test/benchmark/igpu-perf/32-256.yaml rename to python/llm/test/benchmark/igpu-perf/2048-256.yaml index 1947f24e..43948212 100644 --- a/python/llm/test/benchmark/igpu-perf/32-256.yaml +++ b/python/llm/test/benchmark/igpu-perf/2048-256.yaml @@ -18,7 +18,7 @@ num_trials: 3 num_beams: 1 # default to greedy search low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) in_out_pairs: - - '32-256' + - '2048-256' test_api: - "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) diff --git a/python/llm/test/benchmark/igpu-perf/32-256_434.yaml b/python/llm/test/benchmark/igpu-perf/2048-256_434.yaml similarity index 96% rename from python/llm/test/benchmark/igpu-perf/32-256_434.yaml rename to python/llm/test/benchmark/igpu-perf/2048-256_434.yaml index b37cd9c7..693c48b0 100644 --- a/python/llm/test/benchmark/igpu-perf/32-256_434.yaml +++ b/python/llm/test/benchmark/igpu-perf/2048-256_434.yaml @@ -6,7 +6,7 @@ num_trials: 3 num_beams: 1 # default to greedy search low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) in_out_pairs: - - '32-256' + - '2048-256' test_api: - "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) diff --git a/python/llm/test/benchmark/igpu-perf/512-64.yaml b/python/llm/test/benchmark/igpu-perf/512-64.yaml deleted file mode 100644 index 58bed017..00000000 --- a/python/llm/test/benchmark/igpu-perf/512-64.yaml +++ /dev/null @@ -1,24 +0,0 @@ -repo_id: - - 'THUDM/chatglm2-6b' - - 'THUDM/chatglm3-6b' - - 'baichuan-inc/Baichuan2-7B-Chat' - - 'internlm/internlm-chat-7b-8k' - - 'Qwen/Qwen-7B-Chat' - - 'BAAI/AquilaChat2-7B' - - '01-ai/Yi-6B' - - 'meta-llama/Llama-2-7b-chat-hf' - - 'WisdomShell/CodeShell-7B-Chat' - - 'tiiuae/falcon-7b-instruct-with-patch' - - 'mosaicml/mpt-7b-chat' - - 'liuhaotian/llava-v1.5-7b' - - 'RWKV/rwkv-4-world-7b' -local_model_hub: 'path to your local model hub' -warm_up: 1 -num_trials: 3 -num_beams: 1 # default to greedy search -low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) -in_out_pairs: - - '512-64' -test_api: - - "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) -cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) diff --git a/python/llm/test/benchmark/igpu-perf/512-64_434.yaml b/python/llm/test/benchmark/igpu-perf/512-64_434.yaml deleted file mode 100644 index 41afe76b..00000000 --- a/python/llm/test/benchmark/igpu-perf/512-64_434.yaml +++ /dev/null @@ -1,12 +0,0 @@ -repo_id: - - 'mistralai/Mistral-7B-Instruct-v0.1' -local_model_hub: 'path to your local model hub' -warm_up: 1 -num_trials: 3 -num_beams: 1 # default to greedy search -low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) -in_out_pairs: - - '512-64' -test_api: - - "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) -cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)