[LLM] Support longer context in iGPU perf tests (2048-256) (#9910)

.github/workflows/llm_performance_tests.yml

@@ -423,16 +423,15 @@ jobs:
 
           call conda deactivate
 
-      - name: Prepare igpu perf test (32-256)
+      # 2048-256
+      - name: Prepare igpu perf test (2048-256)
         shell: bash
         run: |
-          # hide time info
+          sed -i 's/1024-128/2048-256/g' python/llm/dev/benchmark/all-in-one/run.py
-          sed -i 's/1024-128/32-256/g' python/llm/dev/benchmark/all-in-one/run.py
           sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
-          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-256.yaml
+          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256.yaml
 
-      # 32-256
+      - name: Test on igpu (2048-256)
-      - name: Test on igpu (32-256)
         shell: cmd
         run: |
           call conda activate igpu-perf
@@ -445,20 +444,20 @@ jobs:
           set TRANSFORMERS_OFFLINE=1
 
           cd python\llm\dev\benchmark\all-in-one
-          move ..\..\..\test\benchmark\igpu-perf\32-256.yaml config.yaml
+          move ..\..\..\test\benchmark\igpu-perf\2048-256.yaml config.yaml
           set PYTHONIOENCODING=utf-8
-          python run.py >> %CSV_SAVE_PATH%\32-256\log\%LOG_FILE% 2>&1
+          python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1
           if %ERRORLEVEL% neq 0 (exit /b 1)
 
           call conda deactivate
 
-      - name: Prepare igpu perf test for Mistral (32-256)
+      - name: Prepare igpu perf test for Mistral (2048-256)
         shell: bash
         run: |
           sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
-          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-256_434.yaml
+          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_434.yaml
 
-      - name: Test on igpu for Mistral (32-256)
+      - name: Test on igpu for Mistral (2048-256)
         shell: cmd
         run: |
           call conda activate igpu-perf
@@ -469,14 +468,14 @@ jobs:
           set BIGDL_LLM_XMX_DISABLED=1
 
           cd python\llm\dev\benchmark\all-in-one
-          move ..\..\..\test\benchmark\igpu-perf\32-256_434.yaml config.yaml
+          move ..\..\..\test\benchmark\igpu-perf\2048-256_434.yaml config.yaml
           set PYTHONIOENCODING=utf-8
-          python run.py >> %CSV_SAVE_PATH%\32-256\log\%LOG_FILE% 2>&1
+          python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1
           if %ERRORLEVEL% neq 0 (exit /b 1)
 
           call conda deactivate
 
-      - name: Concat csv and generate html (32-256)
+      - name: Concat csv and generate html (2048-256)
         shell: cmd
         run: |
           call conda activate html-gen
@@ -484,11 +483,11 @@ jobs:
           cd python\llm\dev\benchmark\all-in-one
           python ..\..\..\test\benchmark\concat_csv.py
           del /q *test*.csv
-          move *.csv %CSV_SAVE_PATH%\32-256\
+          move *.csv %CSV_SAVE_PATH%\2048-256\
           cd ..\..\..\test\benchmark
-          python csv_to_html.py -f %CSV_SAVE_PATH%\32-256\
+          python csv_to_html.py -f %CSV_SAVE_PATH%\2048-256\
           if %ERRORLEVEL% neq 0 (exit /b 1)
-          move %CSV_SAVE_PATH%\32-256\*.html %CSV_SAVE_PATH%
+          move %CSV_SAVE_PATH%\2048-256\*.html %CSV_SAVE_PATH%
 
           call conda deactivate
 
@@ -497,7 +496,7 @@ jobs:
         shell: bash
         run: |
           # hide time info
-          sed -i 's/32-256/32-512/g' python/llm/dev/benchmark/all-in-one/run.py
+          sed -i 's/2048-256/32-512/g' python/llm/dev/benchmark/all-in-one/run.py
           sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
           sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-512.yaml
 
@@ -561,74 +560,6 @@ jobs:
 
           call conda deactivate
 
-      # 512-64
-      - name: Prepare igpu perf test (512-64)
-        shell: bash
-        run: |
-          sed -i 's/32-512/512-64/g' python/llm/dev/benchmark/all-in-one/run.py
-          sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
-          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/512-64.yaml
-
-      - name: Test on igpu (512-64)
-        shell: cmd
-        run: |
-          call conda activate igpu-perf
-          pip install transformers==4.31.0
-
-          call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
-          set SYCL_CACHE_PERSISTENT=1
-          set BIGDL_LLM_XMX_DISABLED=1
-          REM for llava
-          set TRANSFORMERS_OFFLINE=1
-
-          cd python\llm\dev\benchmark\all-in-one
-          move ..\..\..\test\benchmark\igpu-perf\512-64.yaml config.yaml
-          set PYTHONIOENCODING=utf-8
-          python run.py >> %CSV_SAVE_PATH%\512-64\log\%LOG_FILE% 2>&1
-          if %ERRORLEVEL% neq 0 (exit /b 1)
-
-          call conda deactivate
-
-      - name: Prepare igpu perf test for Mistral (512-64)
-        shell: bash
-        run: |
-          sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
-          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/512-64_434.yaml
-
-      - name: Test on igpu for Mistral (512-64)
-        shell: cmd
-        run: |
-          call conda activate igpu-perf
-          pip install transformers==4.34.0
-
-          call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
-          set SYCL_CACHE_PERSISTENT=1
-          set BIGDL_LLM_XMX_DISABLED=1
-
-          cd python\llm\dev\benchmark\all-in-one
-          move ..\..\..\test\benchmark\igpu-perf\512-64_434.yaml config.yaml
-          set PYTHONIOENCODING=utf-8
-          python run.py >> %CSV_SAVE_PATH%\512-64\log\%LOG_FILE% 2>&1
-          if %ERRORLEVEL% neq 0 (exit /b 1)
-
-          call conda deactivate
-
-      - name: Concat csv and generate html (512-64)
-        shell: cmd
-        run: |
-          call conda activate html-gen
-
-          cd python\llm\dev\benchmark\all-in-one
-          python ..\..\..\test\benchmark\concat_csv.py
-          del /q *test*.csv
-          move *.csv %CSV_SAVE_PATH%\512-64\
-          cd ..\..\..\test\benchmark
-          python csv_to_html.py -f %CSV_SAVE_PATH%\512-64\
-          if %ERRORLEVEL% neq 0 (exit /b 1)
-          move %CSV_SAVE_PATH%\512-64\*.html %CSV_SAVE_PATH%
-
-          call conda deactivate
-
       - name: Upload results to ftp
         if: ${{ always() }}
         shell: cmd

python/llm/test/benchmark/igpu-perf/2048-256.yaml

@@ -18,7 +18,7 @@ num_trials: 3
 num_beams: 1 # default to greedy search
 low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
 in_out_pairs:
-  - '32-256'
+  - '2048-256'
 test_api:
   - "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
 cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)

python/llm/test/benchmark/igpu-perf/2048-256_434.yaml

@@ -6,7 +6,7 @@ num_trials: 3
 num_beams: 1 # default to greedy search
 low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
 in_out_pairs:
-  - '32-256'
+  - '2048-256'
 test_api:
   - "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
 cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)

python/llm/test/benchmark/igpu-perf/512-64.yaml

@@ -1,24 +0,0 @@
-repo_id:
-  - 'THUDM/chatglm2-6b'
-  - 'THUDM/chatglm3-6b'
-  - 'baichuan-inc/Baichuan2-7B-Chat'
-  - 'internlm/internlm-chat-7b-8k'
-  - 'Qwen/Qwen-7B-Chat'
-  - 'BAAI/AquilaChat2-7B'
-  - '01-ai/Yi-6B'
-  - 'meta-llama/Llama-2-7b-chat-hf'
-  - 'WisdomShell/CodeShell-7B-Chat'
-  - 'tiiuae/falcon-7b-instruct-with-patch'
-  - 'mosaicml/mpt-7b-chat'
-  - 'liuhaotian/llava-v1.5-7b'
-  - 'RWKV/rwkv-4-world-7b'
-local_model_hub: 'path to your local model hub'
-warm_up: 1
-num_trials: 3
-num_beams: 1 # default to greedy search
-low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
-in_out_pairs:
-  - '512-64'
-test_api:
-  - "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
-cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)

python/llm/test/benchmark/igpu-perf/512-64_434.yaml

@@ -1,12 +0,0 @@
-repo_id:
-  - 'mistralai/Mistral-7B-Instruct-v0.1'
-local_model_hub: 'path to your local model hub'
-warm_up: 1
-num_trials: 3
-num_beams: 1 # default to greedy search
-low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
-in_out_pairs:
-  - '512-64'
-test_api:
-  - "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
-cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)