Add Llama 3.2 to iGPU performance test (transformers 4.45) (#12209)

* Add Llama 3.2 to iGPU Perf (#12200)

* Add Llama 3.2 to iGPU Perf

* Downgrade accelerate after step

* Temporarily disable model for test

* Temporarily change ERRORLEVEL check (#12201)

* Restore llama3.2 perf (#12206)

* Revert "Temporarily change ERRORLEVEL check"

This reverts commit 909dbbc930ab4283737161a55bb32006e6ca1991.

* Revert "Temporarily disable model for test"

This reverts commit 95322dc3c6429aa836f21bda0b5ba8d9b48592f8.

---------

Co-authored-by: Jin, Qiao <89779290+JinBridger@users.noreply.github.com>

.github/workflows/llm_performance_tests.yml

@@ -745,6 +745,35 @@ jobs:
           pip uninstall trl -y
           call conda deactivate
 
+      - name: Prepare igpu perf test for transformers 4.45 (32-32 int4+fp16)
+        shell: bash
+        run: |
+          sed -i 's/{today}_test4/{today}_test5/g' python/llm/dev/benchmark/all-in-one/run.py
+          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_445.yaml
+
+      - name: Test on igpu for transformers 4.45 (32-32 int4+fp16)
+        shell: cmd
+        run: |
+          call conda activate igpu-perf
+          pip install transformers==4.45.0
+          pip install accelerate==0.33.0
+          pip install trl
+
+          set SYCL_CACHE_PERSISTENT=1
+          set BIGDL_LLM_XMX_DISABLED=1
+
+          cd python\llm\dev\benchmark\all-in-one
+          move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_445.yaml config.yaml
+          set PYTHONIOENCODING=utf-8
+          python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
+          if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
+          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
+          if %ERRORLEVEL% neq 0 (exit /b 1)
+
+          pip uninstall trl -y
+          pip install accelerate==0.23.0
+          call conda deactivate
+
       - name: Concat csv and generate html (32-32 int4+fp16)
         shell: cmd
         run: |
@@ -768,7 +797,7 @@ jobs:
         shell: bash
         run: |
           sed -i 's/32-32/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
-          sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
+          sed -i 's/{today}_test5/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
           sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml
 
       - name: Test on igpu (1024-128 int4+fp16)
@@ -869,6 +898,35 @@ jobs:
           pip uninstall trl -y
           call conda deactivate
 
+      - name: Prepare igpu perf test for transformers 4.45 (1024-128 int4+fp16)
+        shell: bash
+        run: |
+          sed -i 's/{today}_test4/{today}_test5/g' python/llm/dev/benchmark/all-in-one/run.py
+          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_445.yaml
+
+      - name: Test on igpu for transformers 4.45 (1024-128 int4+fp16)
+        shell: cmd
+        run: |
+          call conda activate igpu-perf
+          pip install transformers==4.45.0
+          pip install accelerate==0.33.0
+          pip install trl
+
+          set SYCL_CACHE_PERSISTENT=1
+          set BIGDL_LLM_XMX_DISABLED=1
+
+          cd python\llm\dev\benchmark\all-in-one
+          move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_445.yaml config.yaml
+          set PYTHONIOENCODING=utf-8
+          python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
+          if %ERRORLEVEL% neq 0 (exit /b 1)
+          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
+          if %ERRORLEVEL% neq 0 (exit /b 1)
+
+          pip uninstall trl -y
+          pip install accelerate==0.23.0
+          call conda deactivate
+
       - name: Concat csv and generate html (1024-128 int4+fp16)
         shell: cmd
         run: |
@@ -891,7 +949,7 @@ jobs:
         shell: bash
         run: |
           sed -i 's/1024-128/2048-256/g' python/llm/dev/benchmark/all-in-one/run.py
-          sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
+          sed -i 's/{today}_test5/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
           sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16.yaml
 
       - name: Test on igpu (2048-256 int4+fp16)
@@ -992,6 +1050,35 @@ jobs:
           pip uninstall trl -y
           call conda deactivate
 
+      - name: Prepare igpu perf test for transformers 4.45 (2048-256 int4+fp16)
+        shell: bash
+        run: |
+          sed -i 's/{today}_test4/{today}_test5/g' python/llm/dev/benchmark/all-in-one/run.py
+          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_445.yaml
+
+      - name: Test on igpu for transformers 4.45 (2048-256 int4+fp16)
+        shell: cmd
+        run: |
+          call conda activate igpu-perf
+          pip install transformers==4.45.0
+          pip install accelerate==0.33.0
+          pip install trl
+
+          set SYCL_CACHE_PERSISTENT=1
+          set BIGDL_LLM_XMX_DISABLED=1
+
+          cd python\llm\dev\benchmark\all-in-one
+          move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_445.yaml config.yaml
+          set PYTHONIOENCODING=utf-8
+          python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
+          if %ERRORLEVEL% neq 0 (exit /b 1)
+          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
+          if %ERRORLEVEL% neq 0 (exit /b 1)
+
+          pip uninstall trl -y
+          pip install accelerate==0.23.0
+          call conda deactivate
+
       - name: Concat csv and generate html (2048-256 int4+fp16)
         shell: cmd
         run: |
@@ -1014,7 +1101,7 @@ jobs:
         shell: bash
         run: |
           sed -i 's/2048-256/3072-384/g' python/llm/dev/benchmark/all-in-one/run.py
-          sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
+          sed -i 's/{today}_test5/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
           sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16.yaml
       
       - name: Test on igpu (3072-384 int4+fp16)
@@ -1115,6 +1202,35 @@ jobs:
           pip uninstall trl -y
           call conda deactivate
 
+      - name: Prepare igpu perf test for transformers 4.45 (3072-384 int4+fp16)
+        shell: bash
+        run: |
+          sed -i 's/{today}_test4/{today}_test5/g' python/llm/dev/benchmark/all-in-one/run.py
+          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16_445.yaml
+
+      - name: Test on igpu for transformers 4.45 (3072-384 int4+fp16)
+        shell: cmd
+        run: |
+          call conda activate igpu-perf
+          pip install transformers==4.45.0
+          pip install accelerate==0.33.0
+          pip install trl
+
+          set SYCL_CACHE_PERSISTENT=1
+          set BIGDL_LLM_XMX_DISABLED=1
+
+          cd python\llm\dev\benchmark\all-in-one
+          move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_445.yaml config.yaml
+          set PYTHONIOENCODING=utf-8
+          python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
+          if %ERRORLEVEL% neq 0 (exit /b 1)
+          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
+          if %ERRORLEVEL% neq 0 (exit /b 1)
+
+          pip uninstall trl -y
+          pip install accelerate==0.23.0
+          call conda deactivate
+
       - name: Concat csv and generate html (3072-384 int4+fp16)
         shell: cmd
         run: |
@@ -1137,7 +1253,7 @@ jobs:
         shell: bash
         run: |
           sed -i 's/3072-384/4096-512/g' python/llm/dev/benchmark/all-in-one/run.py
-          sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
+          sed -i 's/{today}_test5/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
           sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16.yaml
       
       - name: Test on igpu (4096-512 int4+fp16)
@@ -1213,6 +1329,35 @@ jobs:
           pip uninstall trl -y
           call conda deactivate
 
+      - name: Prepare igpu perf test for transformers 4.45 (4096-512 int4+fp16)
+        shell: bash
+        run: |
+          sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
+          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16_445.yaml
+
+      - name: Test on igpu for transformers 4.45 (4096-512 int4+fp16)
+        shell: cmd
+        run: |
+          call conda activate igpu-perf
+          pip install transformers==4.45.0
+          pip install accelerate==0.33.0
+          pip install trl
+
+          set SYCL_CACHE_PERSISTENT=1
+          set BIGDL_LLM_XMX_DISABLED=1
+
+          cd python\llm\dev\benchmark\all-in-one
+          move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_445.yaml config.yaml
+          set PYTHONIOENCODING=utf-8
+          python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
+          if %ERRORLEVEL% neq 0 (exit /b 1)
+          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
+          if %ERRORLEVEL% neq 0 (exit /b 1)
+
+          pip uninstall trl -y
+          pip install accelerate==0.23.0
+          call conda deactivate
+
       - name: Concat csv and generate html (4096-512 int4+fp16)
         shell: cmd
         run: |
@@ -1235,7 +1380,7 @@ jobs:
         shell: bash
         run: |
           sed -i 's/4096-512/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
-          sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
+          sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
           sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit.yaml
 
       - name: Test on igpu (load_low_bit 1024-128 int4+fp16)
@@ -1336,6 +1481,35 @@ jobs:
           pip uninstall trl -y
           call conda deactivate
 
+      - name: Prepare igpu perf test for transformers 4.45 (load_low_bit 1024-128 int4+fp16)
+        shell: bash
+        run: |
+          sed -i 's/{today}_test4/{today}_test5/g' python/llm/dev/benchmark/all-in-one/run.py
+          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_445.yaml
+
+      - name: Test on igpu for transformers 4.45 (load_low_bit 1024-128 int4+fp16)
+        shell: cmd
+        run: |
+          call conda activate igpu-perf
+          pip install transformers==4.45.0
+          pip install accelerate==0.33.0
+          pip install trl
+
+          set SYCL_CACHE_PERSISTENT=1
+          set BIGDL_LLM_XMX_DISABLED=1
+
+          cd python\llm\dev\benchmark\all-in-one
+          move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_445.yaml config.yaml
+          set PYTHONIOENCODING=utf-8
+          python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
+          if %ERRORLEVEL% neq 0 (exit /b 1)
+          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
+          if %ERRORLEVEL% neq 0 (exit /b 1)
+
+          pip uninstall trl -y
+          pip install accelerate==0.23.0
+          call conda deactivate
+
       - name: Concat csv and generate html (load_low_bit 1024-128 int4+fp16)
         shell: cmd
         run: |
@@ -1357,7 +1531,7 @@ jobs:
       - name: Prepare igpu perf test (1024-128)
         shell: bash
         run: |
-          sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
+          sed -i 's/{today}_test5/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
           sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128.yaml
 
       - name: Test on igpu (1024-128)
@@ -1458,6 +1632,35 @@ jobs:
           pip uninstall trl -y
           call conda deactivate
 
+      - name: Prepare igpu perf test for transformers 4.45 (1024-128)
+        shell: bash
+        run: |
+          sed -i 's/{today}_test4/{today}_test5/g' python/llm/dev/benchmark/all-in-one/run.py
+          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_445.yaml
+
+      - name: Test on igpu for transformers 4.45 (1024-128)
+        shell: cmd
+        run: |
+          call conda activate igpu-perf
+          pip install transformers==4.45.0
+          pip install accelerate==0.33.0
+          pip install trl
+
+          set SYCL_CACHE_PERSISTENT=1
+          set BIGDL_LLM_XMX_DISABLED=1
+
+          cd python\llm\dev\benchmark\all-in-one
+          move ..\..\..\test\benchmark\igpu-perf\1024-128_445.yaml config.yaml
+          set PYTHONIOENCODING=utf-8
+          python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
+          if %ERRORLEVEL% neq 0 (exit /b 1)
+          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
+          if %ERRORLEVEL% neq 0 (exit /b 1)
+
+          pip uninstall trl -y
+          pip install accelerate==0.23.0
+          call conda deactivate
+
       - name: Concat csv and generate html (1024-128)
         shell: cmd
         run: |

python/llm/test/benchmark/igpu-perf/1024-128_445.yaml

@@ -0,0 +1,14 @@
+repo_id:
+  - 'meta-llama/Llama-3.2-1B-Instruct'
+  - 'meta-llama/Llama-3.2-3B-Instruct'
+local_model_hub: 'path to your local model hub'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 1 # default to 1
+in_out_pairs:
+  - '1024-128'
+test_api:
+  - "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
+cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)

python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_445.yaml

@@ -0,0 +1,14 @@
+repo_id:
+  - 'meta-llama/Llama-3.2-1B-Instruct'
+  - 'meta-llama/Llama-3.2-3B-Instruct'
+local_model_hub: 'path to your local model hub'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 1 # default to 1
+in_out_pairs:
+  - '1024-128'
+test_api:
+  - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows, use fp16 for non-linear layer
+cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)

python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_445.yaml

@@ -0,0 +1,14 @@
+repo_id:
+  - 'meta-llama/Llama-3.2-1B-Instruct'
+  - 'meta-llama/Llama-3.2-3B-Instruct'
+local_model_hub: 'path to your local model hub'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 1 # default to 1
+in_out_pairs:
+  - '1024-128'
+test_api:
+  - "transformer_int4_fp16_loadlowbit_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
+cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)

python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_445.yaml

@@ -0,0 +1,14 @@
+repo_id:
+  - 'meta-llama/Llama-3.2-1B-Instruct'
+  - 'meta-llama/Llama-3.2-3B-Instruct'
+local_model_hub: 'path to your local model hub'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 1 # default to 1
+in_out_pairs:
+  - '2048-256'
+test_api:
+  - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
+cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)

python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16_445.yaml

@@ -0,0 +1,14 @@
+repo_id:
+  - 'meta-llama/Llama-3.2-1B-Instruct'
+  - 'meta-llama/Llama-3.2-3B-Instruct'
+local_model_hub: 'path to your local model hub'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 1 # default to 1
+in_out_pairs:
+  - '3072-384'
+test_api:
+  - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
+cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)

python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_445.yaml

@@ -0,0 +1,14 @@
+repo_id:
+  - 'meta-llama/Llama-3.2-1B-Instruct'
+  - 'meta-llama/Llama-3.2-3B-Instruct'
+local_model_hub: 'path to your local model hub'
+warm_up: 3
+num_trials: 5
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 1 # default to 1
+in_out_pairs:
+  - '32-32'
+test_api:
+  - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
+cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)

python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16_445.yaml

@@ -0,0 +1,14 @@
+repo_id:
+  - 'meta-llama/Llama-3.2-1B-Instruct'
+  - 'meta-llama/Llama-3.2-3B-Instruct'
+local_model_hub: 'path to your local model hub'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 1 # default to 1
+in_out_pairs:
+  - '4096-512'
+test_api:
+  - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
+cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)