diff --git a/.github/workflows/llm_performance_tests.yml b/.github/workflows/llm_performance_tests.yml index 4b092ed0..1f358f12 100644 --- a/.github/workflows/llm_performance_tests.yml +++ b/.github/workflows/llm_performance_tests.yml @@ -745,6 +745,35 @@ jobs: pip uninstall trl -y call conda deactivate + - name: Prepare igpu perf test for transformers 4.45 (32-32 int4+fp16) + shell: bash + run: | + sed -i 's/{today}_test4/{today}_test5/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_445.yaml + + - name: Test on igpu for transformers 4.45 (32-32 int4+fp16) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.45.0 + pip install accelerate==0.33.0 + pip install trl + + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_445.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1) + python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 + if %ERRORLEVEL% neq 0 (exit /b 1) + + pip uninstall trl -y + pip install accelerate==0.23.0 + call conda deactivate + - name: Concat csv and generate html (32-32 int4+fp16) shell: cmd run: | @@ -768,7 +797,7 @@ jobs: shell: bash run: | sed -i 's/32-32/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i 's/{today}_test5/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml - name: Test on igpu (1024-128 int4+fp16) @@ -869,6 +898,35 @@ jobs: pip uninstall trl -y call conda deactivate + - name: Prepare igpu perf test for transformers 4.45 (1024-128 int4+fp16) + shell: bash + run: | + sed -i 's/{today}_test4/{today}_test5/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_445.yaml + + - name: Test on igpu for transformers 4.45 (1024-128 int4+fp16) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.45.0 + pip install accelerate==0.33.0 + pip install trl + + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_445.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 + if %ERRORLEVEL% neq 0 (exit /b 1) + + pip uninstall trl -y + pip install accelerate==0.23.0 + call conda deactivate + - name: Concat csv and generate html (1024-128 int4+fp16) shell: cmd run: | @@ -891,7 +949,7 @@ jobs: shell: bash run: | sed -i 's/1024-128/2048-256/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i 's/{today}_test5/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16.yaml - name: Test on igpu (2048-256 int4+fp16) @@ -992,6 +1050,35 @@ jobs: pip uninstall trl -y call conda deactivate + - name: Prepare igpu perf test for transformers 4.45 (2048-256 int4+fp16) + shell: bash + run: | + sed -i 's/{today}_test4/{today}_test5/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_445.yaml + + - name: Test on igpu for transformers 4.45 (2048-256 int4+fp16) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.45.0 + pip install accelerate==0.33.0 + pip install trl + + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_445.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 + if %ERRORLEVEL% neq 0 (exit /b 1) + + pip uninstall trl -y + pip install accelerate==0.23.0 + call conda deactivate + - name: Concat csv and generate html (2048-256 int4+fp16) shell: cmd run: | @@ -1014,7 +1101,7 @@ jobs: shell: bash run: | sed -i 's/2048-256/3072-384/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i 's/{today}_test5/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16.yaml - name: Test on igpu (3072-384 int4+fp16) @@ -1115,6 +1202,35 @@ jobs: pip uninstall trl -y call conda deactivate + - name: Prepare igpu perf test for transformers 4.45 (3072-384 int4+fp16) + shell: bash + run: | + sed -i 's/{today}_test4/{today}_test5/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16_445.yaml + + - name: Test on igpu for transformers 4.45 (3072-384 int4+fp16) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.45.0 + pip install accelerate==0.33.0 + pip install trl + + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_445.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 + if %ERRORLEVEL% neq 0 (exit /b 1) + + pip uninstall trl -y + pip install accelerate==0.23.0 + call conda deactivate + - name: Concat csv and generate html (3072-384 int4+fp16) shell: cmd run: | @@ -1137,7 +1253,7 @@ jobs: shell: bash run: | sed -i 's/3072-384/4096-512/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i 's/{today}_test5/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16.yaml - name: Test on igpu (4096-512 int4+fp16) @@ -1213,6 +1329,35 @@ jobs: pip uninstall trl -y call conda deactivate + - name: Prepare igpu perf test for transformers 4.45 (4096-512 int4+fp16) + shell: bash + run: | + sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16_445.yaml + + - name: Test on igpu for transformers 4.45 (4096-512 int4+fp16) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.45.0 + pip install accelerate==0.33.0 + pip install trl + + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_445.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 + if %ERRORLEVEL% neq 0 (exit /b 1) + + pip uninstall trl -y + pip install accelerate==0.23.0 + call conda deactivate + - name: Concat csv and generate html (4096-512 int4+fp16) shell: cmd run: | @@ -1235,7 +1380,7 @@ jobs: shell: bash run: | sed -i 's/4096-512/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit.yaml - name: Test on igpu (load_low_bit 1024-128 int4+fp16) @@ -1336,6 +1481,35 @@ jobs: pip uninstall trl -y call conda deactivate + - name: Prepare igpu perf test for transformers 4.45 (load_low_bit 1024-128 int4+fp16) + shell: bash + run: | + sed -i 's/{today}_test4/{today}_test5/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_445.yaml + + - name: Test on igpu for transformers 4.45 (load_low_bit 1024-128 int4+fp16) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.45.0 + pip install accelerate==0.33.0 + pip install trl + + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_445.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 + if %ERRORLEVEL% neq 0 (exit /b 1) + + pip uninstall trl -y + pip install accelerate==0.23.0 + call conda deactivate + - name: Concat csv and generate html (load_low_bit 1024-128 int4+fp16) shell: cmd run: | @@ -1357,7 +1531,7 @@ jobs: - name: Prepare igpu perf test (1024-128) shell: bash run: | - sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i 's/{today}_test5/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128.yaml - name: Test on igpu (1024-128) @@ -1458,6 +1632,35 @@ jobs: pip uninstall trl -y call conda deactivate + - name: Prepare igpu perf test for transformers 4.45 (1024-128) + shell: bash + run: | + sed -i 's/{today}_test4/{today}_test5/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_445.yaml + + - name: Test on igpu for transformers 4.45 (1024-128) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.45.0 + pip install accelerate==0.33.0 + pip install trl + + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\1024-128_445.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 + if %ERRORLEVEL% neq 0 (exit /b 1) + + pip uninstall trl -y + pip install accelerate==0.23.0 + call conda deactivate + - name: Concat csv and generate html (1024-128) shell: cmd run: | diff --git a/python/llm/test/benchmark/igpu-perf/1024-128_445.yaml b/python/llm/test/benchmark/igpu-perf/1024-128_445.yaml new file mode 100644 index 00000000..100fcb6b --- /dev/null +++ b/python/llm/test/benchmark/igpu-perf/1024-128_445.yaml @@ -0,0 +1,14 @@ +repo_id: + - 'meta-llama/Llama-3.2-1B-Instruct' + - 'meta-llama/Llama-3.2-3B-Instruct' +local_model_hub: 'path to your local model hub' +warm_up: 1 +num_trials: 3 +num_beams: 1 # default to greedy search +low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) +batch_size: 1 # default to 1 +in_out_pairs: + - '1024-128' +test_api: + - "transformer_int4_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) +cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) diff --git a/python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_445.yaml b/python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_445.yaml new file mode 100644 index 00000000..11b2b84a --- /dev/null +++ b/python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_445.yaml @@ -0,0 +1,14 @@ +repo_id: + - 'meta-llama/Llama-3.2-1B-Instruct' + - 'meta-llama/Llama-3.2-3B-Instruct' +local_model_hub: 'path to your local model hub' +warm_up: 1 +num_trials: 3 +num_beams: 1 # default to greedy search +low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) +batch_size: 1 # default to 1 +in_out_pairs: + - '1024-128' +test_api: + - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows, use fp16 for non-linear layer +cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) diff --git a/python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_445.yaml b/python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_445.yaml new file mode 100644 index 00000000..a041e4dd --- /dev/null +++ b/python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_445.yaml @@ -0,0 +1,14 @@ +repo_id: + - 'meta-llama/Llama-3.2-1B-Instruct' + - 'meta-llama/Llama-3.2-3B-Instruct' +local_model_hub: 'path to your local model hub' +warm_up: 1 +num_trials: 3 +num_beams: 1 # default to greedy search +low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) +batch_size: 1 # default to 1 +in_out_pairs: + - '1024-128' +test_api: + - "transformer_int4_fp16_loadlowbit_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) +cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) diff --git a/python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_445.yaml b/python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_445.yaml new file mode 100644 index 00000000..078c7427 --- /dev/null +++ b/python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_445.yaml @@ -0,0 +1,14 @@ +repo_id: + - 'meta-llama/Llama-3.2-1B-Instruct' + - 'meta-llama/Llama-3.2-3B-Instruct' +local_model_hub: 'path to your local model hub' +warm_up: 1 +num_trials: 3 +num_beams: 1 # default to greedy search +low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) +batch_size: 1 # default to 1 +in_out_pairs: + - '2048-256' +test_api: + - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) +cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) diff --git a/python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16_445.yaml b/python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16_445.yaml new file mode 100644 index 00000000..67510f87 --- /dev/null +++ b/python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16_445.yaml @@ -0,0 +1,14 @@ +repo_id: + - 'meta-llama/Llama-3.2-1B-Instruct' + - 'meta-llama/Llama-3.2-3B-Instruct' +local_model_hub: 'path to your local model hub' +warm_up: 1 +num_trials: 3 +num_beams: 1 # default to greedy search +low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) +batch_size: 1 # default to 1 +in_out_pairs: + - '3072-384' +test_api: + - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) +cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) diff --git a/python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_445.yaml b/python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_445.yaml new file mode 100644 index 00000000..87a12ac4 --- /dev/null +++ b/python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_445.yaml @@ -0,0 +1,14 @@ +repo_id: + - 'meta-llama/Llama-3.2-1B-Instruct' + - 'meta-llama/Llama-3.2-3B-Instruct' +local_model_hub: 'path to your local model hub' +warm_up: 3 +num_trials: 5 +num_beams: 1 # default to greedy search +low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) +batch_size: 1 # default to 1 +in_out_pairs: + - '32-32' +test_api: + - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) +cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api) diff --git a/python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16_445.yaml b/python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16_445.yaml new file mode 100644 index 00000000..bfe7cc47 --- /dev/null +++ b/python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16_445.yaml @@ -0,0 +1,14 @@ +repo_id: + - 'meta-llama/Llama-3.2-1B-Instruct' + - 'meta-llama/Llama-3.2-3B-Instruct' +local_model_hub: 'path to your local model hub' +warm_up: 1 +num_trials: 3 +num_beams: 1 # default to greedy search +low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4) +batch_size: 1 # default to 1 +in_out_pairs: + - '4096-512' +test_api: + - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows (catch GPU peak memory) +cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)