From b88c1df3244128968f31d32333c8d94e0963fb7b Mon Sep 17 00:00:00 2001
From: Yuwen Hu <54161268+Oscilloscope98@users.noreply.github.com>
Date: Thu, 17 Oct 2024 21:12:45 +0800
Subject: [PATCH] Add Llama 3.1 & 3.2 to Arc Performance test (#12225)

* Add llama3.1 and llama3.2 in arc perf (#12202)

* Add llama3.1 and llama3.2 in arc perf

* Uninstall trl after arc test on transformers>=4.40

* Fix arc llama3 perf (#12212)

* Fix pip uninstall

* Uninstall trl after test on transformers==4.43.1

* Fix llama3 arc perf (#12218)

---------

Co-authored-by: Jin, Qiao <89779290+JinBridger@users.noreply.github.com>
---
 .github/workflows/llm_performance_tests.yml   | 70 +++++++++++++++++++
 .../arc-perf-transformers-443-batch2.yaml     | 16 +++++
 .../arc-perf-transformers-443-batch4.yaml     | 16 +++++
 .../benchmark/arc-perf-transformers-443.yaml  | 16 +++++
 .../arc-perf-transformers-445-batch2.yaml     | 17 +++++
 .../arc-perf-transformers-445-batch4.yaml     | 17 +++++
 .../benchmark/arc-perf-transformers-445.yaml  | 17 +++++
 7 files changed, 169 insertions(+)
 create mode 100644 python/llm/test/benchmark/arc-perf-transformers-443-batch2.yaml
 create mode 100644 python/llm/test/benchmark/arc-perf-transformers-443-batch4.yaml
 create mode 100644 python/llm/test/benchmark/arc-perf-transformers-443.yaml
 create mode 100644 python/llm/test/benchmark/arc-perf-transformers-445-batch2.yaml
 create mode 100644 python/llm/test/benchmark/arc-perf-transformers-445-batch4.yaml
 create mode 100644 python/llm/test/benchmark/arc-perf-transformers-445.yaml

diff --git a/.github/workflows/llm_performance_tests.yml b/.github/workflows/llm_performance_tests.yml
index 1f358f12..605b0162 100644
--- a/.github/workflows/llm_performance_tests.yml
+++ b/.github/workflows/llm_performance_tests.yml
@@ -233,6 +233,76 @@ jobs:
           python run.py
           mv *.csv test_batch1
 
+      - name: Test on xpu(transformers==4.43.1)
+        shell: bash
+        run: |
+          source /opt/intel/oneapi/setvars.sh
+          export USE_XETLA=OFF
+          export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
+          # upgrade for default transformers version
+          python -m pip install transformers==4.43.1
+          python -m pip install trl
+          # batch_size 1
+          cp python/llm/test/benchmark/arc-perf-transformers-443.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cd python/llm/dev/benchmark/all-in-one
+          # change csv name
+          sed -i 's/test3_batch1/test4_batch1/g' run.py
+          python run.py
+          mv *.csv test_batch1
+          # batch_size 2
+          cd ../../../../../ 
+          cp python/llm/test/benchmark/arc-perf-transformers-443-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cd python/llm/dev/benchmark/all-in-one
+          # change csv name 
+          sed -i 's/batch1/batch2/g' run.py
+          python run.py
+          mv *.csv test_batch2
+          # batch_size 4
+          cd ../../../../../ 
+          cp python/llm/test/benchmark/arc-perf-transformers-443-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cd python/llm/dev/benchmark/all-in-one
+          # change csv name 
+          sed -i 's/batch2/batch4/g' run.py
+          python run.py
+          mv *.csv test_batch4
+          python -m pip uninstall -y trl
+      
+      - name: Test on xpu(transformers==4.45.0)
+        shell: bash
+        run: |
+          source /opt/intel/oneapi/setvars.sh
+          export USE_XETLA=OFF
+          export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
+          # upgrade for default transformers version
+          python -m pip install transformers==4.45.0
+          python -m pip install trl
+          python -m pip install accelerate==0.33.0
+          # batch_size 1
+          cp python/llm/test/benchmark/arc-perf-transformers-445.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cd python/llm/dev/benchmark/all-in-one
+          # change csv name
+          sed -i 's/test4_batch4/test5_batch1/g' run.py
+          python run.py
+          mv *.csv test_batch1
+          # batch_size 2
+          cd ../../../../../ 
+          cp python/llm/test/benchmark/arc-perf-transformers-445-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cd python/llm/dev/benchmark/all-in-one
+          # change csv name 
+          sed -i 's/batch1/batch2/g' run.py
+          python run.py
+          mv *.csv test_batch2
+          # batch_size 4
+          cd ../../../../../ 
+          cp python/llm/test/benchmark/arc-perf-transformers-445-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cd python/llm/dev/benchmark/all-in-one
+          # change csv name 
+          sed -i 's/batch2/batch4/g' run.py
+          python run.py
+          mv *.csv test_batch4
+          python -m pip install accelerate==0.23.0
+          python -m pip uninstall -y trl
+
       - name: Concat csv and generate html
         shell: bash
         run: |
diff --git a/python/llm/test/benchmark/arc-perf-transformers-443-batch2.yaml b/python/llm/test/benchmark/arc-perf-transformers-443-batch2.yaml
new file mode 100644
index 00000000..6f059403
--- /dev/null
+++ b/python/llm/test/benchmark/arc-perf-transformers-443-batch2.yaml
@@ -0,0 +1,16 @@
+repo_id:
+  - 'meta-llama/Llama-3.1-8B-Instruct'
+local_model_hub: '/mnt/disk1/models'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 2 # default to 1
+in_out_pairs:
+  - '32-32'
+  - '1024-128'
+  - '2048-256'
+test_api:
+  - "transformer_int4_fp16_gpu"  # on Intel GPU
+cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-443-batch4.yaml b/python/llm/test/benchmark/arc-perf-transformers-443-batch4.yaml
new file mode 100644
index 00000000..05585695
--- /dev/null
+++ b/python/llm/test/benchmark/arc-perf-transformers-443-batch4.yaml
@@ -0,0 +1,16 @@
+repo_id:
+  - 'meta-llama/Llama-3.1-8B-Instruct'
+local_model_hub: '/mnt/disk1/models'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 4 # default to 1
+in_out_pairs:
+  - '32-32'
+  - '1024-128'
+  - '2048-256'
+test_api:
+  - "transformer_int4_fp16_gpu"  # on Intel GPU
+cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-443.yaml b/python/llm/test/benchmark/arc-perf-transformers-443.yaml
new file mode 100644
index 00000000..d4df9a26
--- /dev/null
+++ b/python/llm/test/benchmark/arc-perf-transformers-443.yaml
@@ -0,0 +1,16 @@
+repo_id:
+  - 'meta-llama/Llama-3.1-8B-Instruct'
+local_model_hub: '/mnt/disk1/models'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 1 # default to 1
+in_out_pairs:
+  - '32-32'
+  - '1024-128'
+  - '2048-256'
+test_api:
+  - "transformer_int4_fp16_gpu"  # on Intel GPU
+cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-445-batch2.yaml b/python/llm/test/benchmark/arc-perf-transformers-445-batch2.yaml
new file mode 100644
index 00000000..f9f160b9
--- /dev/null
+++ b/python/llm/test/benchmark/arc-perf-transformers-445-batch2.yaml
@@ -0,0 +1,17 @@
+repo_id:
+  - 'meta-llama/Llama-3.2-1B-Instruct'
+  - 'meta-llama/Llama-3.2-3B-Instruct'
+local_model_hub: '/mnt/disk1/models'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 2 # default to 1
+in_out_pairs:
+  - '32-32'
+  - '1024-128'
+  - '2048-256'
+test_api:
+  - "transformer_int4_fp16_gpu"  # on Intel GPU
+cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-445-batch4.yaml b/python/llm/test/benchmark/arc-perf-transformers-445-batch4.yaml
new file mode 100644
index 00000000..e2171aaf
--- /dev/null
+++ b/python/llm/test/benchmark/arc-perf-transformers-445-batch4.yaml
@@ -0,0 +1,17 @@
+repo_id:
+  - 'meta-llama/Llama-3.2-1B-Instruct'
+  - 'meta-llama/Llama-3.2-3B-Instruct'
+local_model_hub: '/mnt/disk1/models'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 4 # default to 1
+in_out_pairs:
+  - '32-32'
+  - '1024-128'
+  - '2048-256'
+test_api:
+  - "transformer_int4_fp16_gpu"  # on Intel GPU
+cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-445.yaml b/python/llm/test/benchmark/arc-perf-transformers-445.yaml
new file mode 100644
index 00000000..f1a2e0c5
--- /dev/null
+++ b/python/llm/test/benchmark/arc-perf-transformers-445.yaml
@@ -0,0 +1,17 @@
+repo_id:
+  - 'meta-llama/Llama-3.2-1B-Instruct'
+  - 'meta-llama/Llama-3.2-3B-Instruct'
+local_model_hub: '/mnt/disk1/models'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 1 # default to 1
+in_out_pairs:
+  - '32-32'
+  - '1024-128'
+  - '2048-256'
+test_api:
+  - "transformer_int4_fp16_gpu"  # on Intel GPU
+cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'