8ef4caaf5d
* Add 3k&4k input in workflow for iGPU (#11685) * add 3k&4k input in workflow * comment for test * comment models for accelarate test * remove OOM models * modify typo * change test model (#11696) * reverse test models (#11700)
1334 lines
55 KiB
YAML
1334 lines
55 KiB
YAML
name: LLM Performance Test
|
|
|
|
# Cancel previous runs in the PR when you push new commits
|
|
concurrency:
|
|
group: ${{ github.workflow }}-llm-performance-tests-${{ github.event.pull_request.number || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Controls when the action will run.
|
|
on:
|
|
# schedule:
|
|
# - cron: "30 16 * * *" # GMT time, 16:30 GMT == 00:30 China
|
|
# please uncomment it for PR tests
|
|
# pull_request:
|
|
# branches: [main]
|
|
# paths:
|
|
# - ".github/workflows/llm_performance_tests.yml"
|
|
# - "python/llm/test/benchmark/**"
|
|
# - "python/llm/dev/benchmark/all-in-one/**"
|
|
# workflow_dispatch:
|
|
# inputs:
|
|
# arc:
|
|
# description: "If trigger performance test on Arc"
|
|
# required: false
|
|
# type: boolean
|
|
# default: true
|
|
# spr:
|
|
# description: "If trigger performance test on SPR"
|
|
# required: false
|
|
# type: boolean
|
|
# default: true
|
|
# core:
|
|
# description: "If trigger performance test on Core"
|
|
# required: false
|
|
# type: boolean
|
|
# default: true
|
|
# igpu:
|
|
# description: "If trigger performance test on iGPU"
|
|
# required: false
|
|
# type: boolean
|
|
# default: true
|
|
workflow_call:
|
|
inputs:
|
|
checkout-ref:
|
|
description: 'ref for checking out, including branch, tag or SHA'
|
|
required: true
|
|
type: string
|
|
arc:
|
|
description: "If trigger performance test on Arc"
|
|
required: false
|
|
type: boolean
|
|
default: true
|
|
spr:
|
|
description: "If trigger performance test on SPR"
|
|
required: false
|
|
type: boolean
|
|
default: true
|
|
core:
|
|
description: "If trigger performance test on Core"
|
|
required: false
|
|
type: boolean
|
|
default: true
|
|
igpu:
|
|
description: "If trigger performance test on iGPU"
|
|
required: false
|
|
type: boolean
|
|
default: true
|
|
|
|
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
|
|
jobs:
|
|
llm-cpp-build:
|
|
uses: ./.github/workflows/llm-binary-build.yml
|
|
with:
|
|
platform: ${{ (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main')) && 'Windows,Linux' || 'Dummy' }}
|
|
|
|
llm-performance-test-on-arc:
|
|
if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && inputs.arc ) }} # please comment it for PR tests
|
|
needs: llm-cpp-build
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
python-version: ["3.11"]
|
|
runs-on: [self-hosted, llm, perf]
|
|
env:
|
|
OMP_NUM_THREADS: 16
|
|
THREAD_NUM: 16
|
|
ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
|
|
CSV_SAVE_PATH: ${{ (github.event.schedule || (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main'))) && '/mnt/disk1/nightly_perf_gpu/' || '/mnt/disk1/pr_perf_gpu/' }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
|
|
with:
|
|
repository: 'intel-analytics/ipex-llm'
|
|
ref: ${{ inputs.checkout-ref }}
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Install dependencies
|
|
shell: bash
|
|
# pip install transformers_stream_generator for model internlm-chat-7b-8k
|
|
# pip install tiktoken for model Qwen-7B-Chat-10-12
|
|
# pip install matplotlib for model Qwen-VL-Chat
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
python -m pip install --upgrade wheel
|
|
python -m pip install --upgrade omegaconf
|
|
python -m pip install --upgrade pandas
|
|
python -m pip install --upgrade einops
|
|
python -m pip install --upgrade transformers_stream_generator
|
|
python -m pip install --upgrade tiktoken
|
|
python -m pip install --upgrade matplotlib
|
|
|
|
# specific for test on certain commits
|
|
- name: Download llm binary
|
|
if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
|
|
uses: ./.github/actions/llm/download-llm-binary
|
|
|
|
- name: Install IPEX-LLM from source
|
|
if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
|
|
uses: ./.github/actions/llm/setup-llm-env
|
|
with:
|
|
extra-dependency: "xpu_2.1"
|
|
|
|
- name: Install IPEX-LLM from Pypi
|
|
if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
|
|
shell: bash
|
|
run: |
|
|
pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
|
|
test_version_date=`date -d 'yesterday' '+%Y%m%d'`
|
|
if ! pip show ipex-llm | grep $test_version_date; then
|
|
echo "Did not install ipex-llm with excepted version $test_version_date"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Test installed xpu version
|
|
shell: bash
|
|
run: |
|
|
source /opt/intel/oneapi/setvars.sh
|
|
python -m pip install --upgrade pytest
|
|
python -m pip install --upgrade expecttest
|
|
bash python/llm/test/run-llm-install-tests.sh
|
|
|
|
- name: Test on xpu(transformers==4.36.2)
|
|
shell: bash
|
|
run: |
|
|
date_for_test_version=$(date -d yesterday +%Y-%m-%d)
|
|
sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
|
|
source /opt/intel/oneapi/setvars.sh
|
|
export USE_XETLA=OFF
|
|
export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
|
|
cp python/llm/test/benchmark/arc-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
|
|
cd python/llm/dev/benchmark/all-in-one
|
|
mkdir test_batch1
|
|
mkdir test_batch2
|
|
mkdir test_batch4
|
|
# batch_size 1
|
|
# hide time info
|
|
sed -i 's/str(end - st)/"xxxxxx"/g' run.py
|
|
# change csv name
|
|
sed -i 's/{today}/{today}_test1_batch1/g' run.py
|
|
python run.py
|
|
mv *.csv test_batch1
|
|
# batch_size 2
|
|
cd ../../../../../
|
|
cp python/llm/test/benchmark/arc-perf-test-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
|
|
cd python/llm/dev/benchmark/all-in-one
|
|
# change csv name
|
|
sed -i 's/batch1/batch2/g' run.py
|
|
python run.py
|
|
mv *.csv test_batch2
|
|
# batch_size 4
|
|
cd ../../../../../
|
|
cp python/llm/test/benchmark/arc-perf-test-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
|
|
cd python/llm/dev/benchmark/all-in-one
|
|
# change csv name
|
|
sed -i 's/batch2/batch4/g' run.py
|
|
python run.py
|
|
mv *.csv test_batch4
|
|
|
|
- name: Test on xpu(transformers==4.37.0)
|
|
shell: bash
|
|
run: |
|
|
source /opt/intel/oneapi/setvars.sh
|
|
export USE_XETLA=OFF
|
|
export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
|
|
# upgrade transformers for model Qwen/Qwen1.5-7B-Chat
|
|
python -m pip install transformers==4.37.0
|
|
# batch_size 1
|
|
cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml
|
|
cd python/llm/dev/benchmark/all-in-one
|
|
# change csv name
|
|
sed -i 's/test1_batch4/test2_batch1/g' run.py
|
|
python run.py
|
|
mv *.csv test_batch1
|
|
# batch_size 2
|
|
cd ../../../../../
|
|
cp python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
|
|
cd python/llm/dev/benchmark/all-in-one
|
|
# change csv name
|
|
sed -i 's/batch1/batch2/g' run.py
|
|
python run.py
|
|
mv *.csv test_batch2
|
|
# batch_size 4
|
|
cd ../../../../../
|
|
cp python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
|
|
cd python/llm/dev/benchmark/all-in-one
|
|
# change csv name
|
|
sed -i 's/batch2/batch4/g' run.py
|
|
python run.py
|
|
mv *.csv test_batch4
|
|
|
|
- name: Test on xpu(transformers==4.40.0)
|
|
shell: bash
|
|
run: |
|
|
source /opt/intel/oneapi/setvars.sh
|
|
export USE_XETLA=OFF
|
|
export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
|
|
# upgrade transformers for model Qwen/Qwen1.5-MoE-A2.7B-Chat
|
|
python -m pip install transformers==4.40.0
|
|
python -m pip install trl
|
|
# batch_size 1
|
|
cp python/llm/test/benchmark/arc-perf-transformers-440.yaml python/llm/dev/benchmark/all-in-one/config.yaml
|
|
cd python/llm/dev/benchmark/all-in-one
|
|
# change csv name
|
|
sed -i 's/test2_batch4/test3_batch1/g' run.py
|
|
python run.py
|
|
mv *.csv test_batch1
|
|
|
|
- name: Concat csv and generate html
|
|
shell: bash
|
|
run: |
|
|
# batch_size 1
|
|
cd python/llm/dev/benchmark/all-in-one/test_batch1
|
|
python ../../../../test/benchmark/concat_csv.py
|
|
for file in *.csv; do
|
|
if [[ $file != *test* ]]; then
|
|
cp "$file" $CSV_SAVE_PATH/batch_size_1
|
|
fi
|
|
done
|
|
python -m pip install pandas==1.5.3
|
|
cd ../../../../test/benchmark
|
|
python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_1
|
|
# batch_size 2
|
|
cd ../../../../
|
|
cd python/llm/dev/benchmark/all-in-one/test_batch2
|
|
python ../../../../test/benchmark/concat_csv.py
|
|
for file in *.csv; do
|
|
if [[ $file != *test* ]]; then
|
|
cp "$file" $CSV_SAVE_PATH/batch_size_2
|
|
fi
|
|
done
|
|
cd ../../../../test/benchmark
|
|
python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_2
|
|
# batch_size 4
|
|
cd ../../../../
|
|
cd python/llm/dev/benchmark/all-in-one/test_batch4
|
|
python ../../../../test/benchmark/concat_csv.py
|
|
for file in *.csv; do
|
|
if [[ $file != *test* ]]; then
|
|
cp "$file" $CSV_SAVE_PATH/batch_size_4
|
|
fi
|
|
done
|
|
cd ../../../../test/benchmark
|
|
python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_4
|
|
|
|
- name: Merge and sort csv files of multiple batches and generate html
|
|
shell: bash
|
|
run: |
|
|
cd python/llm/test/benchmark
|
|
mkdir merged_temp
|
|
# go through all the files and go to merged_temp
|
|
cd ../../dev/benchmark/all-in-one/test_batch1
|
|
for file in *.csv; do
|
|
if [[ $file != *test* ]]; then
|
|
cp "$file" ../../../../test/benchmark/merged_temp
|
|
fi
|
|
done
|
|
cd ../test_batch2
|
|
for file in *.csv; do
|
|
if [[ $file != *test* ]]; then
|
|
cp "$file" ../../../../test/benchmark/merged_temp
|
|
fi
|
|
done
|
|
cd ../test_batch4
|
|
for file in *.csv; do
|
|
if [[ $file != *test* ]]; then
|
|
cp "$file" ../../../../test/benchmark/merged_temp
|
|
fi
|
|
done
|
|
cd ../../../../test/benchmark
|
|
python merge_csv_batch.py -f ./merged_temp
|
|
cd merged_temp
|
|
find . -name "*batch*.csv" -delete
|
|
for file in *.csv; do
|
|
cp "$file" $CSV_SAVE_PATH/merged
|
|
done
|
|
cd ..
|
|
python csv_to_html.py -f $CSV_SAVE_PATH/merged
|
|
rm -r merged_temp
|
|
|
|
- name: Update html in parent folder
|
|
shell: bash
|
|
run: |
|
|
cd python/llm/test/benchmark
|
|
python update_html_in_parent_folder.py -f $CSV_SAVE_PATH
|
|
|
|
- name: Check and upload results to ftp
|
|
shell: bash
|
|
run: |
|
|
# batch_size 1
|
|
cd python/llm/dev/benchmark/all-in-one/test_batch1
|
|
python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test.yaml
|
|
python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437.yaml
|
|
python ../../../../test/benchmark/check_results.py -c test3 -y ../../../../test/benchmark/arc-perf-transformers-440.yaml
|
|
find . -name "*test*.csv" -delete
|
|
if [[ ${{ github.event_name }} == "schedule" ]]; then
|
|
curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
|
|
elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
|
|
curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
|
|
fi
|
|
cd ../
|
|
rm -r test_batch1
|
|
# batch_size 2
|
|
cd test_batch2
|
|
python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test-batch2.yaml
|
|
python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch2.yaml
|
|
find . -name "*test*.csv" -delete
|
|
if [[ ${{ github.event_name }} == "schedule" ]]; then
|
|
curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
|
|
elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
|
|
curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
|
|
fi
|
|
cd ../
|
|
rm -r test_batch2
|
|
# batch_size 4
|
|
cd test_batch4
|
|
python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test-batch4.yaml
|
|
python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch4.yaml
|
|
find . -name "*test*.csv" -delete
|
|
if [[ ${{ github.event_name }} == "schedule" ]]; then
|
|
curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
|
|
elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
|
|
curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
|
|
fi
|
|
cd ../
|
|
rm -r test_batch4
|
|
|
|
|
|
llm-performance-test-on-spr:
|
|
if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && inputs.spr ) }} # please comment it for PR tests
|
|
needs: llm-cpp-build
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
python-version: ["3.11"]
|
|
runs-on: [self-hosted, llm, spr-perf]
|
|
env:
|
|
OMP_NUM_THREADS: 16
|
|
THREAD_NUM: 16
|
|
ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
|
|
steps:
|
|
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
|
|
with:
|
|
repository: 'intel-analytics/ipex-llm'
|
|
ref: ${{ inputs.checkout-ref }}
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Install dependencies
|
|
shell: bash
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
python -m pip install --upgrade wheel
|
|
python -m pip install --upgrade omegaconf
|
|
python -m pip install --upgrade pandas
|
|
python -m pip install --upgrade einops
|
|
python -m pip install --upgrade tiktoken
|
|
python -m pip install --upgrade transformers_stream_generator
|
|
|
|
# specific for test on certain commits
|
|
- name: Download llm binary
|
|
if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
|
|
uses: ./.github/actions/llm/download-llm-binary
|
|
|
|
- name: Install IPEX-LLM from source
|
|
if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
|
|
uses: ./.github/actions/llm/setup-llm-env
|
|
|
|
- name: Install IPEX-LLM from Pypi
|
|
if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
|
|
shell: bash
|
|
run: |
|
|
pip install --pre --upgrade ipex-llm[all] --extra-index-url https://download.pytorch.org/whl/cpu
|
|
test_version_date=`date -d 'yesterday' '+%Y%m%d'`
|
|
if ! pip show ipex-llm | grep $test_version_date; then
|
|
echo "Did not install ipex-llm with excepted version $test_version_date"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Test on cpu
|
|
shell: bash
|
|
run: |
|
|
date_for_test_version=$(date -d yesterday +%Y-%m-%d)
|
|
sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
|
|
|
|
mv python/llm/test/benchmark/cpu-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
|
|
cd python/llm/dev/benchmark/all-in-one
|
|
export http_proxy=${HTTP_PROXY}
|
|
export https_proxy=${HTTPS_PROXY}
|
|
source ipex-llm-init -t
|
|
export OMP_NUM_THREADS=48
|
|
# hide time info
|
|
sed -i 's/str(end - st)/"xxxxxx"/g' run.py
|
|
python run.py
|
|
cp ./*.csv /mnt/disk1/models/nightly_perf_cpu
|
|
cd ../../../test/benchmark
|
|
python -m pip install pandas==1.5.3
|
|
python csv_to_html.py -f /mnt/disk1/models/nightly_perf_cpu
|
|
cd /mnt/disk1/models/nightly_perf_cpu
|
|
for f in *.html; do
|
|
curl -T "$f" ${LLM_FTP_URL}/llm/nightly_perf/nightly_perf_cpu/
|
|
done
|
|
|
|
llm-performance-test-on-core:
|
|
if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && inputs.core ) }} # please comment it for PR tests
|
|
needs: llm-cpp-build
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- os: windows
|
|
platform: dp
|
|
python-version: "3.11"
|
|
# - os: windows
|
|
# platform: lp
|
|
# python-version: "3.11"
|
|
runs-on: [self-hosted, "${{ matrix.os }}", llm, perf-core, "${{ matrix.platform }}"]
|
|
env:
|
|
ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
|
|
CSV_SAVE_PATH: ${{ (github.event.schedule || (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main'))) && 'D:/action-runners/nightly_perf_core_' || 'D:/action-runners/pr_perf_core_' }}${{ matrix.platform }}/
|
|
steps:
|
|
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
|
|
with:
|
|
repository: 'intel-analytics/ipex-llm'
|
|
ref: ${{ inputs.checkout-ref }}
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Install dependencies
|
|
shell: bash
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
python -m pip install --upgrade wheel
|
|
python -m pip install --upgrade omegaconf pandas
|
|
python -m pip install --upgrade tiktoken einops transformers_stream_generator
|
|
|
|
# specific for test on certain commits
|
|
- name: Download llm binary
|
|
if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
|
|
uses: ./.github/actions/llm/download-llm-binary
|
|
|
|
- name: Install IPEX-LLM from source
|
|
if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
|
|
uses: ./.github/actions/llm/setup-llm-env
|
|
|
|
- name: Install IPEX-LLM from Pypi
|
|
if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
|
|
shell: bash
|
|
run: |
|
|
pip install --pre --upgrade ipex-llm[all]
|
|
test_version_date=`date -d 'yesterday' '+%Y%m%d'`
|
|
if ! pip show ipex-llm | grep $test_version_date; then
|
|
echo "Did not install ipex-llm with excepted version $test_version_date"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
- name: Test on core ${{ matrix.platform }}
|
|
shell: bash
|
|
run: |
|
|
date_for_test_version=$(date -d yesterday +%Y-%m-%d)
|
|
sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
|
|
|
|
mv python/llm/test/benchmark/core-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
|
|
cd python/llm/dev/benchmark/all-in-one
|
|
export http_proxy=${HTTP_PROXY}
|
|
export https_proxy=${HTTPS_PROXY}
|
|
# hide time info
|
|
sed -i 's/str(end - st)/"xxxxxx"/g' run.py
|
|
python run.py
|
|
cp ./*.csv $CSV_SAVE_PATH
|
|
cd ../../../test/benchmark
|
|
python -m pip install pandas==1.5.3
|
|
python csv_to_html.py -f $CSV_SAVE_PATH
|
|
cd ../../dev/benchmark/all-in-one/
|
|
if [[ ${{ github.event_name }} == "schedule" ]]; then
|
|
curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/core_${{ matrix.platform }}/
|
|
elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
|
|
curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/core_${{ matrix.platform }}/
|
|
fi
|
|
|
|
llm-performance-test-on-igpu:
|
|
if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && inputs.igpu ) }} # please comment it for PR tests
|
|
needs: llm-cpp-build
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- os: windows
|
|
python-version: "3.11"
|
|
runs-on: [self-hosted, "${{ matrix.os }}", llm, perf-igpu]
|
|
env:
|
|
ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
|
|
steps:
|
|
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
|
|
with:
|
|
repository: 'intel-analytics/ipex-llm'
|
|
ref: ${{ inputs.checkout-ref }}
|
|
|
|
# TODO: Put the ipex-llm related install process for win gpu into a action function
|
|
|
|
# specific for test on certain commits
|
|
- name: Download llm binary
|
|
if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
|
|
uses: ./.github/actions/llm/download-llm-binary
|
|
|
|
- name: Prepare for install ipex-llm from source
|
|
if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/"bigdl-core-xe-21==" + CORE_XE_VERSION/"bigdl-core-xe-21"/g' python/llm/setup.py
|
|
sed -i 's/"bigdl-core-xe-batch-21==" + CORE_XE_VERSION/"bigdl-core-xe-batch-21"/g' python/llm/setup.py
|
|
sed -i 's/"bigdl-core-xe-addons-21==" + CORE_XE_VERSION/"bigdl-core-xe-addons-21"/g' python/llm/setup.py
|
|
sed -i 's/"bigdl-core-xe-esimd-21==" + CORE_XE_VERSION/"bigdl-core-xe-esimd-21"/g' python/llm/setup.py
|
|
|
|
- name: Install ipex-llm and other related packages (install from source)
|
|
if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
|
|
shell: cmd
|
|
run: |
|
|
call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y
|
|
call conda activate igpu-perf
|
|
|
|
pip install --upgrade pip
|
|
pip install --upgrade wheel
|
|
pip install --upgrade omegaconf pandas
|
|
pip install --upgrade tiktoken einops transformers_stream_generator matplotlib
|
|
|
|
cd python\llm
|
|
python setup.py clean --all bdist_wheel --win
|
|
if not exist dist\ipex_llm*.whl (exit /b 1)
|
|
for %%i in (dist\ipex_llm*.whl) do set whl_name=%%i
|
|
|
|
pip install --pre --upgrade %whl_name%[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
pip list
|
|
|
|
call conda deactivate
|
|
|
|
- name: Determine desired ipex-llm version
|
|
if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
|
|
shell: bash
|
|
run: |
|
|
test_version_date=`date -d 'yesterday' '+%Y%m%d'`
|
|
echo "TEST_VERSION_DATE=${test_version_date}" >> "$GITHUB_ENV"
|
|
|
|
- name: Install ipex-llm and other related packages (install from pypi)
|
|
if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
|
|
shell: cmd
|
|
run: |
|
|
call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y
|
|
call conda activate igpu-perf
|
|
|
|
pip install --upgrade pip
|
|
pip install --upgrade wheel
|
|
pip install --upgrade omegaconf pandas
|
|
pip install --upgrade tiktoken einops transformers_stream_generator matplotlib
|
|
|
|
pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
|
|
pip show ipex-llm | findstr %TEST_VERSION_DATE%
|
|
if %ERRORLEVEL% neq 0 (
|
|
echo "Did not install ipex-llm with excepted version %TEST_VERSION_DATE%"
|
|
exit /b 1
|
|
)
|
|
pip list
|
|
|
|
call conda deactivate
|
|
|
|
- name: Create env for html generation
|
|
shell: cmd
|
|
run: |
|
|
call conda create -n html-gen python=3.11 -y
|
|
call conda activate html-gen
|
|
|
|
pip install pandas==1.5.3
|
|
pip install Jinja2
|
|
pip install "numpy<2.0.0"
|
|
|
|
call conda deactivate
|
|
|
|
- name: Set directory envs & and fix generated csv date name
|
|
shell: bash
|
|
run: |
|
|
if [[ ${{ github.event_name }} == "schedule" ]]; then
|
|
echo "CSV_SAVE_PATH=${CSV_NIGHTLY_PATH}" >> "$GITHUB_ENV"
|
|
elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
|
|
echo "CSV_SAVE_PATH=${CSV_NIGHTLY_PATH}" >> "$GITHUB_ENV"
|
|
else
|
|
echo "CSV_SAVE_PATH=${CSV_PR_PATH}" >> "$GITHUB_ENV"
|
|
fi
|
|
date_for_test_version=$(date -d yesterday +%Y-%m-%d)
|
|
echo "LOG_FILE=${date_for_test_version}_output.txt" >> "$GITHUB_ENV"
|
|
|
|
sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
|
|
|
|
- name: Add extra warmup for chatglm3-6b int4+fp32 & MiniCPM int4+fp16 int4+fp32 for more stable results
|
|
shell: bash
|
|
run: |
|
|
sed -i '/^\s*result = run_transformer_int4_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)/ i\
|
|
if repo_id in ["THUDM/chatglm3-6b", "THUDM/glm-4-9b-chat", "openbmb/MiniCPM-1B-sft-bf16", "openbmb/MiniCPM-2B-sft-bf16"]:\
|
|
run_transformer_int4_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)
|
|
' python/llm/dev/benchmark/all-in-one/run.py
|
|
|
|
sed -i '/^\s*result = run_transformer_int4_fp16_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)/ i\
|
|
if repo_id in ["openbmb/MiniCPM-1B-sft-bf16", "openbmb/MiniCPM-2B-sft-bf16"]:\
|
|
run_transformer_int4_fp16_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)
|
|
' python/llm/dev/benchmark/all-in-one/run.py
|
|
|
|
# 32-32 int4+fp16
|
|
- name: Prepare igpu perf test (32-32 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
# hide time info
|
|
# sed -i 's/str(end - st)/"xxxxxx"/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i 's/{api}-results-{today}.csv/32-32-{api}-results-{today}_test1.csv/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16.yaml
|
|
|
|
- name: Test on igpu (32-32 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
REM for llava
|
|
set TRANSFORMERS_OFFLINE=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Prepare igpu perf test for transformers 4.37 (32-32 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_437.yaml
|
|
|
|
- name: Test on igpu for transformers 4.37 (32-32 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.37.0
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_437.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Prepare igpu perf test for transformers 4.38 (32-32 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_438.yaml
|
|
|
|
- name: Test on igpu for transformers 4.38 (32-32 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.38.2
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_438.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Concat csv and generate html (32-32 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate html-gen
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
python ..\..\..\test\benchmark\concat_csv.py
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
del /q *test*.csv
|
|
move *.csv %CSV_SAVE_PATH%\32-32_int4_fp16\
|
|
cd ..\..\..\test\benchmark
|
|
python csv_to_html.py -f %CSV_SAVE_PATH%\32-32_int4_fp16\
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
move %CSV_SAVE_PATH%\32-32_int4_fp16\*.html %CSV_SAVE_PATH%
|
|
|
|
call conda deactivate
|
|
|
|
# TODO: create a action function here for different input
|
|
# 1024-128 int4+fp16
|
|
- name: Prepare igpu perf test (1024-128 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/32-32/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml
|
|
|
|
- name: Test on igpu (1024-128 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.36.2
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
REM for llava
|
|
set TRANSFORMERS_OFFLINE=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Prepare igpu perf test for transformers 4.37 (1024-128 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_437.yaml
|
|
|
|
- name: Test on igpu for transformers 4.37 (1024-128 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.37.0
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_437.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Prepare igpu perf test for transformers 4.38 (1024-128 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_438.yaml
|
|
|
|
- name: Test on igpu for transformers 4.38 (1024-128 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.38.2
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_438.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Concat csv and generate html (1024-128 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate html-gen
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
python ..\..\..\test\benchmark\concat_csv.py
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
del /q *test*.csv
|
|
move *.csv %CSV_SAVE_PATH%\1024-128_int4_fp16\
|
|
cd ..\..\..\test\benchmark
|
|
python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_int4_fp16\
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
move %CSV_SAVE_PATH%\1024-128_int4_fp16\*.html %CSV_SAVE_PATH%
|
|
|
|
call conda deactivate
|
|
|
|
# 2048-256 int4+fp16
|
|
- name: Prepare igpu perf test (2048-256 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/1024-128/2048-256/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16.yaml
|
|
|
|
- name: Test on igpu (2048-256 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.36.2
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
REM for llava
|
|
set TRANSFORMERS_OFFLINE=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Prepare igpu perf test for transformers 4.37 (2048-256 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_437.yaml
|
|
|
|
- name: Test on igpu for transformers 4.37 (2048-256 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.37.0
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_437.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Prepare igpu perf test for transformers 4.38 (2048-256 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_438.yaml
|
|
|
|
- name: Test on igpu for transformers 4.38 (2048-256 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.38.2
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_438.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Concat csv and generate html (2048-256 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate html-gen
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
python ..\..\..\test\benchmark\concat_csv.py
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
del /q *test*.csv
|
|
move *.csv %CSV_SAVE_PATH%\2048-256_int4_fp16\
|
|
cd ..\..\..\test\benchmark
|
|
python csv_to_html.py -f %CSV_SAVE_PATH%\2048-256_int4_fp16\
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
move %CSV_SAVE_PATH%\2048-256_int4_fp16\*.html %CSV_SAVE_PATH%
|
|
|
|
call conda deactivate
|
|
|
|
# 3072-384 int4+fp16
|
|
- name: Prepare igpu perf test (3072-384 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/2048-256/3072-384/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16.yaml
|
|
|
|
- name: Test on igpu (3072-384 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.36.2
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
REM for llava
|
|
set TRANSFORMERS_OFFLINE=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Prepare igpu perf test for transformers 4.37 (3072-384 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16_437.yaml
|
|
|
|
- name: Test on igpu for transformers 4.37 (3072-384 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.37.0
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_437.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Prepare igpu perf test for transformers 4.38 (3072-384 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16_438.yaml
|
|
|
|
- name: Test on igpu for transformers 4.38 (3072-384 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.38.2
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_438.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Concat csv and generate html (3072-384 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate html-gen
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
python ..\..\..\test\benchmark\concat_csv.py
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
del /q *test*.csv
|
|
move *.csv %CSV_SAVE_PATH%\3072-384_int4_fp16\
|
|
cd ..\..\..\test\benchmark
|
|
python csv_to_html.py -f %CSV_SAVE_PATH%\3072-384_int4_fp16\
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
move %CSV_SAVE_PATH%\3072-384_int4_fp16\*.html %CSV_SAVE_PATH%
|
|
|
|
call conda deactivate
|
|
|
|
# 4096-512 int4+fp16
|
|
- name: Prepare igpu perf test (4096-512 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/3072-384/4096-512/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16.yaml
|
|
|
|
- name: Test on igpu (4096-512 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.36.2
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
REM for llava
|
|
set TRANSFORMERS_OFFLINE=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Prepare igpu perf test for transformers 4.37 (4096-512 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16_437.yaml
|
|
|
|
- name: Test on igpu for transformers 4.37 (4096-512 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.37.0
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_437.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Prepare igpu perf test for transformers 4.38 (4096-512 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16_438.yaml
|
|
|
|
- name: Test on igpu for transformers 4.38 (4096-512 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.38.2
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_438.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Concat csv and generate html (4096-512 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate html-gen
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
python ..\..\..\test\benchmark\concat_csv.py
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
del /q *test*.csv
|
|
move *.csv %CSV_SAVE_PATH%\4096-512_int4_fp16\
|
|
cd ..\..\..\test\benchmark
|
|
python csv_to_html.py -f %CSV_SAVE_PATH%\4096-512_int4_fp16\
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
move %CSV_SAVE_PATH%\4096-512_int4_fp16\*.html %CSV_SAVE_PATH%
|
|
|
|
call conda deactivate
|
|
|
|
# load_low_bit 1024-128 int4+fp16
|
|
- name: Prepare igpu perf test (load_low_bit 1024-128 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/4096-512/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit.yaml
|
|
|
|
- name: Test on igpu (load_low_bit 1024-128 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.36.2
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
REM for llava
|
|
set TRANSFORMERS_OFFLINE=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Prepare igpu perf test for transformers 4.37 (load_low_bit 1024-128 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_437.yaml
|
|
|
|
- name: Test on igpu for transformers 4.37 (load_low_bit 1024-128 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.37.0
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_437.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Prepare igpu perf test for transformers 4.38 (load_low_bit 1024-128 int4+fp16)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_438.yaml
|
|
|
|
- name: Test on igpu for transformers 4.38 (load_low_bit 1024-128 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.38.2
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_438.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Concat csv and generate html (load_low_bit 1024-128 int4+fp16)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate html-gen
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
python ..\..\..\test\benchmark\concat_csv.py
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
del /q *test*.csv
|
|
move *.csv %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\
|
|
cd ..\..\..\test\benchmark
|
|
python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
move %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\*.html %CSV_SAVE_PATH%
|
|
|
|
call conda deactivate
|
|
|
|
# 1024-128
|
|
- name: Prepare igpu perf test (1024-128)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128.yaml
|
|
|
|
- name: Test on igpu (1024-128)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.36.2
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
REM for llava
|
|
set TRANSFORMERS_OFFLINE=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\1024-128.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Prepare igpu perf test for transformers 4.37 (1024-128)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_437.yaml
|
|
|
|
- name: Test on igpu for transformers 4.37 (1024-128)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.37.0
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\1024-128_437.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Prepare igpu perf test for transformers 4.38 (1024-128)
|
|
shell: bash
|
|
run: |
|
|
sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
|
|
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_438.yaml
|
|
|
|
- name: Test on igpu for transformers 4.38 (1024-128)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate igpu-perf
|
|
pip install transformers==4.38.2
|
|
|
|
set SYCL_CACHE_PERSISTENT=1
|
|
set BIGDL_LLM_XMX_DISABLED=1
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
move ..\..\..\test\benchmark\igpu-perf\1024-128_438.yaml config.yaml
|
|
set PYTHONIOENCODING=utf-8
|
|
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
|
|
call conda deactivate
|
|
|
|
- name: Concat csv and generate html (1024-128)
|
|
shell: cmd
|
|
run: |
|
|
call conda activate html-gen
|
|
|
|
cd python\llm\dev\benchmark\all-in-one
|
|
python ..\..\..\test\benchmark\concat_csv.py
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
del /q *test*.csv
|
|
move *.csv %CSV_SAVE_PATH%\1024-128\
|
|
cd ..\..\..\test\benchmark
|
|
python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128\
|
|
if %ERRORLEVEL% neq 0 (exit /b 1)
|
|
move %CSV_SAVE_PATH%\1024-128\*.html %CSV_SAVE_PATH%
|
|
|
|
call conda deactivate
|
|
|
|
# TODO: avoid duplicated code
|
|
- name: Upload results to ftp
|
|
if: ${{ always() }}
|
|
shell: cmd
|
|
run: |
|
|
cd %CSV_SAVE_PATH%
|
|
IF "${{ github.event_name }}"=="schedule" (
|
|
for %%f in (*.html) do (
|
|
curl -T "%%f" %FTP_IGPU_NIGHTLY_PERF_PATH%
|
|
)
|
|
)
|
|
IF "${{ github.event_name }}"=="workflow_dispatch" (
|
|
IF "${{ inputs.checkout-ref }}"=="main" (
|
|
for %%f in (*.html) do (
|
|
curl -T "%%f" %FTP_IGPU_NIGHTLY_PERF_PATH%
|
|
)
|
|
)
|
|
)
|
|
# for test on machine when encountering error
|
|
# - name: Remove conda env
|
|
# if: ${{ always() }}
|
|
# shell: cmd
|
|
# run: |
|
|
# call conda env remove -n igpu-perf -y
|
|
|