ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1
ipex-llm/.github/workflows/llm_performance_tests.yml
Cheen Hau, 俊豪 1c5eb14128
Update pip install to use --extra-index-url for ipex package (#10557) 
* Change to 'pip install .. --extra-index-url' for readthedocs

* Change to 'pip install .. --extra-index-url' for examples

* Change to 'pip install .. --extra-index-url' for remaining files

* Fix URL for ipex

* Add links for ipex US and CN servers

* Update ipex cpu url

* remove readme

* Update for github actions

* Update for dockerfiles
2024-03-28 09:56:23 +08:00

882 lines
34 KiB
YAML
Raw Blame History

name: LLM Performance Test
# Cancel previous runs in the PR when you push new commits
concurrency:
group: ${{ github.workflow }}-llm-performance-tests-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
# Controls when the action will run.
on:
schedule:
- cron: "30 16 * * *" # GMT time, 16:30 GMT == 00:30 China
# please uncomment it for PR tests
# pull_request:
# branches: [main]
# paths:
# - ".github/workflows/llm_performance_tests.yml"
# - "python/llm/test/benchmark/**"
# - "python/llm/dev/benchmark/all-in-one/**"
workflow_dispatch:
workflow_call:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# llm-cpp-build: # please uncomment it for PR tests
# uses: ./.github/workflows/llm-binary-build.yml
llm-performance-test-on-arc:
if: ${{ github.event.schedule || github.event.inputs.artifact == 'llm-performance-test-on-arc' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests
# needs: llm-cpp-build # please uncomment it for PR tests
strategy:
fail-fast: false
matrix:
python-version: ["3.9"]
runs-on: [self-hosted, llm, perf]
env:
OMP_NUM_THREADS: 16
THREAD_NUM: 16
ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
CSV_SAVE_PATH: ${{ github.event.schedule && '/mnt/disk1/nightly_perf_gpu/' || '/mnt/disk1/pr_perf_gpu/' }}
steps:
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
shell: bash
# pip install transformers_stream_generator for model internlm-chat-7b-8k
# pip install tiktoken for model Qwen-7B-Chat-10-12
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade wheel
python -m pip install --upgrade omegaconf
python -m pip install --upgrade pandas
python -m pip install --upgrade einops
python -m pip install --upgrade transformers_stream_generator
python -m pip install --upgrade tiktoken
# please uncomment it and comment the "Install IPEX-LLM from Pypi" part for PR tests
# - name: Download llm binary
# uses: ./.github/actions/llm/download-llm-binary
# - name: Run LLM install (all) test
# uses: ./.github/actions/llm/setup-llm-env
# with:
# extra-dependency: "xpu_2.1"
- name: Install IPEX-LLM from Pypi
shell: bash
run: |
pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
test_version_date=`date -d 'yesterday' '+%Y%m%d'`
if ! pip show ipex-llm | grep $test_version_date; then
echo "Did not install ipex-llm with excepted version $test_version_date"
exit 1
fi
- name: Test installed xpu version
shell: bash
run: |
source /opt/intel/oneapi/setvars.sh
bash python/llm/test/run-llm-install-tests.sh
- name: Test on xpu(transformers==4.31.0)
shell: bash
run: |
date_for_test_version=$(date -d yesterday +%Y-%m-%d)
sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
source /opt/intel/oneapi/setvars.sh
export USE_XETLA=OFF
export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
cp python/llm/test/benchmark/arc-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
# hide time info
sed -i 's/str(end - st)/"xxxxxx"/g' run.py
# change csv name
sed -i 's/{today}/{today}_test1/g' run.py
python run.py
- name: Test on xpu(transformers==4.34.0)
shell: bash
run: |
source /opt/intel/oneapi/setvars.sh
export USE_XETLA=OFF
export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
# upgrade transformers for model Mistral-7B-v0.1
python -m pip install transformers==4.34.0
cp python/llm/test/benchmark/arc-perf-transformers-434.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
# change csv name
sed -i 's/test1/test2/g' run.py
python run.py
- name: Test on xpu(transformers==4.37.0)
shell: bash
run: |
source /opt/intel/oneapi/setvars.sh
export USE_XETLA=OFF
export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
# upgrade transformers for model Qwen/Qwen1.5-7B-Chat
python -m pip install transformers==4.37.0
cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
# change csv name
sed -i 's/test2/test3/g' run.py
python run.py
- name: Concat csv and generate html
shell: bash
run: |
cd python/llm/dev/benchmark/all-in-one
python ../../../test/benchmark/concat_csv.py
for file in *.csv; do
if [[ $file != *test* ]]; then
cp "$file" $CSV_SAVE_PATH
fi
done
python -m pip install pandas==1.5.3
cd ../../../test/benchmark
python csv_to_html.py -f $CSV_SAVE_PATH
- name: Check and upload results to ftp
shell: bash
run: |
cd python/llm/dev/benchmark/all-in-one
python ../../../test/benchmark/check_results.py -c test1 -y ../../../test/benchmark/arc-perf-test.yaml
python ../../../test/benchmark/check_results.py -c test2 -y ../../../test/benchmark/arc-perf-transformers-434.yaml
find . -name "*test*.csv" -delete
if [ ${{ github.event.schedule}} ]; then
curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
fi
llm-performance-test-on-spr:
if: ${{ github.event.schedule || github.event.inputs.artifact == 'llm-performance-test-on-spr' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests
# needs: llm-cpp-build # please uncomment it for PR tests
strategy:
fail-fast: false
matrix:
python-version: ["3.9"]
runs-on: [self-hosted, llm, spr01-perf]
env:
OMP_NUM_THREADS: 16
THREAD_NUM: 16
ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
steps:
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
shell: bash
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade wheel
python -m pip install --upgrade omegaconf
python -m pip install --upgrade pandas
python -m pip install --upgrade einops
python -m pip install --upgrade tiktoken
python -m pip install --upgrade transformers_stream_generator
# please uncomment it and comment the "Install IPEX-LLM from Pypi" part for PR tests
# - name: Download llm binary
# uses: ./.github/actions/llm/download-llm-binary
# - name: Run LLM install (all) test
# uses: ./.github/actions/llm/setup-llm-env
- name: Install IPEX-LLM from Pypi
shell: bash
run: |
pip install --pre --upgrade ipex-llm[all] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
test_version_date=`date -d 'yesterday' '+%Y%m%d'`
if ! pip show ipex-llm | grep $test_version_date; then
echo "Did not install ipex-llm with excepted version $test_version_date"
exit 1
fi
- name: Test on cpu
shell: bash
run: |
date_for_test_version=$(date -d yesterday +%Y-%m-%d)
sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
mv python/llm/test/benchmark/cpu-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
export http_proxy=${HTTP_PROXY}
export https_proxy=${HTTPS_PROXY}
source ipex-llm-init -t
export OMP_NUM_THREADS=48
# hide time info
sed -i 's/str(end - st)/"xxxxxx"/g' run.py
python run.py
cp ./*.csv /models/nightly_perf_cpu
cd ../../../test/benchmark
python -m pip install pandas==1.5.3
python csv_to_html.py -f /models/nightly_perf_cpu
cd /models/nightly_perf_cpu
for f in *.html; do
curl -T "$f" ${LLM_FTP_URL}/llm/nightly_perf/nightly_perf_cpu/
done
llm-performance-test-on-core:
if: ${{ github.event.schedule || github.event.inputs.artifact == 'llm-performance-test-on-core' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests
# needs: llm-cpp-build # please uncomment it for PR tests
strategy:
fail-fast: false
matrix:
include:
- os: windows
platform: dp
python-version: "3.9"
# - os: windows
# platform: lp
# python-version: "3.9"
runs-on: [self-hosted, "${{ matrix.os }}", llm, perf-core, "${{ matrix.platform }}"]
env:
ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
CSV_SAVE_PATH: ${{ github.event.schedule && 'D:/action-runners/nightly_perf_core_' || 'D:/action-runners/pr_perf_core_' }}${{ matrix.platform }}/
steps:
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
shell: bash
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade wheel
python -m pip install --upgrade omegaconf pandas
python -m pip install --upgrade tiktoken einops transformers_stream_generator
# please uncomment it and comment the "Install IPEX-LLM from Pypi" part for PR tests
# - name: Download llm binary
# uses: ./.github/actions/llm/download-llm-binary
# - name: Run LLM install (all) test
# uses: ./.github/actions/llm/setup-llm-env
- name: Install IPEX-LLM from Pypi
shell: bash
run: |
pip install --pre --upgrade ipex-llm[all] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
test_version_date=`date -d 'yesterday' '+%Y%m%d'`
if ! pip show ipex-llm | grep $test_version_date; then
echo "Did not install ipex-llm with excepted version $test_version_date"
exit 1
fi
- name: Test on core ${{ matrix.platform }}
shell: bash
run: |
date_for_test_version=$(date -d yesterday +%Y-%m-%d)
sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
mv python/llm/test/benchmark/core-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
export http_proxy=${HTTP_PROXY}
export https_proxy=${HTTPS_PROXY}
# hide time info
sed -i 's/str(end - st)/"xxxxxx"/g' run.py
python run.py
cp ./*.csv $CSV_SAVE_PATH
cd ../../../test/benchmark
python -m pip install pandas==1.5.3
python csv_to_html.py -f $CSV_SAVE_PATH
cd ../../dev/benchmark/all-in-one/
if [ ${{ github.event.schedule}} ]; then
curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/core_${{ matrix.platform }}/
fi
llm-performance-test-on-igpu:
if: ${{ github.event.schedule || github.event.inputs.artifact == 'llm-performance-test-on-igpu' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests
# needs: llm-cpp-build # please uncomment it for PR tests
strategy:
fail-fast: false
matrix:
include:
- os: windows
python-version: "3.9"
runs-on: [self-hosted, "${{ matrix.os }}", llm, perf-igpu]
env:
ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
steps:
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
# TODO: Put the ipex-llm related install process for win gpu into a action function
# Please uncomment it and commment the install from pypi for PR tests
# - name: Download llm binary
# uses: ./.github/actions/llm/download-llm-binary
# - name: Prepare for install ipex-llm from source
# shell: bash
# run: |
# sed -i 's/"bigdl-core-xe-21==" + VERSION + "/"bigdl-core-xe-21/g' python/llm/setup.py
# sed -i 's/"bigdl-core-xe-21==" + VERSION/"bigdl-core-xe-21"/g' python/llm/setup.py
# - name: Install ipex-llm and other related packages (install from source)
# shell: cmd
# run: |
# call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y
# call conda activate igpu-perf
# pip install --upgrade pip
# pip install --upgrade wheel
# pip install --upgrade omegaconf pandas
# pip install --upgrade tiktoken einops transformers_stream_generator
# cd python\llm
# python setup.py clean --all bdist_wheel --win
# if not exist dist\ipex_llm*.whl (exit /b 1)
# for %%i in (dist\ipex_llm*.whl) do set whl_name=%%i
# pip install --pre --upgrade %whl_name%[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
# if %ERRORLEVEL% neq 0 (exit /b 1)
# pip list
# call conda deactivate
- name: Determine desired ipex-llm version
shell: bash
run: |
test_version_date=`date -d 'yesterday' '+%Y%m%d'`
echo "TEST_VERSION_DATE=${test_version_date}" >> "$GITHUB_ENV"
- name: Install ipex-llm and other related packages (install from pypi)
shell: cmd
run: |
call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y
call conda activate igpu-perf
pip install --upgrade pip
pip install --upgrade wheel
pip install --upgrade omegaconf pandas
pip install --upgrade tiktoken einops transformers_stream_generator
pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
pip show ipex-llm | findstr %TEST_VERSION_DATE%
if %ERRORLEVEL% neq 0 (
echo "Did not install ipex-llm with excepted version %TEST_VERSION_DATE%"
exit /b 1
)
pip list
call conda deactivate
- name: Create env for html generation
shell: cmd
run: |
call conda create -n html-gen python=3.9 -y
call conda activate html-gen
pip install pandas==1.5.3
pip install Jinja2
call conda deactivate
- name: Set directory envs & and fix generated csv date name
shell: bash
run: |
if [ ${{ github.event_name }} == 'schedule' ]; then
echo "CSV_SAVE_PATH=${CSV_NIGHTLY_PATH}" >> "$GITHUB_ENV"
else
echo "CSV_SAVE_PATH=${CSV_PR_PATH}" >> "$GITHUB_ENV"
fi
date_for_test_version=$(date -d yesterday +%Y-%m-%d)
echo "LOG_FILE=${date_for_test_version}_output.txt" >> "$GITHUB_ENV"
sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
- name: Prepare igpu perf test (32-32)
shell: bash
run: |
# hide time info
# sed -i 's/str(end - st)/"xxxxxx"/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i 's/{api}-results-{today}.csv/32-32-{api}-results-{today}_test1.csv/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32.yaml
- name: Test on igpu (32-32)
shell: cmd
run: |
call conda activate igpu-perf
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
REM for llava
set TRANSFORMERS_OFFLINE=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\32-32.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for Mistral (32-32)
shell: bash
run: |
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_434.yaml
- name: Test on igpu for Mistral (32-32)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.34.0
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\32-32_434.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for Qwen1.5 (32-32)
shell: bash
run: |
sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_437.yaml
- name: Test on igpu for Qwen1.5 (32-32)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.37.0
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\32-32_437.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Concat csv and generate html (32-32)
shell: cmd
run: |
call conda activate html-gen
cd python\llm\dev\benchmark\all-in-one
python ..\..\..\test\benchmark\concat_csv.py
if %ERRORLEVEL% neq 0 (exit /b 1)
del /q *test*.csv
move *.csv %CSV_SAVE_PATH%\32-32\
cd ..\..\..\test\benchmark
python csv_to_html.py -f %CSV_SAVE_PATH%\32-32\
if %ERRORLEVEL% neq 0 (exit /b 1)
move %CSV_SAVE_PATH%\32-32\*.html %CSV_SAVE_PATH%
call conda deactivate
# TODO: create a action function here for different input
# 1024-128
- name: Prepare igpu perf test (1024-128)
shell: bash
run: |
sed -i 's/32-32/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128.yaml
- name: Test on igpu (1024-128)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.31.0
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
REM for llava
set TRANSFORMERS_OFFLINE=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for Mistral (1024-128)
shell: bash
run: |
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_434.yaml
- name: Test on igpu for Mistral (1024-128)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.34.0
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_434.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for Qwen 1.5 (1024-128)
shell: bash
run: |
sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_437.yaml
- name: Test on igpu for Qwen 1.5 (1024-128)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.37.0
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_437.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Concat csv and generate html (1024-128)
shell: cmd
run: |
call conda activate html-gen
cd python\llm\dev\benchmark\all-in-one
python ..\..\..\test\benchmark\concat_csv.py
if %ERRORLEVEL% neq 0 (exit /b 1)
del /q *test*.csv
move *.csv %CSV_SAVE_PATH%\1024-128\
cd ..\..\..\test\benchmark
python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128\
if %ERRORLEVEL% neq 0 (exit /b 1)
move %CSV_SAVE_PATH%\1024-128\*.html %CSV_SAVE_PATH%
call conda deactivate
# 2048-256
- name: Prepare igpu perf test (2048-256)
shell: bash
run: |
sed -i 's/1024-128/2048-256/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256.yaml
- name: Test on igpu (2048-256)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.31.0
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
REM for llava
set TRANSFORMERS_OFFLINE=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\2048-256.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for Mistral (2048-256)
shell: bash
run: |
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_434.yaml
- name: Test on igpu for Mistral (2048-256)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.34.0
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\2048-256_434.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for Qwen 1.5 (2048-256)
shell: bash
run: |
sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_437.yaml
- name: Test on igpu for Qwen 1.5 (2048-256)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.37.0
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\2048-256_437.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Concat csv and generate html (2048-256)
shell: cmd
run: |
call conda activate html-gen
cd python\llm\dev\benchmark\all-in-one
python ..\..\..\test\benchmark\concat_csv.py
if %ERRORLEVEL% neq 0 (exit /b 1)
del /q *test*.csv
move *.csv %CSV_SAVE_PATH%\2048-256\
cd ..\..\..\test\benchmark
python csv_to_html.py -f %CSV_SAVE_PATH%\2048-256\
if %ERRORLEVEL% neq 0 (exit /b 1)
move %CSV_SAVE_PATH%\2048-256\*.html %CSV_SAVE_PATH%
call conda deactivate
# load_low_bit 1024-128
- name: Prepare igpu perf test (load_low_bit 1024-128)
shell: bash
run: |
sed -i 's/2048-256/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit.yaml
- name: Test on igpu (load_low_bit 1024-128)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.31.0
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
REM for llava
set TRANSFORMERS_OFFLINE=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for Mistral (load_low_bit 1024-128)
shell: bash
run: |
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_434.yaml
- name: Test on igpu for Mistral (load_low_bit 1024-128)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.34.0
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit_434.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for Qwen 1.5 (load_low_bit 1024-128)
shell: bash
run: |
sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_437.yaml
- name: Test on igpu for Qwen 1.5 (load_low_bit 1024-128)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.37.0
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit_437.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Concat csv and generate html (load_low_bit 1024-128)
shell: cmd
run: |
call conda activate html-gen
cd python\llm\dev\benchmark\all-in-one
python ..\..\..\test\benchmark\concat_csv.py
if %ERRORLEVEL% neq 0 (exit /b 1)
del /q *test*.csv
move *.csv %CSV_SAVE_PATH%\1024-128_loadlowbit\
cd ..\..\..\test\benchmark
python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_loadlowbit\
if %ERRORLEVEL% neq 0 (exit /b 1)
move %CSV_SAVE_PATH%\1024-128_loadlowbit\*.html %CSV_SAVE_PATH%
call conda deactivate
- name: Prepare igpu perf test (int4+fp16 1024-128)
shell: bash
run: |
sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml
- name: Test on igpu (int4+fp16 1024-128)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.31.0
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
REM for llava
set TRANSFORMERS_OFFLINE=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for Mistral (int4+fp16 1024-128)
shell: bash
run: |
sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_434.yaml
- name: Test on igpu for Mistral (int4+fp16 1024-128)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.34.0
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_434.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Prepare igpu perf test for Qwen 1.5 (int4+fp16 1024-128)
shell: bash
run: |
sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_437.yaml
- name: Test on igpu for Qwen 1.5 (int4+fp16 1024-128)
shell: cmd
run: |
call conda activate igpu-perf
pip install transformers==4.37.0
call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
set SYCL_CACHE_PERSISTENT=1
set BIGDL_LLM_XMX_DISABLED=1
cd python\llm\dev\benchmark\all-in-one
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_437.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
call conda deactivate
- name: Concat csv and generate html (int4+fp16 1024-128)
shell: cmd
run: |
call conda activate html-gen
cd python\llm\dev\benchmark\all-in-one
python ..\..\..\test\benchmark\concat_csv.py
if %ERRORLEVEL% neq 0 (exit /b 1)
del /q *test*.csv
move *.csv %CSV_SAVE_PATH%\1024-128_int4_fp16\
cd ..\..\..\test\benchmark
python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_int4_fp16\
if %ERRORLEVEL% neq 0 (exit /b 1)
move %CSV_SAVE_PATH%\1024-128_int4_fp16\*.html %CSV_SAVE_PATH%
call conda deactivate
- name: Upload results to ftp
if: ${{ always() }}
shell: cmd
run: |
cd %CSV_SAVE_PATH%
IF "${{ github.event_name }}"=="schedule" (
for %%f in (*.html) do (
curl -T "%%f" %FTP_IGPU_NIGHTLY_PERF_PATH%
)
)
# for test on machine when encountering error
# - name: Remove conda env
# if: ${{ always() }}
# shell: cmd
# run: |
# call conda env remove -n igpu-perf -y
Reference in a new issue View git blame Copy permalink