ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Support pr validate perf test (#11486)

Browse source 
* Support triggering performance tests through commits

* Small fix

* Small fix

* Small fixes
This commit is contained in:
Yuwen Hu 2024-07-02 15:20:42 +08:00 committed by GitHub
parent 4390e7dc49
commit bb6953c19e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 94 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
.github/workflows/llm-binary-build.yml vendored
View file

@ -11,26 +11,26 @@ permissions:
# Controls when the action will run. # Controls when the action will run.
on: on:
# Triggers the workflow on push or pull request events but only for the main branch # Triggers the workflow on push or pull request events but only for the main branch
push: # push:
branches: [main] # branches: [main]
paths: # paths:
- ".github/workflows/llm-binary-build.yml" # - ".github/workflows/llm-binary-build.yml"
pull_request: # pull_request:
branches: [main] # branches: [main]
paths: # paths:
- ".github/workflows/llm-binary-build.yml" # - ".github/workflows/llm-binary-build.yml"
workflow_dispatch: # workflow_dispatch:
inputs: # inputs:
llmcpp-ref: # llmcpp-ref:
description: 'Ref of llm.cpp code' # description: 'Ref of llm.cpp code'
default: '' # default: ''
required: false # required: false
type: string # type: string
platform: # platform:
description: 'Platforms to built on' # description: 'Platforms to built on'
default: '["Windows", "Linux"]' # default: '["Windows", "Linux"]'
required: false # required: false
type: string # type: string
workflow_call: workflow_call:
inputs: inputs:
llmcpp-ref: llmcpp-ref:

131
.github/workflows/llm_performance_tests.yml vendored
View file

@ -70,12 +70,13 @@ on:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel # A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs: jobs:
# llm-cpp-build: # please uncomment it for PR tests llm-cpp-build:
# uses: ./.github/workflows/llm-binary-build.yml uses: ./.github/workflows/llm-binary-build.yml
with: ${{ (github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref != 'main')) && 'Windows,Linux' || 'Dummy' }}
llm-performance-test-on-arc: llm-performance-test-on-arc:
if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && (inputs.arc == 'true')) || github.event.inputs.artifact == 'llm-performance-test-on-arc' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && (github.event.inputs.arc == 'true')) || github.event.inputs.artifact == 'llm-performance-test-on-arc' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests
# needs: llm-cpp-build # please uncomment it for PR tests needs: llm-cpp-build
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
@ -85,7 +86,7 @@ jobs:
OMP_NUM_THREADS: 16 OMP_NUM_THREADS: 16
THREAD_NUM: 16 THREAD_NUM: 16
ANALYTICS_ZOO_ROOT: ${{ github.workspace }} ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
CSV_SAVE_PATH: ${{ (github.event.schedule || github.event_name == 'workflow_dispatch') && '/mnt/disk1/nightly_perf_gpu/' || '/mnt/disk1/pr_perf_gpu/' }} CSV_SAVE_PATH: ${{ (github.event.schedule || (github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref == 'main'))) && '/mnt/disk1/nightly_perf_gpu/' || '/mnt/disk1/pr_perf_gpu/' }}
steps: steps:
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3 - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
@ -111,16 +112,19 @@ jobs:
python -m pip install --upgrade transformers_stream_generator python -m pip install --upgrade transformers_stream_generator
python -m pip install --upgrade tiktoken python -m pip install --upgrade tiktoken
# please uncomment it and comment the "Install IPEX-LLM from Pypi" part for PR tests # specific for test on certain commits
# - name: Download llm binary - name: Download llm binary
# uses: ./.github/actions/llm/download-llm-binary if: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref != 'main') }}
uses: ./.github/actions/llm/download-llm-binary
# - name: Run LLM install (all) test - name: Install IPEX-LLM from source
# uses: ./.github/actions/llm/setup-llm-env if: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref != 'main') }}
# with: uses: ./.github/actions/llm/setup-llm-env
# extra-dependency: "xpu_2.1" with:
extra-dependency: "xpu_2.1"
- name: Install IPEX-LLM from Pypi - name: Install IPEX-LLM from Pypi
if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref == 'main')) }}
shell: bash shell: bash
run: | run: |
pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/ pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
@ -339,8 +343,8 @@ jobs:
llm-performance-test-on-spr: llm-performance-test-on-spr:
if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && (inputs.spr == 'true')) || github.event.inputs.artifact == 'llm-performance-test-on-spr' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && (github.event.inputs.spr == 'true')) || github.event.inputs.artifact == 'llm-performance-test-on-spr' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests
# needs: llm-cpp-build # please uncomment it for PR tests needs: llm-cpp-build
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
@ -372,14 +376,17 @@ jobs:
python -m pip install --upgrade tiktoken python -m pip install --upgrade tiktoken
python -m pip install --upgrade transformers_stream_generator python -m pip install --upgrade transformers_stream_generator
# please uncomment it and comment the "Install IPEX-LLM from Pypi" part for PR tests # specific for test on certain commits
# - name: Download llm binary - name: Download llm binary
# uses: ./.github/actions/llm/download-llm-binary if: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref != 'main') }}
uses: ./.github/actions/llm/download-llm-binary
# - name: Run LLM install (all) test - name: Install IPEX-LLM from source
# uses: ./.github/actions/llm/setup-llm-env if: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref != 'main') }}
uses: ./.github/actions/llm/setup-llm-env
- name: Install IPEX-LLM from Pypi - name: Install IPEX-LLM from Pypi
if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref == 'main')) }}
shell: bash shell: bash
run: | run: |
pip install --pre --upgrade ipex-llm[all] --extra-index-url https://download.pytorch.org/whl/cpu pip install --pre --upgrade ipex-llm[all] --extra-index-url https://download.pytorch.org/whl/cpu
@ -414,8 +421,8 @@ jobs:
done done
llm-performance-test-on-core: llm-performance-test-on-core:
if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && (inputs.core== 'true')) || github.event.inputs.artifact == 'llm-performance-test-on-core' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && (github.event.inputs.core == 'true')) || github.event.inputs.artifact == 'llm-performance-test-on-core' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests
# needs: llm-cpp-build # please uncomment it for PR tests needs: llm-cpp-build
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
@ -429,7 +436,7 @@ jobs:
runs-on: [self-hosted, "${{ matrix.os }}", llm, perf-core, "${{ matrix.platform }}"] runs-on: [self-hosted, "${{ matrix.os }}", llm, perf-core, "${{ matrix.platform }}"]
env: env:
ANALYTICS_ZOO_ROOT: ${{ github.workspace }} ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
CSV_SAVE_PATH: ${{ (github.event.schedule || github.event_name == 'workflow_dispatch') && 'D:/action-runners/nightly_perf_core_' || 'D:/action-runners/pr_perf_core_' }}${{ matrix.platform }}/ CSV_SAVE_PATH: ${{ (github.event.schedule || (github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref == 'main'))) && 'D:/action-runners/nightly_perf_core_' || 'D:/action-runners/pr_perf_core_' }}${{ matrix.platform }}/
steps: steps:
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3 - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
with: with:
@ -449,14 +456,17 @@ jobs:
python -m pip install --upgrade omegaconf pandas python -m pip install --upgrade omegaconf pandas
python -m pip install --upgrade tiktoken einops transformers_stream_generator python -m pip install --upgrade tiktoken einops transformers_stream_generator
# please uncomment it and comment the "Install IPEX-LLM from Pypi" part for PR tests # specific for test on certain commits
# - name: Download llm binary - name: Download llm binary
# uses: ./.github/actions/llm/download-llm-binary if: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref != 'main') }}
uses: ./.github/actions/llm/download-llm-binary
# - name: Run LLM install (all) test - name: Install IPEX-LLM from source
# uses: ./.github/actions/llm/setup-llm-env if: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref != 'main') }}
uses: ./.github/actions/llm/setup-llm-env
- name: Install IPEX-LLM from Pypi - name: Install IPEX-LLM from Pypi
if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref == 'main')) }}
shell: bash shell: bash
run: | run: |
pip install --pre --upgrade ipex-llm[all] pip install --pre --upgrade ipex-llm[all]
@ -490,8 +500,8 @@ jobs:
fi fi
llm-performance-test-on-igpu: llm-performance-test-on-igpu:
if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && (inputs.igpu== 'true')) || github.event.inputs.artifact == 'llm-performance-test-on-igpu' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && (github.event.inputs.igpu == 'true')) || github.event.inputs.artifact == 'llm-performance-test-on-igpu' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests
# needs: llm-cpp-build # please uncomment it for PR tests needs: llm-cpp-build
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
@ -509,47 +519,52 @@ jobs:
# TODO: Put the ipex-llm related install process for win gpu into a action function # TODO: Put the ipex-llm related install process for win gpu into a action function
# Please uncomment it and commment the install from pypi for PR tests # specific for test on certain commits
# - name: Download llm binary - name: Download llm binary
# uses: ./.github/actions/llm/download-llm-binary if: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref != 'main') }}
uses: ./.github/actions/llm/download-llm-binary
# - name: Prepare for install ipex-llm from source - name: Prepare for install ipex-llm from source
# shell: bash if: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref != 'main') }}
# run: | shell: bash
# sed -i 's/"bigdl-core-xe-21==" + CORE_XE_VERSION/"bigdl-core-xe-21"/g' python/llm/setup.py run: |
# sed -i 's/"bigdl-core-xe-batch-21==" + CORE_XE_VERSION/"bigdl-core-xe-batch-21"/g' python/llm/setup.py sed -i 's/"bigdl-core-xe-21==" + CORE_XE_VERSION/"bigdl-core-xe-21"/g' python/llm/setup.py
# sed -i 's/"bigdl-core-xe-addons-21==" + CORE_XE_VERSION/"bigdl-core-xe-addons-21"/g' python/llm/setup.py sed -i 's/"bigdl-core-xe-batch-21==" + CORE_XE_VERSION/"bigdl-core-xe-batch-21"/g' python/llm/setup.py
# sed -i 's/"bigdl-core-xe-esimd-21==" + CORE_XE_VERSION/"bigdl-core-xe-esimd-21"/g' python/llm/setup.py sed -i 's/"bigdl-core-xe-addons-21==" + CORE_XE_VERSION/"bigdl-core-xe-addons-21"/g' python/llm/setup.py
sed -i 's/"bigdl-core-xe-esimd-21==" + CORE_XE_VERSION/"bigdl-core-xe-esimd-21"/g' python/llm/setup.py
# - name: Install ipex-llm and other related packages (install from source) - name: Install ipex-llm and other related packages (install from source)
# shell: cmd if: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref != 'main') }}
# run: | shell: cmd
# call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y run: |
# call conda activate igpu-perf call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y
call conda activate igpu-perf
# pip install --upgrade pip pip install --upgrade pip
# pip install --upgrade wheel pip install --upgrade wheel
# pip install --upgrade omegaconf pandas pip install --upgrade omegaconf pandas
# pip install --upgrade tiktoken einops transformers_stream_generator pip install --upgrade tiktoken einops transformers_stream_generator
# cd python\llm cd python\llm
# python setup.py clean --all bdist_wheel --win python setup.py clean --all bdist_wheel --win
# if not exist dist\ipex_llm*.whl (exit /b 1) if not exist dist\ipex_llm*.whl (exit /b 1)
# for %%i in (dist\ipex_llm*.whl) do set whl_name=%%i for %%i in (dist\ipex_llm*.whl) do set whl_name=%%i
# pip install --pre --upgrade %whl_name%[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/ pip install --pre --upgrade %whl_name%[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
# if %ERRORLEVEL% neq 0 (exit /b 1) if %ERRORLEVEL% neq 0 (exit /b 1)
# pip list pip list
# call conda deactivate call conda deactivate
- name: Determine desired ipex-llm version - name: Determine desired ipex-llm version
if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref == 'main')) }}
shell: bash shell: bash
run: | run: |
test_version_date=`date -d 'yesterday' '+%Y%m%d'` test_version_date=`date -d 'yesterday' '+%Y%m%d'`
echo "TEST_VERSION_DATE=${test_version_date}" >> "$GITHUB_ENV" echo "TEST_VERSION_DATE=${test_version_date}" >> "$GITHUB_ENV"
- name: Install ipex-llm and other related packages (install from pypi) - name: Install ipex-llm and other related packages (install from pypi)
if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (github.event.inputs.checkout-ref == 'main')) }}
shell: cmd shell: cmd
run: | run: |
call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y
@ -585,7 +600,9 @@ jobs:
- name: Set directory envs & and fix generated csv date name - name: Set directory envs & and fix generated csv date name
shell: bash shell: bash
run: | run: |
if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then if [[ ${{ github.event_name }} == "schedule" ]]; then
echo "CSV_SAVE_PATH=${CSV_NIGHTLY_PATH}" >> "$GITHUB_ENV"
elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ github.event.inputs.checkout-ref }} == "main" ]]; then
echo "CSV_SAVE_PATH=${CSV_NIGHTLY_PATH}" >> "$GITHUB_ENV" echo "CSV_SAVE_PATH=${CSV_NIGHTLY_PATH}" >> "$GITHUB_ENV"
else else
echo "CSV_SAVE_PATH=${CSV_PR_PATH}" >> "$GITHUB_ENV" echo "CSV_SAVE_PATH=${CSV_PR_PATH}" >> "$GITHUB_ENV"