ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

split stable version tests (#10076)

Browse source 
Co-authored-by: Your Name <Your Email>
This commit is contained in:
Shaojun Liu 2024-02-04 11:08:12 +08:00 committed by GitHub
parent 428b7105f6
commit 698f84648c
2 changed files with 130 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

110
.github/workflows/llm_tests_for_stable_version.yml → .github/workflows/llm_tests_for_stable_version_on_arc.yml vendored
View file

@ -1,4 +1,4 @@
name: LLM Test for Stable Version name: ARC LLM Test for Stable Version
# Cancel previous runs in the PR when you push new commits # Cancel previous runs in the PR when you push new commits
concurrency: concurrency:
@ -184,111 +184,3 @@ jobs:
cd ../../../test/benchmark cd ../../../test/benchmark
python -m pip install pandas==1.5.3 python -m pip install pandas==1.5.3
python csv_to_html.py -f $CSV_SAVE_PATH/fp8 python csv_to_html.py -f $CSV_SAVE_PATH/fp8
llm-perf-regression-test-on-spr:
needs: llm-cpp-build
strategy:
fail-fast: false
matrix:
python-version: ["3.9"]
runs-on: [self-hosted, llm, spr01-perf]
env:
OMP_NUM_THREADS: 16
THREAD_NUM: 16
ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
shell: bash
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade wheel
python -m pip install --upgrade omegaconf
python -m pip install --upgrade pandas
python -m pip install --upgrade einops
python -m pip install --upgrade tiktoken
python -m pip install --upgrade transformers_stream_generator
- name: Download llm binary
uses: ./.github/actions/llm/download-llm-binary
- name: Run LLM install (all) test
uses: ./.github/actions/llm/setup-llm-env
- name: Test on cpu
shell: bash
run: |
mv python/llm/test/benchmark/stable-version-cpu-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
export http_proxy=${HTTP_PROXY}
export https_proxy=${HTTPS_PROXY}
source bigdl-llm-init -t
export OMP_NUM_THREADS=48
# hide time info
sed -i 's/str(end - st)/"xxxxxx"/g' run.py
python run.py
cp ./*.csv /models/stable_version_perf_regression_test_cpu/
cd ../../../test/benchmark
python -m pip install pandas==1.5.3
python csv_to_html.py -f /models/stable_version_perf_regression_test_cpu/ -b /models/stable_version_perf_regression_test_cpu/transformer_int4-results-1baseline.csv -t 5.0
llm-stress-test-on-spr:
needs: llm-perf-regression-test-on-spr
strategy:
fail-fast: false
matrix:
python-version: ["3.9"]
runs-on: [self-hosted, llm, spr01-perf]
env:
OMP_NUM_THREADS: 16
THREAD_NUM: 16
ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
shell: bash
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade wheel
python -m pip install --upgrade omegaconf
python -m pip install --upgrade pandas
python -m pip install --upgrade einops
python -m pip install --upgrade tiktoken
python -m pip install --upgrade transformers_stream_generator
- name: Download llm binary
uses: ./.github/actions/llm/download-llm-binary
- name: Run LLM install (all) test
uses: ./.github/actions/llm/setup-llm-env
- name: Test on cpu
shell: bash
run: |
mv python/llm/test/benchmark/stable-version-cpu-stress-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
export http_proxy=${HTTP_PROXY}
export https_proxy=${HTTPS_PROXY}
source bigdl-llm-init -t
export OMP_NUM_THREADS=48
# hide time info
sed -i 's/str(end - st)/"xxxxxx"/g' run-stress-test.py
python run-stress-test.py
cp ./*.csv /models/stable_version_stress_test_cpu/
cd ../../../test/benchmark
python -m pip install pandas==1.5.3
python csv_to_html.py -f /models/stable_version_stress_test_cpu/

129
.github/workflows/llm_tests_for_stable_version_on_spr.yml vendored Normal file
View file

@ -0,0 +1,129 @@
name: SPR LLM Test for Stable Version
# Cancel previous runs in the PR when you push new commits
concurrency:
group: ${{ github.workflow }}-llm-performance-tests-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
# Controls when the action will run.
on:
# pull_request:
# branches: [main]
# paths:
# - ".github/workflows/llm_performance_tests.yml"
# - "python/llm/test/benchmark/**"
# - "python/llm/dev/benchmark/all-in-one/**"
workflow_dispatch:
workflow_call:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
llm-cpp-build:
uses: ./.github/workflows/llm-binary-build.yml
llm-perf-regression-test-on-spr:
needs: llm-cpp-build
strategy:
fail-fast: false
matrix:
python-version: ["3.9"]
runs-on: [self-hosted, llm, spr01-perf]
env:
OMP_NUM_THREADS: 16
THREAD_NUM: 16
ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
shell: bash
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade wheel
python -m pip install --upgrade omegaconf
python -m pip install --upgrade pandas
python -m pip install --upgrade einops
python -m pip install --upgrade tiktoken
python -m pip install --upgrade transformers_stream_generator
- name: Download llm binary
uses: ./.github/actions/llm/download-llm-binary
- name: Run LLM install (all) test
uses: ./.github/actions/llm/setup-llm-env
- name: Test on cpu
shell: bash
run: |
mv python/llm/test/benchmark/stable-version-cpu-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
export http_proxy=${HTTP_PROXY}
export https_proxy=${HTTPS_PROXY}
source bigdl-llm-init -t
export OMP_NUM_THREADS=48
# hide time info
sed -i 's/str(end - st)/"xxxxxx"/g' run.py
python run.py
cp ./*.csv /models/stable_version_perf_regression_test_cpu/
cd ../../../test/benchmark
python -m pip install pandas==1.5.3
python csv_to_html.py -f /models/stable_version_perf_regression_test_cpu/ -b /models/stable_version_perf_regression_test_cpu/transformer_int4-results-1baseline.csv -t 5.0
llm-stress-test-on-spr:
needs: llm-perf-regression-test-on-spr
strategy:
fail-fast: false
matrix:
python-version: ["3.9"]
runs-on: [self-hosted, llm, spr01-perf]
env:
OMP_NUM_THREADS: 16
THREAD_NUM: 16
ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
shell: bash
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade wheel
python -m pip install --upgrade omegaconf
python -m pip install --upgrade pandas
python -m pip install --upgrade einops
python -m pip install --upgrade tiktoken
python -m pip install --upgrade transformers_stream_generator
- name: Download llm binary
uses: ./.github/actions/llm/download-llm-binary
- name: Run LLM install (all) test
uses: ./.github/actions/llm/setup-llm-env
- name: Test on cpu
shell: bash
run: |
mv python/llm/test/benchmark/stable-version-cpu-stress-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
export http_proxy=${HTTP_PROXY}
export https_proxy=${HTTPS_PROXY}
source bigdl-llm-init -t
export OMP_NUM_THREADS=48
# hide time info
sed -i 's/str(end - st)/"xxxxxx"/g' run-stress-test.py
python run-stress-test.py
cp ./*.csv /models/stable_version_stress_test_cpu/
cd ../../../test/benchmark
python -m pip install pandas==1.5.3
python csv_to_html.py -f /models/stable_version_stress_test_cpu/