diff --git a/.github/workflows/llm_tests_for_stable_version.yml b/.github/workflows/llm_tests_for_stable_version_on_arc.yml similarity index 63% rename from .github/workflows/llm_tests_for_stable_version.yml rename to .github/workflows/llm_tests_for_stable_version_on_arc.yml index a72e576c..e8e716a7 100644 --- a/.github/workflows/llm_tests_for_stable_version.yml +++ b/.github/workflows/llm_tests_for_stable_version_on_arc.yml @@ -1,4 +1,4 @@ -name: LLM Test for Stable Version +name: ARC LLM Test for Stable Version # Cancel previous runs in the PR when you push new commits concurrency: @@ -184,111 +184,3 @@ jobs: cd ../../../test/benchmark python -m pip install pandas==1.5.3 python csv_to_html.py -f $CSV_SAVE_PATH/fp8 - - - llm-perf-regression-test-on-spr: - needs: llm-cpp-build - strategy: - fail-fast: false - matrix: - python-version: ["3.9"] - runs-on: [self-hosted, llm, spr01-perf] - env: - OMP_NUM_THREADS: 16 - THREAD_NUM: 16 - ANALYTICS_ZOO_ROOT: ${{ github.workspace }} - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - shell: bash - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade wheel - python -m pip install --upgrade omegaconf - python -m pip install --upgrade pandas - python -m pip install --upgrade einops - python -m pip install --upgrade tiktoken - python -m pip install --upgrade transformers_stream_generator - - - name: Download llm binary - uses: ./.github/actions/llm/download-llm-binary - - - name: Run LLM install (all) test - uses: ./.github/actions/llm/setup-llm-env - - - name: Test on cpu - shell: bash - run: | - mv python/llm/test/benchmark/stable-version-cpu-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml - cd python/llm/dev/benchmark/all-in-one - export http_proxy=${HTTP_PROXY} - export https_proxy=${HTTPS_PROXY} - source bigdl-llm-init -t - export OMP_NUM_THREADS=48 - # hide time info - sed -i 's/str(end - st)/"xxxxxx"/g' run.py - python run.py - cp ./*.csv /models/stable_version_perf_regression_test_cpu/ - cd ../../../test/benchmark - python -m pip install pandas==1.5.3 - python csv_to_html.py -f /models/stable_version_perf_regression_test_cpu/ -b /models/stable_version_perf_regression_test_cpu/transformer_int4-results-1baseline.csv -t 5.0 - - - llm-stress-test-on-spr: - needs: llm-perf-regression-test-on-spr - strategy: - fail-fast: false - matrix: - python-version: ["3.9"] - runs-on: [self-hosted, llm, spr01-perf] - env: - OMP_NUM_THREADS: 16 - THREAD_NUM: 16 - ANALYTICS_ZOO_ROOT: ${{ github.workspace }} - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - shell: bash - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade wheel - python -m pip install --upgrade omegaconf - python -m pip install --upgrade pandas - python -m pip install --upgrade einops - python -m pip install --upgrade tiktoken - python -m pip install --upgrade transformers_stream_generator - - - name: Download llm binary - uses: ./.github/actions/llm/download-llm-binary - - - name: Run LLM install (all) test - uses: ./.github/actions/llm/setup-llm-env - - - name: Test on cpu - shell: bash - run: | - mv python/llm/test/benchmark/stable-version-cpu-stress-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml - cd python/llm/dev/benchmark/all-in-one - export http_proxy=${HTTP_PROXY} - export https_proxy=${HTTPS_PROXY} - source bigdl-llm-init -t - export OMP_NUM_THREADS=48 - # hide time info - sed -i 's/str(end - st)/"xxxxxx"/g' run-stress-test.py - python run-stress-test.py - cp ./*.csv /models/stable_version_stress_test_cpu/ - cd ../../../test/benchmark - python -m pip install pandas==1.5.3 - python csv_to_html.py -f /models/stable_version_stress_test_cpu/ diff --git a/.github/workflows/llm_tests_for_stable_version_on_spr.yml b/.github/workflows/llm_tests_for_stable_version_on_spr.yml new file mode 100644 index 00000000..59f4d64e --- /dev/null +++ b/.github/workflows/llm_tests_for_stable_version_on_spr.yml @@ -0,0 +1,129 @@ +name: SPR LLM Test for Stable Version + +# Cancel previous runs in the PR when you push new commits +concurrency: + group: ${{ github.workflow }}-llm-performance-tests-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +# Controls when the action will run. +on: + # pull_request: + # branches: [main] + # paths: + # - ".github/workflows/llm_performance_tests.yml" + # - "python/llm/test/benchmark/**" + # - "python/llm/dev/benchmark/all-in-one/**" + workflow_dispatch: + workflow_call: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + llm-cpp-build: + uses: ./.github/workflows/llm-binary-build.yml + + llm-perf-regression-test-on-spr: + needs: llm-cpp-build + strategy: + fail-fast: false + matrix: + python-version: ["3.9"] + runs-on: [self-hosted, llm, spr01-perf] + env: + OMP_NUM_THREADS: 16 + THREAD_NUM: 16 + ANALYTICS_ZOO_ROOT: ${{ github.workspace }} + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade wheel + python -m pip install --upgrade omegaconf + python -m pip install --upgrade pandas + python -m pip install --upgrade einops + python -m pip install --upgrade tiktoken + python -m pip install --upgrade transformers_stream_generator + + - name: Download llm binary + uses: ./.github/actions/llm/download-llm-binary + + - name: Run LLM install (all) test + uses: ./.github/actions/llm/setup-llm-env + + - name: Test on cpu + shell: bash + run: | + mv python/llm/test/benchmark/stable-version-cpu-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml + cd python/llm/dev/benchmark/all-in-one + export http_proxy=${HTTP_PROXY} + export https_proxy=${HTTPS_PROXY} + source bigdl-llm-init -t + export OMP_NUM_THREADS=48 + # hide time info + sed -i 's/str(end - st)/"xxxxxx"/g' run.py + python run.py + cp ./*.csv /models/stable_version_perf_regression_test_cpu/ + cd ../../../test/benchmark + python -m pip install pandas==1.5.3 + python csv_to_html.py -f /models/stable_version_perf_regression_test_cpu/ -b /models/stable_version_perf_regression_test_cpu/transformer_int4-results-1baseline.csv -t 5.0 + + + llm-stress-test-on-spr: + needs: llm-perf-regression-test-on-spr + strategy: + fail-fast: false + matrix: + python-version: ["3.9"] + runs-on: [self-hosted, llm, spr01-perf] + env: + OMP_NUM_THREADS: 16 + THREAD_NUM: 16 + ANALYTICS_ZOO_ROOT: ${{ github.workspace }} + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade wheel + python -m pip install --upgrade omegaconf + python -m pip install --upgrade pandas + python -m pip install --upgrade einops + python -m pip install --upgrade tiktoken + python -m pip install --upgrade transformers_stream_generator + + - name: Download llm binary + uses: ./.github/actions/llm/download-llm-binary + + - name: Run LLM install (all) test + uses: ./.github/actions/llm/setup-llm-env + + - name: Test on cpu + shell: bash + run: | + mv python/llm/test/benchmark/stable-version-cpu-stress-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml + cd python/llm/dev/benchmark/all-in-one + export http_proxy=${HTTP_PROXY} + export https_proxy=${HTTPS_PROXY} + source bigdl-llm-init -t + export OMP_NUM_THREADS=48 + # hide time info + sed -i 's/str(end - st)/"xxxxxx"/g' run-stress-test.py + python run-stress-test.py + cp ./*.csv /models/stable_version_stress_test_cpu/ + cd ../../../test/benchmark + python -m pip install pandas==1.5.3 + python csv_to_html.py -f /models/stable_version_stress_test_cpu/