ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

add nightly_build workflow (#10533)

Browse source 
* add nightly_build workflow

* add create-job-status-badge action

* update

* update

* update

* update setup.py

* release

* revert
This commit is contained in:
Shaojun Liu 2024-03-26 12:47:38 +08:00 committed by GitHub
parent 0a3e4e788f
commit c563b41491
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 171 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
.github/actions/create-job-status-badge/action.yml vendored Normal file
View file

@ -0,0 +1,72 @@
name: 'Create Job Status Badge'
description: 'In a workflow with multiple jobs, create a badge that can display the completion time and status of the job'
inputs:
secret:
description: 'use to access gist'
required: true
gist-id:
description: 'gist-id'
required: true
file-name:
description: 'gist file name(.json)'
required: true
type:
description: 'workflow or job'
required: true
job-name:
description: 'job name'
is-self-hosted-runner:
description: 'If use self-hosted runner it will be true'
required: true
default: false
runner-hosted-on:
description: 'Select the region of the runner host to set the proxy'
required: true
default: ''
runs:
using: "composite"
steps:
- name: print time
shell: bash
run: |
export NOW=$( date '+%F_%H:%M:%S' )
echo "TIME=${NOW}" >> $GITHUB_ENV
- name: get job status
shell: bash
run: |
echo "JOB-STATUS=${{ job.status }}" >>$GITHUB_ENV
- name: set badge color
shell: bash
run: if [ ${{ env.JOB-STATUS }} == "success" ] ; then echo "COLOR=green" >> $GITHUB_ENV ; else echo "COLOR=red" >> $GITHUB_ENV; fi
- name: Create job badge
if: ${{ inputs.type == 'job' && inputs.secret != '' }}
uses: analytics-zoo/dynamic-badges-action@master
with:
auth: ${{ inputs.secret }}
gistID: ${{ inputs.gist-id }}
isSelfHostedRunner: ${{ inputs.is-self-hosted-runner }}
filename: ${{ inputs.file-name }}
label: ${{ inputs.job-name }}
message: ${{ env.JOB-STATUS }}
color: ${{ env.COLOR }}
runnerHostedOn: ${{ inputs.runner-hosted-on }}
- name: Create time badge
if: ${{ inputs.type == 'workflow' && inputs.secret != '' }}
uses: analytics-zoo/dynamic-badges-action@master
with:
auth: ${{ inputs.secret }}
gistID: ${{ inputs.gist-id }}
isSelfHostedRunner: ${{ inputs.is-self-hosted-runner }}
filename: ${{ inputs.file-name }}
label: Time is
message: ${{ env.TIME }}
runnerHostedOn: ${{ inputs.runner-hosted-on }}
- name: print error message
if: ${{ inputs.secret == '' }}
shell: bash
run: |
echo "Can not get secret! Please check out your workflow!(mainly caused by pr trigger)"

8
.github/actions/llm/setup-llm-env/action.yml vendored
View file

@ -16,10 +16,10 @@ runs:
pip uninstall bigdl-core-xe-esimd -y || true
pip uninstall bigdl-core-xe-21 -y || true
pip uninstall bigdl-core-xe-esimd-21 -y || true
sed -i 's/"bigdl-core-xe==" + VERSION + "/"bigdl-core-xe/g' python/llm/setup.py
sed -i 's/"bigdl-core-xe-esimd==" + VERSION + "/"bigdl-core-xe-esimd/g' python/llm/setup.py
sed -i 's/"bigdl-core-xe-21==" + VERSION/"bigdl-core-xe-21"/g' python/llm/setup.py
sed -i 's/"bigdl-core-xe-esimd-21==" + VERSION + "/"bigdl-core-xe-esimd-21/g' python/llm/setup.py
sed -i 's/"bigdl-core-xe==" + CORE_XE_VERSION + "/"bigdl-core-xe/g' python/llm/setup.py
sed -i 's/"bigdl-core-xe-esimd==" + CORE_XE_VERSION + "/"bigdl-core-xe-esimd/g' python/llm/setup.py
sed -i 's/"bigdl-core-xe-21==" + CORE_XE_VERSION/"bigdl-core-xe-21"/g' python/llm/setup.py
sed -i 's/"bigdl-core-xe-esimd-21==" + CORE_XE_VERSION + "/"bigdl-core-xe-esimd-21/g' python/llm/setup.py
pip install requests
if [[ ${{ runner.os }} == 'Linux' ]]; then

6
.github/workflows/llm_performance_tests.yml vendored
View file

@ -225,9 +225,9 @@ jobs:
python -m pip install pandas==1.5.3
python csv_to_html.py -f /models/nightly_perf_cpu
cd /models/nightly_perf_cpu
# for f in *.html; do
# curl -T "$f" ${LLM_FTP_URL}/llm/nightly_perf/nightly_perf_cpu/
# done
for f in *.html; do
curl -T "$f" ${LLM_FTP_URL}/llm/nightly_perf/nightly_perf_cpu/
done
llm-performance-test-on-core:
if: ${{ github.event.schedule || github.event.inputs.artifact == 'llm-performance-test-on-core' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests

84
.github/workflows/nightly_build.yml vendored Normal file
View file

@ -0,0 +1,84 @@
name: Nightly Build
on:
# pull_request:
# branches: [ main ]
# paths:
# - '.github/workflows/nightly_build.yml'
schedule:
- cron: '30 11 * * *' # GMT time, 11:30 GMT == 19:30 China
workflow_dispatch:
env:
GIST_ID: 48dbd87983219d4fe264adfea701815a
permissions:
contents: read
jobs:
llm-cpp-build:
uses: ./.github/workflows/llm-binary-build.yml
ipex-llm-build:
# python build can only be published once a day, please do not publish it manually
if: ${{ github.event.schedule || github.event_name == 'workflow_dispatch' }}
runs-on: [self-hosted, Bree]
needs: llm-cpp-build
steps:
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.7.15'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install build
pip install wheel
pip install twine
- name: Download llm binary
uses: ./.github/actions/llm/download-llm-binary
- name: Build package
run: |
export TIMESTAMP=`date '+%Y%m%d'`
# export TIMESTAMP=20240323
export PYPI_VERSION=2.1.0
nb_version=${PYPI_VERSION}b${TIMESTAMP}
echo ${nb_version}
## windows ##
bash python/llm/dev/release_default_windows.sh ${nb_version} true
## linux ##
bash python/llm/dev/release_default_linux.sh ${nb_version} true
- name: Create Job Badge
if: ${{ always() }}
uses: ./.github/actions/create-job-status-badge
with:
secret: ${{ secrets.GIST_SECRET }}
gist-id: ${{env.GIST_ID}}
is-self-hosted-runner: true
file-name: nb-python-build.json
type: job
job-name: python-build
runner-hosted-on: 'Shanghai'
create-workflow-badge:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
- name: create workflow badge
if: ${{ always() }}
uses: ./.github/actions/create-job-status-badge
with:
secret: ${{ secrets.GIST_SECRET }}
gist-id: ${{env.GIST_ID}}
file-name: nightly-build.json
type: workflow

4
python/llm/dev/release.sh
View file

@ -35,10 +35,10 @@ upload=$3 # Whether to upload the whl to pypi
if [ "${version}" != "default" ]; then
echo "User specified version: ${version}"
echo $version > $BIGDL_DIR/python/version.txt
echo $version > $BIGDL_DIR/python/llm/version.txt
fi
ipex_llm_version=$(cat $BIGDL_DIR/python/version.txt | head -1)
ipex_llm_version=$(cat $BIGDL_DIR/python/llm/version.txt | head -1)
echo "The effective version is: ${ipex_llm_version}"
if [ "$platform" == "linux" ]; then

11
python/llm/setup.py
View file

@ -46,6 +46,7 @@ exclude_patterns = ["*__pycache__*", "*ipynb_checkpoints*"]
IPEX_LLM_PYTHON_HOME = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
VERSION = open(os.path.join(IPEX_LLM_PYTHON_HOME,
'./llm/version.txt'), 'r').read().strip()
CORE_XE_VERSION = VERSION.replace("2.1.0", "2.5.0")
llm_home = os.path.join(os.path.dirname(os.path.abspath(__file__)), "src")
github_artifact_dir = os.path.join(llm_home, '../llm-binary')
libs_dir = os.path.join(llm_home, "ipex_llm", "libs")
@ -281,16 +282,16 @@ def setup_package():
xpu_20_requires += ["torch==2.0.1a0;platform_system=='Linux'",
"torchvision==0.15.2a0;platform_system=='Linux'",
"intel_extension_for_pytorch==2.0.110+xpu;platform_system=='Linux'",
"bigdl-core-xe==" + VERSION + ";platform_system=='Linux'",
"bigdl-core-xe-esimd==" + VERSION + ";platform_system=='Linux'"]
"bigdl-core-xe==" + CORE_XE_VERSION + ";platform_system=='Linux'",
"bigdl-core-xe-esimd==" + CORE_XE_VERSION + ";platform_system=='Linux'"]
xpu_21_requires = copy.deepcopy(all_requires)
xpu_21_requires.remove('torch')
xpu_21_requires += ["torch==2.1.0a0",
"torchvision==0.16.0a0",
"intel_extension_for_pytorch==2.1.10+xpu",
"bigdl-core-xe-21==" + VERSION,
"bigdl-core-xe-esimd-21==" + VERSION + ";platform_system=='Linux'"]
"bigdl-core-xe-21==" + CORE_XE_VERSION,
"bigdl-core-xe-esimd-21==" + CORE_XE_VERSION + ";platform_system=='Linux'"]
# default to ipex 2.1 for linux and windows
xpu_requires = copy.deepcopy(xpu_21_requires)
@ -323,7 +324,7 @@ def setup_package():
"xpu-2-0": xpu_20_requires,
"xpu-2-1": xpu_21_requires,
"serving": serving_requires,
"cpp": ["bigdl-core-cpp==" + VERSION]},
"cpp": ["bigdl-core-cpp==" + CORE_XE_VERSION]},
classifiers=[
'License :: OSI Approved :: Apache Software License',
'Programming Language :: Python :: 3',