[LLM] Merge the llm.cpp build and the pypi release (#8503)

* checkout llm.cpp to build new binary
* use artifact to get latest built binary files
* rename quantize
* modify all release workflow

.github/actions/llm/download-llm-binary/action.yml

@@ -0,0 +1,19 @@
+name: Download LLM binary files
+description: Download built binary files from github artifact
+
+runs:
+  using: "composite"
+  steps:
+    - name: Download all build files
+      uses: actions/download-artifact@v3
+    - name: Move build resources
+      shell: bash
+      run: |
+        rm -rf python/llm/llm-binary || true
+        mkdir python/llm/llm-binary
+        mv linux-avx2/* python/llm/llm-binary/
+        mv linux-avx512/* python/llm/llm-binary/
+        mv windows-avx2/* python/llm/llm-binary/
+        rm -rf linux-avx2
+        rm -rf linux-avx512
+        rm -rf windows-avx2

.github/workflows/llm-binary-build.yml

@@ -0,0 +1,144 @@
+name: LLM Binary Build
+
+# Cancel previous runs in the PR when you push new commits
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push or pull request events but only for the main branch
+  pull_request:
+    branches: [main]
+    paths:
+      - ".github/workflows/llm-pypi-release.yml"
+  workflow_dispatch:
+  
+  workflow_call:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  linux-build-avx2:
+    runs-on: [self-hosted, AVX2, centos7]
+    steps:
+      - name: Set access token
+        run: |
+          echo "github_access_token=${GITHUB_ACCESS_TOKEN}" >> "$GITHUB_ENV"
+      - uses: actions/checkout@v3
+        with:
+          repository: "intel-analytics/llm.cpp"
+          token: ${{ env.github_access_token }}
+      - name: Install Build Environment
+        shell: bash
+        run: |
+          yum update -y
+          yum install -y centos-release-scl
+          yum install -y devtoolset-11
+          yum install -y sshpass netcat
+      - name: Build binary
+        shell: bash
+        run: |
+          scl enable devtoolset-11 'make'
+      - name: Move release binary
+        shell: bash
+        run: |
+          mkdir release
+          mv build/main-bloom release/main-bloom_avx2
+          mv build/quantize-bloom release/quantize-bloom
+          mv build/libbloom.so release/libbloom_avx2.so
+          mv build/main-llama release/main-llama_avx2
+          mv build/quantize-llama release/quantize-llama
+          mv build/libllama.so release/libllama_avx2.so
+          mv build/main-gptneox release/main-gptneox_avx2
+          mv build/quantize-gptneox release/quantize-gptneox
+          mv build/libgptneox.so release/libgptneox_avx2.so
+          mv build/main-starcoder release/main-starcoder_avx2
+          mv build/quantize-starcoder release/quantize-starcoder
+          mv build/libstarcoder.so release/libstarcoder_avx2.so
+      - name: Archive build files
+        uses: actions/upload-artifact@v3
+        with:
+          name: linux-avx2
+          path: |
+            release
+      - name: Clean up test environment
+        shell: bash
+        run: |
+          make clean
+
+  linux-build-avx512:
+    runs-on: [self-hosted, AVX512, centos7]
+    steps:
+      - name: Set access token
+        run: |
+          echo "github_access_token=${GITHUB_ACCESS_TOKEN}" >> "$GITHUB_ENV"
+      - uses: actions/checkout@v3
+        with:
+          repository: "intel-analytics/llm.cpp"
+          token: ${{ env.github_access_token }}
+      - name: Install Build Environment
+        shell: bash
+        run: |
+          yum update -y
+          yum install -y centos-release-scl
+          yum install -y devtoolset-11
+          yum install -y sshpass netcat
+      - name: Build binary
+        shell: bash
+        run: |
+          scl enable devtoolset-11 'make'
+      - name: Move release binary
+        shell: bash
+        run: |
+          mkdir release
+          mv build/main-bloom release/main-bloom_avx512
+          mv build/quantize-bloom release/quantize-bloom_avx512
+          mv build/libbloom.so release/libbloom_avx512.so
+          mv build/main-llama release/main-llama_avx512
+          mv build/quantize-llama release/quantize-llama_avx512
+          mv build/libllama.so release/libllama_avx512.so
+          mv build/main-gptneox release/main-gptneox_avx512
+          mv build/quantize-gptneox release/quantize-gptneox_avx512
+          mv build/libgptneox.so release/libgptneox_avx512.so
+          mv build/main-starcoder release/main-starcoder_avx512
+          mv build/quantize-starcoder release/quantize-starcoder_avx512
+          mv build/libstarcoder.so release/libstarcoder_avx512.so
+      - name: Archive build files
+        uses: actions/upload-artifact@v3
+        with:
+          name: linux-avx512
+          path: |
+            release
+      - name: Clean up test environment
+        shell: bash
+        run: |
+          make clean
+
+  windows-build-avx2:
+    runs-on: [self-hosted, Windows]
+    steps:
+      - name: Set access token
+        run: |
+          echo "github_access_token=$env:GITHUB_ACCESS_TOKEN" >> $env:GITHUB_ENV
+          echo "github_access_token=$env:GITHUB_ACCESS_TOKEN"
+      - uses: actions/checkout@v3
+        with:
+          repository: "intel-analytics/llm.cpp"
+          token: ${{ env.github_access_token }}
+      - name: Add msbuild to PATH
+        uses: microsoft/setup-msbuild@v1.1
+        with:
+          msbuild-architecture: x64
+      - name: Add cmake to PATH
+        uses: ilammy/msvc-dev-cmd@v1
+      - name: Build binary
+        shell: powershell
+        run: |
+          cmake .
+          cmake --build . --config Release
+      - name: Archive build files
+        uses: actions/upload-artifact@v3
+        with:
+          name: windows-avx2
+          path: |
+            build/Release

.github/workflows/llm-nightly-test.yml

@@ -16,8 +16,11 @@ on:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
+  llm-cpp-build:
+    uses: ./.github/workflows/llm-binary-build.yml
   llm-nightly-convert-test-avx512:
     runs-on: [ self-hosted, llm, AVX512, ubuntu-20.04-lts ]
+    needs: llm-cpp-build
     strategy:
       fail-fast: false
       matrix:
@@ -44,6 +47,9 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install --upgrade setuptools==58.0.4
           python -m pip install --upgrade wheel
+
+      - name: Download llm binary
+        uses: ./.github/actions/llm/download-llm-binary
       
       - name: Install BigDL-LLM
         uses: ./.github/actions/llm/setup-llm-env

.github/workflows/llm_unit_tests_linux.yml

@@ -13,21 +13,25 @@ on:
     paths:
       - 'python/llm/**'
       - '.github/workflows/llm_unit_tests_linux.yml'
+      - '.github/workflows/llm-binary-build.yml'
       - '.github/actions/llm/setup-llm-env/action.yml'
       - '.github/actions/llm/remove-llm-env/action.yml'
       - '.github/actions/llm/cli-test/action.yml'
       - '.github/actions/llm/inference-test/action.yml'
       - '.github/actions/llm/langchain-test/action.yml'
+      - '.github/actions/llm/download-llm-binary/action.yml'
   pull_request:
     branches: [ main ]
     paths:
       - 'python/llm/**'
       - '.github/workflows/llm_unit_tests_linux.yml'
+      - '.github/workflows/llm-binary-build.yml'
       - '.github/actions/llm/setup-llm-env/action.yml'
       - '.github/actions/llm/remove-llm-env/action.yml'
       - '.github/actions/llm/cli-test/action.yml'
       - '.github/actions/llm/inference-test/action.yml'
       - '.github/actions/llm/langchain-test/action.yml'
+      - '.github/actions/llm/download-llm-binary/action.yml'
   workflow_dispatch:
 
 env:
@@ -39,8 +43,11 @@ env:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
+  llm-cpp-build:
+    uses: ./.github/workflows/llm-binary-build.yml
   llm-unit-test-linux-avx512:
     runs-on: [ self-hosted, llm, AVX512, ubuntu-20.04-lts ]
+    needs: llm-cpp-build
     strategy:
       fail-fast: false
       matrix:
@@ -59,6 +66,9 @@ jobs:
           python -m pip install --upgrade setuptools==58.0.4
           python -m pip install --upgrade wheel
 
+      - name: Download llm binary
+        uses: ./.github/actions/llm/download-llm-binary
+
       - name: Run LLM install (all) test
         uses: ./.github/actions/llm/setup-llm-env
         env:

.github/workflows/llm_unit_tests_windows.yml

@@ -22,8 +22,11 @@ on:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
+  llm-cpp-build:
+    uses: ./.github/workflows/llm-binary-build.yml
   llm-unit-test-windows:
     runs-on: ${{ matrix.os }}
+    needs: llm-cpp-build
     strategy:
       fail-fast: false
       matrix:
@@ -40,7 +43,8 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install --upgrade setuptools==58.0.4
           python -m pip install --upgrade wheel
-
+      - name: Download llm binary
+        uses: ./.github/actions/llm/download-llm-binary
       - name: Run LLM-init test
         shell: bash
         run: |

python/llm/setup.py

@@ -33,6 +33,7 @@ import sys
 import urllib.request
 import requests
 import re
+import glob
 
 from setuptools import setup
 
@@ -44,6 +45,7 @@ exclude_patterns = ["*__pycache__*", "*ipynb_checkpoints*"]
 BIGDL_PYTHON_HOME = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 VERSION = open(os.path.join(BIGDL_PYTHON_HOME, 'version.txt'), 'r').read().strip()
 llm_home = os.path.join(os.path.dirname(os.path.abspath(__file__)), "src")
+github_artifact_dir = os.path.join(llm_home,'../llm-binary')
 libs_dir = os.path.join(llm_home, "bigdl", "llm", "libs")
 CONVERT_DEP = ['numpy >= 1.22', 'torch', 'transformers', 'sentencepiece', 'accelerate']
 
@@ -129,8 +131,10 @@ def download_libs(url: str, change_permission=False):
     if not os.path.exists(libso_file):
         print(">> Downloading from ", url)
         urllib.request.urlretrieve(url, libso_file)
-        if change_permission:
-            os.chmod(libso_file, 0o775)
+    else:
+        print('>> Skip downloading ', libso_file)
+    if change_permission:
+        os.chmod(libso_file, 0o775)
 
 
 def setup_package():
@@ -193,6 +197,11 @@ def setup_package():
         print(f"Deleting existing libs_dir {libs_dir} ....")
         shutil.rmtree(libs_dir)
     os.makedirs(libs_dir, exist_ok=True)
+    
+    # copy built files for github workflow
+    for built_file in glob.glob(os.path.join(github_artifact_dir,'*')):
+        print(f'Copy workflow built file: {built_file}')
+        shutil.copy(built_file, libs_dir)
 
     lib_urls = obtain_lib_urls()