From 773255e009e47c31e80bc9d42b8785be0409f0c8 Mon Sep 17 00:00:00 2001
From: "Pingchuan Ma (Henry)" <58333343+HensonMa@users.noreply.github.com>
Date: Thu, 8 Jun 2023 00:49:57 +0800
Subject: [PATCH] [LLM] Add dev wheel building and basic UT script for LLM
 package on Linux (#8264)

* add wheel build for linux

* test fix

* test self-hosted runner

* test fix

* update runner

* update runner

* update fix

* init cicd

* init cicd

* test conda

* update fix

* update no need manual python deps

* test fix bugs

* test fix bugs

* test fix bugs

* fix bugs
---
 .../workflows/llm_unit_tests_basic_linux.yml  | 71 +++++++++++++++++++
 python/llm/dev/release_default_linux.sh       | 37 ++++++++++
 2 files changed, 108 insertions(+)
 create mode 100644 .github/workflows/llm_unit_tests_basic_linux.yml
 create mode 100644 python/llm/dev/release_default_linux.sh

diff --git a/.github/workflows/llm_unit_tests_basic_linux.yml b/.github/workflows/llm_unit_tests_basic_linux.yml
new file mode 100644
index 00000000..5683af4e
--- /dev/null
+++ b/.github/workflows/llm_unit_tests_basic_linux.yml
@@ -0,0 +1,71 @@
+name: LLM Unit Tests Basic on Linux
+
+# Cancel previous runs in the PR when you push new commits
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+# Controls when the action will run. 
+on:
+  # Triggers the workflow on push or pull request events but only for the main branch
+  push:
+    branches: [ main ]
+    paths:
+      - 'python/llm/**'
+      - '.github/workflows/llm_unit_tests_basic_linux.yml'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'python/llm/**'
+      - '.github/workflows/llm_unit_tests_basic_linux.yml'
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  llm-unit-test-basic-linux:
+    runs-on: [ self-hosted, Gondolin, ubuntu-20.04-lts ]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9"]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade setuptools==58.0.4
+          python -m pip install --upgrade wheel
+
+      - name: Run LLM-init test
+        shell: bash
+        run: |
+          $CONDA_HOME/bin/conda env remove -y -n bigdl-init-llm || true
+          $CONDA_HOME/bin/conda create -n bigdl-init-llm -y python==3.9 setuptools==58.0.4 -c ${GONDOLIN_CONDA_CHANNEL} --override-channels
+          source $CONDA_HOME/bin/activate bigdl-init-llm
+          $CONDA_HOME/bin/conda info
+          bash python/llm/dev/release_default_linux.sh default false
+          pip install -i https://pypi.python.org/simple python/llm/dist/bigdl_llm*.whl
+          source $CONDA_HOME/bin/deactivate
+          $CONDA_HOME/bin/conda remove -n bigdl-init-llm --all
+        env:
+          ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
+      
+      - name: Run LLM naive installation test
+        shell: bash
+        run: |
+          $CONDA_HOME/bin/conda env remove -y -n bigdl-init-llm || true
+          $CONDA_HOME/bin/conda create -n bigdl-init-llm -y python==3.9 setuptools==58.0.4 -c ${GONDOLIN_CONDA_CHANNEL} --override-channels
+          source $CONDA_HOME/bin/activate bigdl-init-llm
+          $CONDA_HOME/bin/conda info
+          bash python/llm/dev/release_default_linux.sh default false
+          whl_name=$(ls python/llm/dist)
+          pip install -i https://pypi.python.org/simple "python/llm/dist/${whl_name}[all]"
+          pip install pytest
+          bash python/llm/test/run-llm-basic-tests.sh
+          source $CONDA_HOME/bin/deactivate
+          $CONDA_HOME/bin/conda remove -n bigdl-init-llm --all
+        env:
+          ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
diff --git a/python/llm/dev/release_default_linux.sh b/python/llm/dev/release_default_linux.sh
new file mode 100644
index 00000000..9c5c1214
--- /dev/null
+++ b/python/llm/dev/release_default_linux.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+#
+# Copyright 2016 The BigDL Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is the default script with maven parameters to release bigdl-math for linux.
+# Note that if the maven parameters to build bigdl-math need to be changed,
+# make sure to change this file accordingly.
+# If you want to customize the release, please use release.sh and specify maven parameters instead.
+
+set -e
+RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd)
+echo $RUN_SCRIPT_DIR
+
+if (( $# < 2)); then
+  echo "Usage: release_default_linux.sh version upload"
+  echo "Usage example: bash release_default_linux.sh default true"
+  exit -1
+fi
+
+version=$1
+upload=$2
+
+bash ${RUN_SCRIPT_DIR}/release.sh linux ${version} ${upload}
\ No newline at end of file