From 936d21635f004160b70914c57d850aabb4c8c2a5 Mon Sep 17 00:00:00 2001
From: Yuwen Hu <54161268+Oscilloscope98@users.noreply.github.com>
Date: Wed, 5 Jul 2023 10:09:10 +0800
Subject: [PATCH] [LLM] Extract tests to `.github/actions` to improve
 reusability (#8457)

* Extract tests to .github/actions for better reusing in nightly tests

* Small fix

* Small fix
---
 .github/actions/llm/setup-llm-env/action.yml | 20 +++++
 .github/workflows/llm-nightly-test.yml       | 56 ++-----------
 .github/workflows/llm_unit_tests_linux.yml   | 86 ++++++--------------
 3 files changed, 54 insertions(+), 108 deletions(-)
 create mode 100644 .github/actions/llm/setup-llm-env/action.yml

diff --git a/.github/actions/llm/setup-llm-env/action.yml b/.github/actions/llm/setup-llm-env/action.yml
new file mode 100644
index 00000000..06b21415
--- /dev/null
+++ b/.github/actions/llm/setup-llm-env/action.yml
@@ -0,0 +1,20 @@
+name: 'Setup BigDL-LLM Env'
+description: 'Conda env create and BigDL-LLM installation'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Create conda env for llm tests and conduct install tests
+      shell: bash
+      run: |
+        $CONDA_HOME/bin/conda env remove -y -n bigdl-llm-test || true
+        $CONDA_HOME/bin/conda create -n bigdl-llm-test -y python==3.9 setuptools==58.0.4
+        source $CONDA_HOME/bin/activate bigdl-llm-test
+        $CONDA_HOME/bin/conda info
+        pip install requests
+        bash python/llm/dev/release_default_linux.sh default false
+        whl_name=$(ls python/llm/dist)
+        pip install -i https://pypi.python.org/simple "python/llm/dist/${whl_name}[all]"
+        pip install pytest
+        bash python/llm/test/run-llm-install-tests.sh
+        source $CONDA_HOME/bin/deactivate
diff --git a/.github/workflows/llm-nightly-test.yml b/.github/workflows/llm-nightly-test.yml
index cca96aa6..4e5040b4 100644
--- a/.github/workflows/llm-nightly-test.yml
+++ b/.github/workflows/llm-nightly-test.yml
@@ -8,12 +8,15 @@ on:
     branches: [ main ]
     paths:
       - '.github/workflows/llm-nightly-test.yml'
+      - '.github/actions/llm/setup-llm-env/action.yml'
+      - '.github/actions/llm/remove-llm-env/action.yml'
+      - '.github/actions/llm/convert-test/action.yml'
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  llm-nightly-convert-test:
+  llm-nightly-convert-test-avx512:
     runs-on: [ self-hosted, llm, AVX512, ubuntu-20.04-lts ]
     strategy:
       fail-fast: false
@@ -43,55 +46,12 @@ jobs:
           python -m pip install --upgrade wheel
       
       - name: Install BigDL-LLM
-        shell: bash
-        run: |
-          $CONDA_HOME/bin/conda env remove -y -n bigdl-init-llm || true
-          $CONDA_HOME/bin/conda create -n bigdl-init-llm -y python==3.9 setuptools==58.0.4
-          source $CONDA_HOME/bin/activate bigdl-init-llm
-          $CONDA_HOME/bin/conda info
-          pip install requests
-          bash python/llm/dev/release_default_linux.sh default false
-          whl_name=$(ls python/llm/dist)
-          pip install -i https://pypi.python.org/simple "python/llm/dist/${whl_name}[all]"
-          pip install pytest
-          source $CONDA_HOME/bin/deactivate
+        uses: ./.github/actions/llm/setup-llm-env
         env:
           ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
 
       - name: Download original models & convert 
-        shell: bash
-        run: |
-          source $CONDA_HOME/bin/activate bigdl-init-llm
-
-          if [ ! -d $LLAMA_ORIGIN_PATH ]; then
-            echo "Directory $LLAMA_ORIGIN_PATH not found. Downloading from FTP server..."
-            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/llama-7b-hf -P $ORIGIN_DIR
-          fi
-          bash python/llm/test/run-llm-convert-tests.sh llama
-          rm -rf $LLAMA_ORIGIN_PATH
-
-          if [ ! -d $GPTNEOX_ORIGIN_PATH ]; then
-            echo "Directory $GPTNEOX_ORIGIN_PATH not found. Downloading from FTP server..."
-            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/gptneox-7b-redpajama-bf16 -P $ORIGIN_DIR
-          fi
-          bash python/llm/test/run-llm-convert-tests.sh gptneox
-          rm -rf $GPTNEOX_ORIGIN_PATH
-
-          if [ ! -d $BLOOM_ORIGIN_PATH ]; then
-            echo "Directory $BLOOM_ORIGIN_PATH not found. Downloading from FTP server..."
-            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/bloomz-7b1 -P $ORIGIN_DIR
-          fi
-          bash python/llm/test/run-llm-convert-tests.sh bloom
-          rm -rf $BLOOM_ORIGIN_PATH
-
-          if [ ! -d $STARCODER_ORIGIN_PATH ]; then
-            echo "Directory $STARCODER_ORIGIN_PATH not found. Downloading from FTP server..."
-            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/gpt_bigcode-santacoder -P $ORIGIN_DIR
-          fi
-          bash python/llm/test/run-llm-convert-tests.sh starcoder
-          rm -rf $STARCODER_ORIGIN_PATH
-
-          source $CONDA_HOME/bin/deactivate
+        uses: ./.github/actions/llm/convert-test
         env:
           ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
 
@@ -104,8 +64,6 @@ jobs:
           tnftp -u ${LLM_FTP_URL}/${INT4_CKPT_DIR:1}/bigdl_llm_santacoder_1b_q4_0.bin $STARCODER_INT4_CKPT_PATH
 
       - name: Clean up test environment
-        shell: bash
-        run: |
-          $CONDA_HOME/bin/conda env remove -y -n bigdl-init-llm || true
+        uses: ./.github/actions/llm/remove-llm-env
         env:
           ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
diff --git a/.github/workflows/llm_unit_tests_linux.yml b/.github/workflows/llm_unit_tests_linux.yml
index 02f2b535..bd8cc6cb 100644
--- a/.github/workflows/llm_unit_tests_linux.yml
+++ b/.github/workflows/llm_unit_tests_linux.yml
@@ -13,11 +13,28 @@ on:
     paths:
       - 'python/llm/**'
       - '.github/workflows/llm_unit_tests_linux.yml'
+      - '.github/actions/llm/setup-llm-env/action.yml'
+      - '.github/actions/llm/remove-llm-env/action.yml'
+      - '.github/actions/llm/cli-test/action.yml'
+      - '.github/actions/llm/inference-test/action.yml'
+      - '.github/actions/llm/langchain-test/action.yml'
   pull_request:
     branches: [ main ]
     paths:
       - 'python/llm/**'
       - '.github/workflows/llm_unit_tests_linux.yml'
+      - '.github/actions/llm/setup-llm-env/action.yml'
+      - '.github/actions/llm/remove-llm-env/action.yml'
+      - '.github/actions/llm/cli-test/action.yml'
+      - '.github/actions/llm/inference-test/action.yml'
+      - '.github/actions/llm/langchain-test/action.yml'
+
+env:
+  INT4_CKPT_DIR: ./llm/ggml-actions/stable
+  LLAMA_INT4_CKPT_PATH: ./llm/ggml-actions/stable/bigdl_llm_llama_7b_q4_0.bin
+  GPTNEOX_INT4_CKPT_PATH: ./llm/ggml-actions/stable/bigdl_llm_redpajama_7b_q4_0.bin
+  BLOOM_INT4_CKPT_PATH: ./llm/ggml-actions/stable/bigdl_llm_bloom_7b_q4_0.bin
+  STARCODER_INT4_CKPT_PATH: ./llm/ggml-actions/stable/bigdl_llm_santacoder_1b_q4_0.bin
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
@@ -28,11 +45,6 @@ jobs:
       matrix:
         python-version: ["3.9"]
     env:
-      INT4_CKPT_DIR: ./llm/ggml-actions/stable
-      LLAMA_INT4_CKPT_PATH: ./llm/ggml-actions/stable/bigdl_llm_llama_7b_q4_0.bin
-      GPTNEOX_INT4_CKPT_PATH: ./llm/ggml-actions/stable/bigdl_llm_redpajama_7b_q4_0.bin
-      BLOOM_INT4_CKPT_PATH: ./llm/ggml-actions/stable/bigdl_llm_bloom_7b_q4_0.bin
-      STARCODER_INT4_CKPT_PATH: ./llm/ggml-actions/stable/bigdl_llm_santacoder_1b_q4_0.bin
       THREAD_NUM: 6
     steps:
       - uses: actions/checkout@v2
@@ -46,35 +58,8 @@ jobs:
           python -m pip install --upgrade setuptools==58.0.4
           python -m pip install --upgrade wheel
 
-      - name: Run LLM-init test
-        shell: bash
-        run: |
-          $CONDA_HOME/bin/conda env remove -y -n bigdl-init-llm || true
-          $CONDA_HOME/bin/conda create -n bigdl-init-llm -y python==3.9 setuptools==58.0.4
-          source $CONDA_HOME/bin/activate bigdl-init-llm
-          $CONDA_HOME/bin/conda info
-          pip install requests
-          bash python/llm/dev/release_default_linux.sh default false
-          pip install -i https://pypi.python.org/simple python/llm/dist/bigdl_llm*.whl
-          source $CONDA_HOME/bin/deactivate
-          $CONDA_HOME/bin/conda remove -n bigdl-init-llm --all
-        env:
-          ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
-      
       - name: Run LLM install (all) test
-        shell: bash
-        run: |
-          $CONDA_HOME/bin/conda env remove -y -n bigdl-init-llm || true
-          $CONDA_HOME/bin/conda create -n bigdl-init-llm -y python==3.9 setuptools==58.0.4
-          source $CONDA_HOME/bin/activate bigdl-init-llm
-          $CONDA_HOME/bin/conda info
-          pip install requests
-          bash python/llm/dev/release_default_linux.sh default false
-          whl_name=$(ls python/llm/dist)
-          pip install -i https://pypi.python.org/simple "python/llm/dist/${whl_name}[all]"
-          pip install pytest
-          bash python/llm/test/run-llm-install-tests.sh
-          source $CONDA_HOME/bin/deactivate
+        uses: ./.github/actions/llm/setup-llm-env
         env:
           ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
 
@@ -97,39 +82,22 @@ jobs:
             wget --no-verbose $LLM_FTP_URL/${STARCODER_INT4_CKPT_PATH:1} -P $INT4_CKPT_DIR
           fi
 
+      - name: Run LLM cli test
+        uses: ./.github/actions/llm/cli-test
+        env:
+          ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
+
       - name: Run LLM inference test
-        shell: bash
-        run: |
-          source $CONDA_HOME/bin/activate bigdl-init-llm
-          $CONDA_HOME/bin/conda info
-          bash python/llm/test/run-llm-inference-tests.sh
-          source $CONDA_HOME/bin/deactivate
+        uses: ./.github/actions/llm/inference-test
         env:
           ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
 
       - name: Run LLM langchain test
-        shell: bash
-        run: |
-          source $CONDA_HOME/bin/activate bigdl-init-llm
-          $CONDA_HOME/bin/conda info
-          pip install -U langchain==0.0.184
-          pip install -U chromadb==0.3.25
-          pip install -U typing_extensions==4.5.0
-          bash python/llm/test/run-llm-langchain-tests.sh
-          source $CONDA_HOME/bin/deactivate
-        env:
-          ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
-
-      # new test steps should be added here
-
-      - name: Run LLM cli test
-        uses: ./.github/actions/llm/llm-cli-flow-verification-linux
+        uses: ./.github/actions/llm/langchain-test
         env:
           ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
 
       - name: Clean up test environment
-        shell: bash
-        run: |
-          $CONDA_HOME/bin/conda env remove -y -n bigdl-init-llm || true
+        uses: ./.github/actions/llm/remove-llm-env
         env:
-          ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
\ No newline at end of file
+          ANALYTICS_ZOO_ROOT: ${{ github.workspace }}