diff --git a/.github/workflows/llm-nightly-test.yml b/.github/workflows/llm-nightly-test.yml
index c39fad75..67ecad81 100644
--- a/.github/workflows/llm-nightly-test.yml
+++ b/.github/workflows/llm-nightly-test.yml
@@ -85,4 +85,4 @@ jobs:
     uses: ./.github/workflows/llm_unit_tests_linux.yml
   llm-inference-test-on-windows:
     needs: llm-nightly-convert-test-avx512
-    uses: ./.github/workflows/llm_unit_tests_windows.yml
\ No newline at end of file
+    uses: ./.github/workflows/llm-nightly-test-windows.yml
\ No newline at end of file
diff --git a/python/llm/test/win/ftp-get-stable-ckpts.txt.example b/python/llm/test/win/ftp-get-stable-ckpts.txt.example
deleted file mode 100644
index 065932db..00000000
--- a/python/llm/test/win/ftp-get-stable-ckpts.txt.example
+++ /dev/null
@@ -1,8 +0,0 @@
-open $HOST_IP $PORT
-ftp_username
-ftp_password
-prompt
-cd llm/ggml-actions/stable/
-lcd $BASE_DIR\stable_ckpts
-mget *
-quit
diff --git a/python/llm/test/win/ftp.txt.example b/python/llm/test/win/ftp.txt.example
deleted file mode 100644
index 81e11030..00000000
--- a/python/llm/test/win/ftp.txt.example
+++ /dev/null
@@ -1,7 +0,0 @@
-open $HOST_IP $PORT
-ftp_username
-ftp_password
-prompt
-cd llm
-put path\\to\\file
-quit
diff --git a/python/llm/test/win/win_env_setup_and_test.bat b/python/llm/test/win/win_env_setup_and_test.bat
deleted file mode 100644
index 25bd3e6e..00000000
--- a/python/llm/test/win/win_env_setup_and_test.bat
+++ /dev/null
@@ -1,88 +0,0 @@
-@REM #
-@REM # Copyright 2016 The BigDL Authors.
-@REM #
-@REM # Licensed under the Apache License, Version 2.0 (the "License");
-@REM # you may not use this file except in compliance with the License.
-@REM # You may obtain a copy of the License at
-@REM #
-@REM #     http://www.apache.org/licenses/LICENSE-2.0
-@REM #
-@REM # Unless required by applicable law or agreed to in writing, software
-@REM # distributed under the License is distributed on an "AS IS" BASIS,
-@REM # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-@REM # See the License for the specific language governing permissions and
-@REM # limitations under the License.
-@REM #
-
-@REM Usage: call win_test_starter.bat %base_dir%
-@REM Example: win_test_starter.bat C:\Users\obe\bigdl-llm-test
-@REM 
-@REM base_dir
-@REM ├───BigDL
-@REM ├───logs
-@REM │   └───previous_logs
-@REM ├───models
-@REM │   ├───bloomz-7b1
-@REM │   ├───gpt4all-7b-hf
-@REM │   └───gptneox-7b-redpajama-bf16
-@REM ├───ftp.txt
-@REM └───ftp-get-stable-ckpts.txt
-
-echo The current directory is %CD%
-set base_dir=%1
-echo %base_dir%
-
-@REM Download stable int4 ckpt from ftp... 
-@REM ---- Disabled since the downloaded file will cause windowsError 0xe06d7363
-@REM ---- Should be downloaded manually
-set stable_ckpt_path=%base_dir%\stable_ckpts
-@REM mkdir %stable_ckpt_path%
-@REM ftp -s:%base_dir%\ftp-get-stable-ckpts.txt
-
-@REM Pull the latest code
-cd %base_dir%\BigDL
-git pull
-
-@REM Build and install bigdl-llm
-pip uninstall bigdl-llm -y
-pip uninstall numpy torch transformers sentencepiece accelerate peft -y
-pip install requests pytest
-cd python\llm
-@REM pip install .[all] --use-pep517
-python setup.py clean --all bdist_wheel
-for %%x in (%base_dir%\BigDL\python\llm\dist\*.whl) do set whl_name=%%x
-pip install -i https://pypi.python.org/simple "%whl_name%[all]"
-
-
-@REM Run pytest
-mkdir %base_dir%\converted_models
-mkdir %base_dir%\stable_ckpts
-set BLOOM_ORIGIN_PATH=%base_dir%\models\bloomz-7b1
-set LLAMA_ORIGIN_PATH=%base_dir%\models\gpt4all-7b-hf
-set STARCODER_ORIGIN_PATH=%base_dir%\models\gpt_bigcode-santacoder
-set GPTNEOX_ORIGIN_PATH=%base_dir%\models\gptneox-7b-redpajama-bf16
-set INT4_CKPT_DIR=%base_dir%\converted_models
-set LLAMA_INT4_CKPT_PATH=%stable_ckpt_path%\bigdl_llm_llama_7b_q4_0.bin
-set GPTNEOX_INT4_CKPT_PATH=%stable_ckpt_path%\bigdl_llm_redpajama_7b_q4_0.bin
-set BLOOM_INT4_CKPT_PATH=%stable_ckpt_path%\bigdl_llm_bloom_7b_q4_0.bin
-set STARCODER_INT4_CKPT_PATH=%stable_ckpt_path%\bigdl_llm_santacoder_1b_q4_0.bin
-
-echo "Running the convert models tests..."
-python -m pytest -s .\test\convert\test_convert_model.py
-
-echo "Running the inference models tests..."
-python -m pytest -s .\test\inference\test_call_models.py
-
-@REM Clean up
-pip uninstall bigdl-llm -y
-pip uninstall numpy torch transformers sentencepiece accelerate peft -y
-echo "Removing the quantized models and libs..."
-rmdir /s /q %INT4_CKPT_DIR%
-rmdir /s /q %base_dir%\BigDL\python\llm\src\bigdl\llm\libs
-rmdir /s /q %base_dir%\BigDL\python\llm\dist
-
-@REM Upload the log file
-echo "Uploading the test logs to ftp..."
-ftp -s:..\..\..\ftp.txt
-
-exit 0
diff --git a/python/llm/test/win/win_test_log.py b/python/llm/test/win/win_test_log.py
deleted file mode 100644
index 454267d9..00000000
--- a/python/llm/test/win/win_test_log.py
+++ /dev/null
@@ -1,62 +0,0 @@
-#
-# Copyright 2016 The BigDL Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import os
-from datetime import datetime
-
-log_file_name = 'win_llm_test.log'
-
-def create_new_log(logger_dir):
-    archive_previous_log_file(logger_dir)
-    new_log_path = os.path.join(logger_dir, log_file_name)
-    with open(new_log_path, "w") as f:
-        now = datetime.now()
-        date_time = now.strftime("%Y%m%d-%H%M%S")
-        f.write(date_time)
-    return new_log_path
-
-
-def archive_previous_log_file(logger_dir):
-    log_file_path = os.path.join(logger_dir, log_file_name)
-    if os.path.exists(log_file_path):
-        with open(log_file_path) as f:
-            time_info = f.readline().strip('\n')
-        log_file_name_list = log_file_name.split('.')
-        new_log_file_name = log_file_name_list[0] + "_" + time_info + "." + log_file_name_list[1]
-        os.makedirs(os.path.join(logger_dir, "previous_logs"), exist_ok=True)
-        os.rename(log_file_path, 
-                os.path.join(logger_dir, "previous_logs", new_log_file_name))
-
-
-def manage_logs(logger_dir):
-    os.makedirs(logger_dir, exist_ok=True)
-    new_log_path = create_new_log(logger_dir)
-    return new_log_path
-
-
-def main():
-    parser = argparse.ArgumentParser(description='Win test logger')
-    parser.add_argument('--logger_dir', type=str, 
-                        default=r"C:\Users\obe\bigdl-llm-test\logs", required=True,
-                        help='The directory to log files.')
-    args = parser.parse_args()
-
-    os.makedirs(args.logger_dir, exist_ok=True)
-    create_new_log(args.logger_dir)
-
-
-if __name__ == '__main__':
-    main()
diff --git a/python/llm/test/win/win_test_starter.bat b/python/llm/test/win/win_test_starter.bat
deleted file mode 100644
index 7255da07..00000000
--- a/python/llm/test/win/win_test_starter.bat
+++ /dev/null
@@ -1,30 +0,0 @@
-@REM #
-@REM # Copyright 2016 The BigDL Authors.
-@REM #
-@REM # Licensed under the Apache License, Version 2.0 (the "License");
-@REM # you may not use this file except in compliance with the License.
-@REM # You may obtain a copy of the License at
-@REM #
-@REM #     http://www.apache.org/licenses/LICENSE-2.0
-@REM #
-@REM # Unless required by applicable law or agreed to in writing, software
-@REM # distributed under the License is distributed on an "AS IS" BASIS,
-@REM # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-@REM # See the License for the specific language governing permissions and
-@REM # limitations under the License.
-@REM #
-
-@REM Usage: call win_test_starter.bat %conda_activate_path% %base_dir%
-@REM Example: win_test_starter.bat C:\ProgramData\Anaconda3\Scripts\activate.bat C:\Users\obe\bigdl-llm-test
-
-set conda_activate_path=%1
-set base_dir=%2
-
-call %conda_activate_path% bigdl-llm
-python %base_dir%\BigDL\python\llm\test\win\win_test_log.py --logger_dir %base_dir%\logs
-set logger_file=%base_dir%\logs\win_llm_test.log
-call %base_dir%\BigDL\python\llm\test\win\win_env_setup_and_test.bat %base_dir% >> %logger_file% 2>&1
-
-pause
-
-exit 0