[LLM] LLM windows daily test (#8328)
* llm-win-init * test action * test * add types * update for schtasks * update pytests * update * update * update doc * use stable ckpt from ftp instead of the converted model * download using batch -> manually * add starcoder test
This commit is contained in:
Yina Chen
GitHub
parent
ca5a4b6e3a
commit
783aea3309
5 changed files with 195 additions and 0 deletions
8
python/llm/test/win/ftp-get-stable-ckpts.txt.example
Normal file
8
python/llm/test/win/ftp-get-stable-ckpts.txt.example
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
open $HOST_IP $PORT
|
||||
ftp_username
|
||||
ftp_password
|
||||
prompt
|
||||
cd llm/ggml-actions/stable/
|
||||
lcd $BASE_DIR\stable_ckpts
|
||||
mget *
|
||||
quit
|
||||
7
python/llm/test/win/ftp.txt.example
Normal file
7
python/llm/test/win/ftp.txt.example
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
open $HOST_IP $PORT
|
||||
ftp_username
|
||||
ftp_password
|
||||
prompt
|
||||
cd llm
|
||||
put path\\to\\file
|
||||
quit
|
||||
88
python/llm/test/win/win_env_setup_and_test.bat
Normal file
88
python/llm/test/win/win_env_setup_and_test.bat
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
@REM #
|
||||
@REM # Copyright 2016 The BigDL Authors.
|
||||
@REM #
|
||||
@REM # Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@REM # you may not use this file except in compliance with the License.
|
||||
@REM # You may obtain a copy of the License at
|
||||
@REM #
|
||||
@REM # http://www.apache.org/licenses/LICENSE-2.0
|
||||
@REM #
|
||||
@REM # Unless required by applicable law or agreed to in writing, software
|
||||
@REM # distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@REM # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
@REM # See the License for the specific language governing permissions and
|
||||
@REM # limitations under the License.
|
||||
@REM #
|
||||
|
||||
@REM Usage: call win_test_starter.bat %base_dir%
|
||||
@REM Example: win_test_starter.bat C:\Users\obe\bigdl-llm-test
|
||||
@REM
|
||||
@REM base_dir
|
||||
@REM ├───BigDL
|
||||
@REM ├───logs
|
||||
@REM │ └───previous_logs
|
||||
@REM ├───models
|
||||
@REM │ ├───bloomz-7b1
|
||||
@REM │ ├───gpt4all-7b-hf
|
||||
@REM │ └───gptneox-7b-redpajama-bf16
|
||||
@REM ├───ftp.txt
|
||||
@REM └───ftp-get-stable-ckpts.txt
|
||||
|
||||
echo The current directory is %CD%
|
||||
set base_dir=%1
|
||||
echo %base_dir%
|
||||
|
||||
@REM Download stable int4 ckpt from ftp...
|
||||
@REM ---- Disabled since the downloaded file will cause windowsError 0xe06d7363
|
||||
@REM ---- Should be downloaded manually
|
||||
set stable_ckpt_path=%base_dir%\stable_ckpts
|
||||
@REM mkdir %stable_ckpt_path%
|
||||
@REM ftp -s:%base_dir%\ftp-get-stable-ckpts.txt
|
||||
|
||||
@REM Pull the latest code
|
||||
cd %base_dir%\BigDL
|
||||
git pull
|
||||
|
||||
@REM Build and install bigdl-llm
|
||||
pip uninstall bigdl-llm -y
|
||||
pip uninstall numpy torch transformers sentencepiece accelerate peft -y
|
||||
pip install requests pytest
|
||||
cd python\llm
|
||||
@REM pip install .[all] --use-pep517
|
||||
python setup.py clean --all bdist_wheel
|
||||
for %%x in (%base_dir%\BigDL\python\llm\dist\*.whl) do set whl_name=%%x
|
||||
pip install -i https://pypi.python.org/simple "%whl_name%[all]"
|
||||
|
||||
|
||||
@REM Run pytest
|
||||
mkdir %base_dir%\converted_models
|
||||
mkdir %base_dir%\stable_ckpts
|
||||
set BLOOM_ORIGIN_PATH=%base_dir%\models\bloomz-7b1
|
||||
set LLAMA_ORIGIN_PATH=%base_dir%\models\gpt4all-7b-hf
|
||||
set STARCODER_ORIGIN_PATH=%base_dir%\models\gpt_bigcode-santacoder
|
||||
set GPTNEOX_ORIGIN_PATH=%base_dir%\models\gptneox-7b-redpajama-bf16
|
||||
set INT4_CKPT_DIR=%base_dir%\converted_models
|
||||
set LLAMA_INT4_CKPT_PATH=%stable_ckpt_path%\bigdl_llm_llama_7b_q4_0.bin
|
||||
set GPTNEOX_INT4_CKPT_PATH=%stable_ckpt_path%\bigdl_llm_redpajama_7b_q4_0.bin
|
||||
set BLOOM_INT4_CKPT_PATH=%stable_ckpt_path%\bigdl_llm_bloom_7b_q4_0.bin
|
||||
set STARCODER_INT4_CKPT_PATH=%stable_ckpt_path%\bigdl_llm_santacoder_1b_q4_0.bin
|
||||
|
||||
echo "Running the convert models tests..."
|
||||
python -m pytest -s .\test\convert\test_convert_model.py
|
||||
|
||||
echo "Running the inference models tests..."
|
||||
python -m pytest -s .\test\inference\test_call_models.py
|
||||
|
||||
@REM Clean up
|
||||
pip uninstall bigdl-llm -y
|
||||
pip uninstall numpy torch transformers sentencepiece accelerate peft -y
|
||||
echo "Removing the quantized models and libs..."
|
||||
rmdir /s /q %INT4_CKPT_DIR%
|
||||
rmdir /s /q %base_dir%\BigDL\python\llm\src\bigdl\llm\libs
|
||||
rmdir /s /q %base_dir%\BigDL\python\llm\dist
|
||||
|
||||
@REM Upload the log file
|
||||
echo "Uploading the test logs to ftp..."
|
||||
ftp -s:..\..\..\ftp.txt
|
||||
|
||||
exit 0
|
||||
62
python/llm/test/win/win_test_log.py
Normal file
62
python/llm/test/win/win_test_log.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
#
|
||||
# Copyright 2016 The BigDL Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
log_file_name = 'win_llm_test.log'
|
||||
|
||||
def create_new_log(logger_dir):
|
||||
archive_previous_log_file(logger_dir)
|
||||
new_log_path = os.path.join(logger_dir, log_file_name)
|
||||
with open(new_log_path, "w") as f:
|
||||
now = datetime.now()
|
||||
date_time = now.strftime("%Y%m%d-%H%M%S")
|
||||
f.write(date_time)
|
||||
return new_log_path
|
||||
|
||||
|
||||
def archive_previous_log_file(logger_dir):
|
||||
log_file_path = os.path.join(logger_dir, log_file_name)
|
||||
if os.path.exists(log_file_path):
|
||||
with open(log_file_path) as f:
|
||||
time_info = f.readline().strip('\n')
|
||||
log_file_name_list = log_file_name.split('.')
|
||||
new_log_file_name = log_file_name_list[0] + "_" + time_info + "." + log_file_name_list[1]
|
||||
os.makedirs(os.path.join(logger_dir, "previous_logs"), exist_ok=True)
|
||||
os.rename(log_file_path,
|
||||
os.path.join(logger_dir, "previous_logs", new_log_file_name))
|
||||
|
||||
|
||||
def manage_logs(logger_dir):
|
||||
os.makedirs(logger_dir, exist_ok=True)
|
||||
new_log_path = create_new_log(logger_dir)
|
||||
return new_log_path
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Win test logger')
|
||||
parser.add_argument('--logger_dir', type=str,
|
||||
default=r"C:\Users\obe\bigdl-llm-test\logs", required=True,
|
||||
help='The directory to log files.')
|
||||
args = parser.parse_args()
|
||||
|
||||
os.makedirs(args.logger_dir, exist_ok=True)
|
||||
create_new_log(args.logger_dir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
30
python/llm/test/win/win_test_starter.bat
Normal file
30
python/llm/test/win/win_test_starter.bat
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
@REM #
|
||||
@REM # Copyright 2016 The BigDL Authors.
|
||||
@REM #
|
||||
@REM # Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@REM # you may not use this file except in compliance with the License.
|
||||
@REM # You may obtain a copy of the License at
|
||||
@REM #
|
||||
@REM # http://www.apache.org/licenses/LICENSE-2.0
|
||||
@REM #
|
||||
@REM # Unless required by applicable law or agreed to in writing, software
|
||||
@REM # distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@REM # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
@REM # See the License for the specific language governing permissions and
|
||||
@REM # limitations under the License.
|
||||
@REM #
|
||||
|
||||
@REM Usage: call win_test_starter.bat %conda_activate_path% %base_dir%
|
||||
@REM Example: win_test_starter.bat C:\ProgramData\Anaconda3\Scripts\activate.bat C:\Users\obe\bigdl-llm-test
|
||||
|
||||
set conda_activate_path=%1
|
||||
set base_dir=%2
|
||||
|
||||
call %conda_activate_path% bigdl-llm
|
||||
python %base_dir%\BigDL\python\llm\test\win\win_test_log.py --logger_dir %base_dir%\logs
|
||||
set logger_file=%base_dir%\logs\win_llm_test.log
|
||||
call %base_dir%\BigDL\python\llm\test\win\win_env_setup_and_test.bat %base_dir% >> %logger_file% 2>&1
|
||||
|
||||
pause
|
||||
|
||||
exit 0
|
||||
Loading…
Reference in a new issue