From 3361b66449c7af5c8f2a2395f5de95e92f78e1d3 Mon Sep 17 00:00:00 2001 From: xingyuan li <108672484+hoshibara@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:15:44 +0900 Subject: [PATCH] [LLM] Revert llm-cli to disable selecting executables on Windows (#8630) * revert vnni file select * revert setup.py * add model-api.dll --- .github/workflows/llm-binary-build.yml | 8 +- python/llm/setup.py | 138 ++++++++++------------ python/llm/src/bigdl/llm/cli/llm-chat.ps1 | 4 +- python/llm/src/bigdl/llm/cli/llm-cli.ps1 | 8 +- 4 files changed, 75 insertions(+), 83 deletions(-) diff --git a/.github/workflows/llm-binary-build.yml b/.github/workflows/llm-binary-build.yml index 83f99a35..9df8b36a 100644 --- a/.github/workflows/llm-binary-build.yml +++ b/.github/workflows/llm-binary-build.yml @@ -226,19 +226,19 @@ jobs: run: | if (Test-Path ./release) { rm -r -fo release } mkdir release - mv build/Release/main-bloom.exe release/main-bloom_vnni.exe + # mv build/Release/main-bloom.exe release/main-bloom_vnni.exe mv build/Release/quantize-bloom.exe release/quantize-bloom_vnni.exe mv build/Release/bloom.dll release/libbloom_vnni.dll - mv build/Release/main-llama.exe release/main-llama_vnni.exe + # mv build/Release/main-llama.exe release/main-llama_vnni.exe mv build/Release/quantize-llama.exe release/quantize-llama_vnni.exe mv build/Release/llama.dll release/libllama_vnni.dll - mv build/Release/main-gptneox.exe release/main-gptneox_vnni.exe + # mv build/Release/main-gptneox.exe release/main-gptneox_vnni.exe mv build/Release/quantize-gptneox.exe release/quantize-gptneox_vnni.exe mv build/Release/gptneox.dll release/libgptneox_vnni.dll - mv build/Release/main-starcoder.exe release/main-starcoder_vnni.exe + # mv build/Release/main-starcoder.exe release/main-starcoder_vnni.exe mv build/Release/quantize-starcoder.exe release/quantize-starcoder_vnni.exe mv build/Release/starcoder.dll release/libstarcoder_vnni.dll diff --git a/python/llm/setup.py b/python/llm/setup.py index 8913a758..c7237719 100644 --- a/python/llm/setup.py +++ b/python/llm/setup.py @@ -43,11 +43,61 @@ long_description = ''' exclude_patterns = ["*__pycache__*", "*ipynb_checkpoints*"] BIGDL_PYTHON_HOME = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -VERSION = open(os.path.join(BIGDL_PYTHON_HOME, 'version.txt'), 'r').read().strip() +VERSION = open(os.path.join(BIGDL_PYTHON_HOME, + 'version.txt'), 'r').read().strip() llm_home = os.path.join(os.path.dirname(os.path.abspath(__file__)), "src") github_artifact_dir = os.path.join(llm_home, '../llm-binary') libs_dir = os.path.join(llm_home, "bigdl", "llm", "libs") -CONVERT_DEP = ['numpy >= 1.22', 'torch', 'transformers', 'sentencepiece', 'accelerate'] +CONVERT_DEP = ['numpy >= 1.22', 'torch', + 'transformers', 'sentencepiece', 'accelerate'] +windows_binarys = [ + "llama.dll", + "gptneox.dll", + "bloom.dll", + "starcoder.dll", + "llama-api.dll", + "gptneox-api.dll", + "bloom-api.dll", + "starcoder-api.dll", + "quantize-llama.exe", + "quantize-gptneox.exe", + "quantize-bloom.exe", + "quantize-starcoder.exe", + "main-llama.exe", + "main-gptneox.exe", + "main-bloom.exe", + "main-starcoder.exe", + "libllama_vnni.dll", + "libgptneox_vnni.dll", + "libbloom_vnni.dll", + "libstarcoder_vnni.dll", + "quantize-llama_vnni.exe", + "quantize-gptneox_vnni.exe", + "quantize-bloom_vnni.exe", + "quantize-starcoder_vnni.exe", +] +linux_binarys = [ + "libllama_avx2.so", + "libgptneox_avx2.so", + "libbloom_avx2.so", + "libllama_avx512.so", + "libgptneox_avx512.so", + "libbloom_avx512.so", + "quantize-llama", + "quantize-gptneox", + "quantize-bloom", + "main-llama_avx2", + "main-gptneox_avx2", + "main-bloom_avx2", + "main-llama_avx512", + "main-gptneox_avx512", + "main-bloom_avx512", + "libstarcoder_avx512.so", + "main-starcoder_avx512", + "quantize-starcoder", + "libstarcoder_avx2.so", + "main-starcoder_avx2" +] def get_llm_packages(): @@ -66,21 +116,6 @@ def get_llm_packages(): def obtain_lib_urls(): base_url = "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/" - windows_binarys = ["llama.dll", "gptneox.dll", "bloom.dll", - "quantize-llama.exe", "quantize-gptneox.exe", "quantize-bloom.exe", - "main-llama.exe", "main-gptneox.exe", "main-bloom.exe", - "starcoder.dll", "quantize-starcoder.exe", "main-starcoder.exe", - "libllama_vnni.dll", "libgptneox_vnni.dll", "libbloom_vnni.dll", - "quantize-llama_vnni.exe", "quantize-gptneox_vnni.exe", "quantize-bloom_vnni.exe", - "main-llama_vnni.exe", "main-gptneox_vnni.exe", "main-bloom_vnni.exe", - "libstarcoder_vnni.dll", "quantize-starcoder_vnni.exe", "main-starcoder_vnni.exe"] - linux_binarys = ["libllama_avx2.so", "libgptneox_avx2.so", "libbloom_avx2.so", - "libllama_avx512.so", "libgptneox_avx512.so", "libbloom_avx512.so", - "quantize-llama", "quantize-gptneox", "quantize-bloom", - "main-llama_avx2", "main-gptneox_avx2", "main-bloom_avx2", - "main-llama_avx512", "main-gptneox_avx512", "main-bloom_avx512", - "libstarcoder_avx512.so", "main-starcoder_avx512", "quantize-starcoder", - "libstarcoder_avx2.so", "main-starcoder_avx2"] def get_date_urls(base_url): # obtain all urls based on date(format: xxxx-xx-xx) @@ -88,10 +123,10 @@ def obtain_lib_urls(): try: text = requests.get(base_url).text except Exception as e: - print("error - > ",base_url,e) + print("error - > ", base_url, e) pass reg = "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}/" - urls = re.findall(reg, text) + urls = re.findall(reg, text) return urls def get_urls_for_binary(date_urls, binarys): @@ -143,55 +178,10 @@ def download_libs(url: str, change_permission=False): def setup_package(): package_data = {} - package_data["Windows"] = [ - "libs/llama.dll", - "libs/gptneox.dll", - "libs/bloom.dll", - "libs/starcoder.dll", - "libs/quantize-llama.exe", - "libs/quantize-gptneox.exe", - "libs/quantize-bloom.exe", - "libs/quantize-starcoder.exe", - "libs/main-bloom.exe", - "libs/main-gptneox.exe", - "libs/main-llama.exe", - "libs/main-starcoder.exe", - "libs/libllama_vnni.dll", - "libs/libgptneox_vnni.dll", - "libs/libbloom_vnni.dll", - "libs/libstarcoder_vnni.dll", - "libs/quantize-llama_vnni.exe", - "libs/quantize-gptneox_vnni.exe", - "libs/quantize-bloom_vnni.exe", - "libs/quantize-starcoder_vnni.exe", - "libs/main-llama_vnni.exe", - "libs/main-gptneox_vnni.exe", - "libs/main-bloom_vnni.exe", - "libs/main-starcoder_vnni.exe" - ] - package_data["Linux"] = [ - "libs/libllama_avx2.so", - "libs/libllama_avx512.so", - "libs/quantize-llama", - "libs/libgptneox_avx2.so", - "libs/libgptneox_avx512.so", - "libs/quantize-gptneox", - "libs/libbloom_avx2.so", - "libs/libbloom_avx512.so", - "libs/quantize-bloom", - "libs/libstarcoder_avx512.so", - "libs/libstarcoder_avx2.so", - "libs/quantize-starcoder", - "libs/main-bloom_avx2", - "libs/main-bloom_avx512", - "libs/main-gptneox_avx2", - "libs/main-gptneox_avx512", - "libs/main-llama_avx2", - "libs/main-llama_avx512", - "libs/main-starcoder_avx512", - "libs/main-starcoder_avx2", - ] - + package_data["Windows"] = list(map(lambda x: os.path.join('libs', x), + windows_binarys)) + package_data["Linux"] = list(map(lambda x: os.path.join('libs', x), + linux_binarys)) platform_name = None if "--win" in sys.argv: platform_name = "Windows" @@ -213,9 +203,9 @@ def setup_package(): print(f"Deleting existing libs_dir {libs_dir} ....") shutil.rmtree(libs_dir) os.makedirs(libs_dir, exist_ok=True) - + # copy built files for github workflow - for built_file in glob.glob(os.path.join(github_artifact_dir,'*')): + for built_file in glob.glob(os.path.join(github_artifact_dir, '*')): print(f'Copy workflow built file: {built_file}') shutil.copy(built_file, libs_dir) @@ -223,13 +213,14 @@ def setup_package(): for url in lib_urls[platform_name]: download_libs(url, change_permission=change_permission) - + # Check if all package files are ready for file in package_data[platform_name]: file_path = os.path.join(libs_dir, os.path.basename(file)) if not os.path.exists(file_path): print(f'Could not find package dependency file: {file_path}') - raise FileNotFoundError(f'Could not find package dependency file: {file_path}') + raise FileNotFoundError( + f'Could not find package dependency file: {file_path}') all_requires = ['py-cpuinfo'] all_requires += CONVERT_DEP @@ -246,7 +237,8 @@ def setup_package(): url='https://github.com/intel-analytics/BigDL', packages=get_llm_packages(), package_dir={"": "src"}, - package_data={"bigdl.llm": package_data[platform_name] + ["cli/prompts/*.txt"]}, + package_data={ + "bigdl.llm": package_data[platform_name] + ["cli/prompts/*.txt"]}, include_package_data=True, entry_points={ "console_scripts": [ diff --git a/python/llm/src/bigdl/llm/cli/llm-chat.ps1 b/python/llm/src/bigdl/llm/cli/llm-chat.ps1 index 06ffe82b..b3ff0403 100644 --- a/python/llm/src/bigdl/llm/cli/llm-chat.ps1 +++ b/python/llm/src/bigdl/llm/cli/llm-chat.ps1 @@ -24,7 +24,7 @@ function Display-Help function llama { - $exec_file = if ($vnni_enable) { "main-llama_vnni.exe" } else { "main-llama.exe" } + $exec_file = "main-llama.exe" $prompt_file = Join-Path $prompt_dir "chat-with-llm.txt" $command = "$lib_dir/$exec_file -t $threads -n $n_predict -f $prompt_file -i --color --reverse-prompt 'USER:' --in-prefix ' ' $filteredArguments" Write-Host "$command" @@ -33,7 +33,7 @@ function llama function gptneox { - $exec_file = if ($vnni_enable) { "main-gptneox_vnni.exe" } else { "main-gptneox.exe" } + $exec_file = "main-gptneox.exe" $prompt = "A chat between a curious human and an artificial intelligence assistant.` The assistant gives helpful, detailed, and polite answers." $command = "$lib_dir/$exec_file -t $threads -n $n_predict --color --instruct -p '$prompt' $filteredArguments" diff --git a/python/llm/src/bigdl/llm/cli/llm-cli.ps1 b/python/llm/src/bigdl/llm/cli/llm-cli.ps1 index f541d54b..40160e10 100755 --- a/python/llm/src/bigdl/llm/cli/llm-cli.ps1 +++ b/python/llm/src/bigdl/llm/cli/llm-cli.ps1 @@ -23,7 +23,7 @@ function Display-Help function llama { - $exec_file = if ($vnni_enable) { "main-llama_vnni.exe" } else { "main-llama.exe" } + $exec_file = "main-llama.exe" $command = "$lib_dir/$exec_file -t $threads -n $n_predict $filteredArguments" Write-Host "$command" Invoke-Expression $command @@ -31,7 +31,7 @@ function llama function bloom { - $exec_file = if ($vnni_enable) { "main-bloom_vnni.exe" } else { "main-bloom.exe" } + $exec_file = "main-bloom.exe" $command = "$lib_dir/$exec_file -t $threads -n $n_predict $filteredArguments" Write-Host "$command" Invoke-Expression $command @@ -39,7 +39,7 @@ function bloom function gptneox { - $exec_file = if ($vnni_enable) { "main-gptneox_vnni.exe" } else { "main-gptneox.exe" } + $exec_file = "main-gptneox.exe" $command = "$lib_dir/$exec_file -t $threads -n $n_predict $filteredArguments" Write-Host "$command" Invoke-Expression $command @@ -47,7 +47,7 @@ function gptneox function starcoder { - $exec_file = if ($vnni_enable) { "main-starcoder_vnni.exe" } else { "main-starcoder.exe" } + $exec_file = "main-starcoder.exe" $command = "$lib_dir/$exec_file -t $threads -n $n_predict $filteredArguments" Write-Host "$command" Invoke-Expression $command