From 91a1528fce9c763722d66740be5b7e811395f28f Mon Sep 17 00:00:00 2001 From: Yina Chen <33650826+cyita@users.noreply.github.com> Date: Fri, 2 Jun 2023 13:51:35 +0800 Subject: [PATCH] [LLM]Support for linux package (llama, NeoX) & quantize (llama) (#8246) * temp * update * update * remove cmake * runtime get platform -> change platform name using sed * update * update * add platform flags(default: current platform) & delete legacy libs & add neox quantize --- python/llm/CMakeLists.txt | 58 -------------------------- python/llm/setup-linux.py | 74 --------------------------------- python/llm/setup.py | 71 +++++++++++++++++++++++++++---- python/llm/vendor/bloomz.cpp | 1 - python/llm/vendor/llama.cpp | 1 - python/llm/vendor/redpajama.cpp | 1 - 6 files changed, 62 insertions(+), 144 deletions(-) delete mode 100644 python/llm/CMakeLists.txt delete mode 100644 python/llm/setup-linux.py delete mode 160000 python/llm/vendor/bloomz.cpp delete mode 160000 python/llm/vendor/llama.cpp delete mode 160000 python/llm/vendor/redpajama.cpp diff --git a/python/llm/CMakeLists.txt b/python/llm/CMakeLists.txt deleted file mode 100644 index 460c2c1e..00000000 --- a/python/llm/CMakeLists.txt +++ /dev/null @@ -1,58 +0,0 @@ -cmake_minimum_required(VERSION 3.4...3.22) - -project(bigdl_llm) - -option(FORCE_CMAKE "Force CMake build of Related CPP project" OFF) - -set(FORCE_CMAKE $ENV{FORCE_CMAKE}) - -add_custom_command( - OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/libllama.so ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/quantize - COMMAND make libllama.so quantize - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp -) -add_custom_command( - OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/vendor/redpajama.cpp/libgptneox.so ${CMAKE_CURRENT_SOURCE_DIR}/vendor/redpajama.cpp/quantize-gptneox - COMMAND make libgptneox.so quantize-gptneox - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/vendor/redpajama.cpp -) -add_custom_command( - OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/vendor/bloomz.cpp/libbloom.so ${CMAKE_CURRENT_SOURCE_DIR}/vendor/bloomz.cpp/quantize - COMMAND make libbloom.so quantize - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/vendor/bloomz.cpp -) -add_custom_target( - run ALL - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/libllama.so - ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/quantize - ${CMAKE_CURRENT_SOURCE_DIR}/vendor/redpajama.cpp/libgptneox.so - ${CMAKE_CURRENT_SOURCE_DIR}/vendor/redpajama.cpp/quantize-gptneox - ${CMAKE_CURRENT_SOURCE_DIR}/vendor/bloomz.cpp/libbloom.so - ${CMAKE_CURRENT_SOURCE_DIR}/vendor/bloomz.cpp/quantize -) -install( - FILES ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/libllama.so - DESTINATION src/bigdl/llm/libs -) -install( - PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/quantize - RENAME quantize-llama - DESTINATION src/bigdl/llm/libs -) -install( - FILES ${CMAKE_CURRENT_SOURCE_DIR}/vendor/redpajama.cpp/libgptneox.so - DESTINATION src/bigdl/llm/libs -) -install( - PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/vendor/redpajama.cpp/quantize-gptneox - DESTINATION src/bigdl/llm/libs -) -install( - FILES ${CMAKE_CURRENT_SOURCE_DIR}/vendor/bloomz.cpp/libbloom.so - DESTINATION src/bigdl/llm/libs -) -install( - PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/vendor/bloomz.cpp/quantize - RENAME quantize-bloomz - DESTINATION src/bigdl/llm/libs -) diff --git a/python/llm/setup-linux.py b/python/llm/setup-linux.py deleted file mode 100644 index bfdc4cdb..00000000 --- a/python/llm/setup-linux.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python - -# -# Copyright 2016 The BigDL Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import fnmatch -from skbuild import setup - -long_description = ''' - BigDL LLM -''' - -exclude_patterns = ["*__pycache__*", "*ipynb_checkpoints*"] -BIGDL_PYTHON_HOME = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -VERSION = open(os.path.join(BIGDL_PYTHON_HOME, 'version.txt'), 'r').read().strip() -llm_home = os.path.join(os.path.dirname(os.path.abspath(__file__)), "src") - - -def get_llm_packages(): - llm_packages = [] - for dirpath, _, _ in os.walk(os.path.join(llm_home, "bigdl")): - print(dirpath) - package = dirpath.split(llm_home + os.sep)[1].replace(os.sep, '.') - if any(fnmatch.fnmatchcase(package, pat=pattern) - for pattern in exclude_patterns): - print("excluding", package) - else: - llm_packages.append(package) - print("including", package) - return llm_packages - - -def setup_package(): - metadata = dict( - name='bigdl-llm', - version=VERSION, - description='Large Language Model Develop Toolkit', - long_description=long_description, - long_description_content_type="text/markdown", - author='BigDL Authors', - author_email='bigdl-user-group@googlegroups.com', - license='Apache License, Version 2.0', - url='https://github.com/intel-analytics/BigDL', - packages=get_llm_packages(), - package_dir={"": "src"}, - install_requires=[""], - include_package_data=True, - classifiers=[ - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: Implementation :: CPython'], - platforms=['linux'] - ) - - setup(**metadata) - - -if __name__ == '__main__': - setup_package() diff --git a/python/llm/setup.py b/python/llm/setup.py index 6559071d..9bf0ec75 100644 --- a/python/llm/setup.py +++ b/python/llm/setup.py @@ -16,10 +16,22 @@ # limitations under the License. # +# >> Usage: +# +# >>>> Build for the current platform: +# python setup.py clean --all bdist_wheel +# >>>> Windows: +# python setup.py clean --all bdist_wheel --win +# >>>> Linux： +# python setup.py clean --all bdist_wheel --linux + import os +import sys import fnmatch from setuptools import setup import urllib.request +import platform +import shutil long_description = ''' BigDL LLM @@ -29,6 +41,7 @@ exclude_patterns = ["*__pycache__*", "*ipynb_checkpoints*"] BIGDL_PYTHON_HOME = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) VERSION = open(os.path.join(BIGDL_PYTHON_HOME, 'version.txt'), 'r').read().strip() llm_home = os.path.join(os.path.dirname(os.path.abspath(__file__)), "src") +libs_dir = os.path.join(llm_home, "bigdl", "llm", "libs") def get_llm_packages(): @@ -45,36 +58,76 @@ def get_llm_packages(): return llm_packages -lib_urls = [ +lib_urls = {} +lib_urls["Windows"] = [ "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/llama.dll", "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/quantize-llama.exe", "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/gptneox.dll", "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/quantize-gptneox.exe", # TODO: add bloomz ] +lib_urls["Linux"] = [ + "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/libllama_avx2.so", + "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/libllama_avx512.so", + "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/quantize-llama", + "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/libgptneox_avx2.so", + "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/libgptneox_avx512.so", + "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/quantize-gptneox", +] -def download_libs(url: str): - libs_dir = os.path.join(llm_home, "bigdl", "llm", "libs") - if not os.path.exists(libs_dir): - os.makedirs(libs_dir, exist_ok=True) +def download_libs(url: str, change_permission=False): libso_file_name = url.split('/')[-1] libso_file = os.path.join(libs_dir, libso_file_name) if not os.path.exists(libso_file): + print(">> Downloading from ", url) urllib.request.urlretrieve(url, libso_file) + if change_permission: + os.chmod(libso_file, 0o775) def setup_package(): - package_data = [ + package_data = {} + package_data["Windows"] = [ "libs/llama.dll", "libs/quantize-llama.exe", "libs/gptneox.dll", "libs/quantize-gptneox.exe", ] + package_data["Linux"] = [ + "libs/libllama_avx2.so", + "libs/libllama_avx512.so", + "libs/quantize-llama", + "libs/libgptneox_avx2.so", + "libs/libgptneox_avx512.so", + "libs/quantize-gptneox", + ] - for url in lib_urls: - download_libs(url) + platform_name = None + if "--win" in sys.argv: + platform_name = "Windows" + sys.argv.remove("--win") + if "--linux" in sys.argv: + platform_name = "Linux" + sys.argv.remove("--linux") + + if platform_name is None: + if platform.platform().startswith('Windows'): + platform_name = "Windows" + else: + platform_name = "Linux" + + change_permission = True if platform_name == "Linux" else False + + # Delete legacy libs + if os.path.exists(libs_dir): + print(f"Deleting existing libs_dir {libs_dir} ....") + shutil.rmtree(libs_dir) + os.makedirs(libs_dir, exist_ok=True) + + for url in lib_urls[platform_name]: + download_libs(url, change_permission=change_permission) metadata = dict( name='bigdl-llm', @@ -88,7 +141,7 @@ def setup_package(): url='https://github.com/intel-analytics/BigDL', packages=get_llm_packages(), package_dir={"": "src"}, - package_data={"bigdl.llm": package_data}, + package_data={"bigdl.llm": package_data[platform_name]}, include_package_data=True, classifiers=[ 'License :: OSI Approved :: Apache Software License', diff --git a/python/llm/vendor/bloomz.cpp b/python/llm/vendor/bloomz.cpp deleted file mode 160000 index 6d2dee07..00000000 --- a/python/llm/vendor/bloomz.cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6d2dee07626b7e8b6af1ada2bfb1bb17de4deaea diff --git a/python/llm/vendor/llama.cpp b/python/llm/vendor/llama.cpp deleted file mode 160000 index 5026ad5e..00000000 --- a/python/llm/vendor/llama.cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5026ad5e525e3bb8354d9169aaf9b147ef8b6b85 diff --git a/python/llm/vendor/redpajama.cpp b/python/llm/vendor/redpajama.cpp deleted file mode 160000 index bec989c2..00000000 --- a/python/llm/vendor/redpajama.cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bec989c272fd733d1db1069fb47ad5008ae1cd7e