ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

[LLM] add convert's python deps for LLM (#8260)

Browse source 
* add python deps for LLM

* update release.sh

* change deps group name

* update all

* fix update

* test fix

* update
This commit is contained in:
Pingchuan Ma (Henry) 2023-06-06 16:01:17 +08:00 committed by GitHub
parent 38be471140
commit 2ed5842448
3 changed files with 10 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.github/workflows/llm_unit_tests_basic_windows.yml vendored
View file

@ -52,12 +52,9 @@ jobs:
shell: bash shell: bash
run: | run: |
bash python/llm/dev/release_default_windows.sh default false bash python/llm/dev/release_default_windows.sh default false
pip install -i https://pypi.python.org/simple python/llm/dist/bigdl_llm*.whl whl_name=$(ls python/llm/dist)
pip install -i https://pypi.python.org/simple "python/llm/dist/${whl_name}[all]"
pip install pytest pip install pytest
pip install numpy
pip install torch
pip install transformers
pip install sentencepiece
bash python/llm/test/run-llm-basic-tests.sh bash python/llm/test/run-llm-basic-tests.sh
env: env:
ANALYTICS_ZOO_ROOT: ${{ github.workspace }} ANALYTICS_ZOO_ROOT: ${{ github.workspace }}

4
python/llm/dev/release.sh
View file

@ -43,8 +43,10 @@ echo "The effective version is: ${bigdl_version}"
if [ "$platform" == "linux" ]; then if [ "$platform" == "linux" ]; then
verbose_pname="manylinux2010_x86_64" verbose_pname="manylinux2010_x86_64"
platform_name="--linux"
elif [ "$platform" == "windows" ]; then elif [ "$platform" == "windows" ]; then
verbose_pname="win_amd64" verbose_pname="win_amd64"
platform_name="--win"
else else
echo "Unsupported platform" echo "Unsupported platform"
fi fi
@ -59,7 +61,7 @@ fi
cd $BIGDL_PYTHON_DIR cd $BIGDL_PYTHON_DIR
wheel_command="python setup.py clean --all bdist_wheel --plat-name ${verbose_pname} --python-tag py3" wheel_command="python setup.py clean --all bdist_wheel ${platform_name} --plat-name ${verbose_pname} --python-tag py3"
echo "Packing python distribution: $wheel_command" echo "Packing python distribution: $wheel_command"
${wheel_command} ${wheel_command}

5
python/llm/setup.py
View file

@ -42,6 +42,7 @@ BIGDL_PYTHON_HOME = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
VERSION = open(os.path.join(BIGDL_PYTHON_HOME, 'version.txt'), 'r').read().strip() VERSION = open(os.path.join(BIGDL_PYTHON_HOME, 'version.txt'), 'r').read().strip()
llm_home = os.path.join(os.path.dirname(os.path.abspath(__file__)), "src") llm_home = os.path.join(os.path.dirname(os.path.abspath(__file__)), "src")
libs_dir = os.path.join(llm_home, "bigdl", "llm", "libs") libs_dir = os.path.join(llm_home, "bigdl", "llm", "libs")
CONVERT_DEP = ['numpy', 'torch', 'transformers', 'sentencepiece']
def get_llm_packages(): def get_llm_packages():
@ -133,6 +134,9 @@ def setup_package():
for url in lib_urls[platform_name]: for url in lib_urls[platform_name]:
download_libs(url, change_permission=change_permission) download_libs(url, change_permission=change_permission)
all_requires = []
all_requires += CONVERT_DEP
metadata = dict( metadata = dict(
name='bigdl-llm', name='bigdl-llm',
version=VERSION, version=VERSION,
@ -147,6 +151,7 @@ def setup_package():
package_dir={"": "src"}, package_dir={"": "src"},
package_data={"bigdl.llm": package_data[platform_name]}, package_data={"bigdl.llm": package_data[platform_name]},
include_package_data=True, include_package_data=True,
extras_require={"all": all_requires},
classifiers=[ classifiers=[
'License :: OSI Approved :: Apache Software License', 'License :: OSI Approved :: Apache Software License',
'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3',