ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

LLM: binary file version control on source forge (#8329)

Browse source 
* support version control for llm based on date

* update action
This commit is contained in:
Ruonan Wang 2023-06-15 09:53:27 +08:00 committed by GitHub
parent 5094970175
commit 8840dadd86
3 changed files with 62 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/llm_unit_tests_basic_linux.yml vendored
View file

@ -46,6 +46,7 @@ jobs:
$CONDA_HOME/bin/conda create -n bigdl-init-llm -y python==3.9 setuptools==58.0.4 -c ${GONDOLIN_CONDA_CHANNEL} --override-channels $CONDA_HOME/bin/conda create -n bigdl-init-llm -y python==3.9 setuptools==58.0.4 -c ${GONDOLIN_CONDA_CHANNEL} --override-channels
source $CONDA_HOME/bin/activate bigdl-init-llm source $CONDA_HOME/bin/activate bigdl-init-llm
$CONDA_HOME/bin/conda info $CONDA_HOME/bin/conda info
pip install requests
bash python/llm/dev/release_default_linux.sh default false bash python/llm/dev/release_default_linux.sh default false
pip install -i https://pypi.python.org/simple python/llm/dist/bigdl_llm*.whl pip install -i https://pypi.python.org/simple python/llm/dist/bigdl_llm*.whl
source $CONDA_HOME/bin/deactivate source $CONDA_HOME/bin/deactivate
@ -60,6 +61,7 @@ jobs:
$CONDA_HOME/bin/conda create -n bigdl-init-llm -y python==3.9 setuptools==58.0.4 -c ${GONDOLIN_CONDA_CHANNEL} --override-channels $CONDA_HOME/bin/conda create -n bigdl-init-llm -y python==3.9 setuptools==58.0.4 -c ${GONDOLIN_CONDA_CHANNEL} --override-channels
source $CONDA_HOME/bin/activate bigdl-init-llm source $CONDA_HOME/bin/activate bigdl-init-llm
$CONDA_HOME/bin/conda info $CONDA_HOME/bin/conda info
pip install requests
bash python/llm/dev/release_default_linux.sh default false bash python/llm/dev/release_default_linux.sh default false
whl_name=$(ls python/llm/dist) whl_name=$(ls python/llm/dist)
pip install -i https://pypi.python.org/simple "python/llm/dist/${whl_name}[all]" pip install -i https://pypi.python.org/simple "python/llm/dist/${whl_name}[all]"

2
.github/workflows/llm_unit_tests_basic_windows.yml vendored
View file

@ -43,6 +43,7 @@ jobs:
- name: Run LLM-init test - name: Run LLM-init test
shell: bash shell: bash
run: | run: |
pip install requests
bash python/llm/dev/release_default_windows.sh default false bash python/llm/dev/release_default_windows.sh default false
pip install -i https://pypi.python.org/simple python/llm/dist/bigdl_llm*.whl pip install -i https://pypi.python.org/simple python/llm/dist/bigdl_llm*.whl
env: env:
@ -51,6 +52,7 @@ jobs:
- name: Run LLM naive installation test - name: Run LLM naive installation test
shell: bash shell: bash
run: | run: |
pip install requests
bash python/llm/dev/release_default_windows.sh default false bash python/llm/dev/release_default_windows.sh default false
whl_name=$(ls python/llm/dist) whl_name=$(ls python/llm/dist)
pip install -i https://pypi.python.org/simple "python/llm/dist/${whl_name}[all]" pip install -i https://pypi.python.org/simple "python/llm/dist/${whl_name}[all]"

87
python/llm/setup.py
View file

@ -31,6 +31,8 @@ import platform
import shutil import shutil
import sys import sys
import urllib.request import urllib.request
import requests
import re
from setuptools import setup from setuptools import setup
@ -60,35 +62,60 @@ def get_llm_packages():
return llm_packages return llm_packages
lib_urls = {} def obtain_lib_urls():
lib_urls["Windows"] = [ base_url = "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/"
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/llama.dll", windows_binarys = ["llama.dll", "gptneox.dll", "bloom.dll",
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/quantize-llama.exe", "quantize-llama.exe", "quantize-gptneox.exe", "quantize-bloom.exe",
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/gptneox.dll", "main-llama.exe", "main-gptneox.exe", "main-bloom.exe"]
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/quantize-gptneox.exe", linux_binarys = ["libllama_avx2.so", "libgptneox_avx2.so", "libbloom_avx2.so",
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/bloom.dll", "libllama_avx512.so", "libgptneox_avx512.so", "libbloom_avx512.so",
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/quantize-bloom.exe", "quantize-llama", "quantize-gptneox", "quantize-bloom",
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/main-llama.exe", "main-llama_avx2", "main-gptneox_avx2", "main-bloom_avx2",
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/main-bloom.exe", "main-llama_avx512", "main-gptneox_avx512", "main-bloom_avx512"]
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/main-gptneox.exe",
] def get_date_urls(base_url):
lib_urls["Linux"] = [ # obtain all urls based on date(format: xxxx-xx-xx)
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/libllama_avx2.so", text = ''
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/libllama_avx512.so", try:
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/quantize-llama", text = requests.get(base_url).text
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/libgptneox_avx2.so", except Exception as e:
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/libgptneox_avx512.so", print("error - > ",base_url,e)
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/quantize-gptneox", pass
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/libbloom_avx2.so", reg = "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}/"
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/libbloom_avx512.so", urls = re.findall(reg, text)
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/main-llama_avx2", return urls
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/main-bloom_avx2",
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/main-gptneox_avx2", def get_urls_for_binary(date_urls, binarys):
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/main-llama_avx512", # Sort by time from near to far
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/main-bloom_avx512", date_urls = sorted(date_urls, reverse=True)
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/main-gptneox_avx512", binary_url = {}
"https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/quantize-bloom", download_num = len(binarys)
] for url in date_urls:
try:
text = requests.get(url).text
except Exception as e:
print("error - > ", url, e)
continue
for binary in binarys:
if binary in binary_url:
continue
if binary in text:
lib_url = url + binary
binary_url[binary] = lib_url
download_num -= 1
if download_num == 0:
break
if download_num == 0:
break
return binary_url
lib_urls = {}
date_urls = get_date_urls(base_url)
windows_binary_urls = get_urls_for_binary(date_urls, windows_binarys)
lib_urls["Windows"] = list(windows_binary_urls.values())
linux_binary_urls = get_urls_for_binary(date_urls, linux_binarys)
lib_urls["Linux"] = list(linux_binary_urls.values())
return lib_urls
def download_libs(url: str, change_permission=False): def download_libs(url: str, change_permission=False):
@ -154,6 +181,8 @@ def setup_package():
shutil.rmtree(libs_dir) shutil.rmtree(libs_dir)
os.makedirs(libs_dir, exist_ok=True) os.makedirs(libs_dir, exist_ok=True)
lib_urls = obtain_lib_urls()
for url in lib_urls[platform_name]: for url in lib_urls[platform_name]:
download_libs(url, change_permission=change_permission) download_libs(url, change_permission=change_permission)