ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Add speculative support in docker (#10058)

Browse source 
* add speculative environment

* add speculative environment

* add speculative environment
This commit is contained in:
Lilac09 2024-02-01 09:53:53 +08:00 committed by GitHub
parent 4b92235bdb
commit 72e67eedbb
1 changed files with 21 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
docker/llm/inference/cpu/docker/Dockerfile
View file

@ -47,6 +47,26 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get update && \
pip install --upgrade transformers_stream_generator einops && \ pip install --upgrade transformers_stream_generator einops && \
# Copy vLLM-Serving # Copy vLLM-Serving
cp -r ./BigDL/python/llm/example/CPU/vLLM-Serving/ ./vLLM-Serving && \ cp -r ./BigDL/python/llm/example/CPU/vLLM-Serving/ ./vLLM-Serving && \
rm -rf ./BigDL rm -rf ./BigDL && \
# Install miniconda
curl -LO "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" && \
bash Miniconda3-latest-Linux-x86_64.sh -b && \
rm -f Miniconda3-latest-Linux-x86_64.sh && \
# Install environment for speculative
/root/miniconda3/condabin/conda init bash && \
/bin/bash -c "source /root/.bashrc" && \
/root/miniconda3/condabin/conda create -n bigdl-speculative-py39 -y python=3.9
SHELL ["/root/miniconda3/condabin/conda", "run", "-n", "bigdl-speculative-py39", "/bin/bash", "-c"]
RUN cd /llm && \
pip3 install --no-cache-dir --upgrade torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu && \
pip install --pre --upgrade bigdl-llm[all] && \
wget https://raw.githubusercontent.com/intel/intel-extension-for-pytorch/0c63936d7a6740679987920367ae2e0cdb375b2e/scripts/compile_bundle.sh && \
sed -i 's/VER_IPEX=main/VER_IPEX=0c63936d7a6740679987920367ae2e0cdb375b2e/g' "compile_bundle.sh" && \
bash compile_bundle.sh && \
cd ./intel-extension-for-pytorch && \
pip install -r requirements.txt && \
pip install transformers==4.36.2 && \
pip install transformers_stream_generator && \
rm -rf llvm-project llvm-release intel-extension-for-pytorch compile_bundle.sh
ENTRYPOINT ["/bin/bash"] ENTRYPOINT ["/bin/bash"]