upgrade python (#10769)
This commit is contained in:
Shaojun Liu
GitHub
parent
1abd77507e
commit
7297036c03
4 changed files with 62 additions and 29 deletions
|
|
@ -19,10 +19,24 @@ COPY --from=key-getter /root/intel-oneapi-archive-keyring.gpg /usr/share/keyring
|
|||
RUN echo "deb [signed-by=/usr/share/keyrings/intel-oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list
|
||||
|
||||
RUN mkdir /ipex_llm/data && mkdir /ipex_llm/model && \
|
||||
# Install python 3.11.1
|
||||
apt-get update && apt-get install -y curl wget gpg gpg-agent software-properties-common git gcc g++ make libunwind8-dev zlib1g-dev libssl-dev libffi-dev && \
|
||||
mkdir -p /opt/python && \
|
||||
cd /opt/python && \
|
||||
wget https://www.python.org/ftp/python/3.11.1/Python-3.11.1.tar.xz && \
|
||||
tar -xf Python-3.11.1.tar.xz && \
|
||||
cd Python-3.11.1 && \
|
||||
./configure --enable-optimizations --with-zlib && \
|
||||
make altinstall && \
|
||||
rm /usr/bin/python3 && \
|
||||
# Create a symbolic link pointing to Python 3.11 at /usr/bin/python3
|
||||
ln -s /opt/python/Python-3.11.1/python /usr/bin/python3 && \
|
||||
# Create a symbolic link pointing to /usr/bin/python3 at /usr/bin/python
|
||||
ln -s /usr/bin/python3 /usr/bin/python && \
|
||||
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
|
||||
python3 get-pip.py && \
|
||||
# install pytorch 2.0.1
|
||||
apt-get update && \
|
||||
apt-get install -y python3-pip python3.9-dev python3-wheel git software-properties-common && \
|
||||
pip3 install --upgrade pip && \
|
||||
pip install --upgrade pip && \
|
||||
export PIP_DEFAULT_TIMEOUT=100 && \
|
||||
pip install --upgrade torch==2.1.0 --index-url https://download.pytorch.org/whl/cpu && \
|
||||
# install ipex and oneccl
|
||||
|
|
@ -35,12 +49,6 @@ RUN mkdir /ipex_llm/data && mkdir /ipex_llm/model && \
|
|||
git reset --hard 057e1d74733f52817dc05b673a340b4e3ebea08c && \
|
||||
pip install . && \
|
||||
pip install -r /ipex_llm/requirements.txt && \
|
||||
# install python
|
||||
add-apt-repository ppa:deadsnakes/ppa -y && \
|
||||
apt-get install -y python3.9 && \
|
||||
rm /usr/bin/python3 && \
|
||||
ln -s /usr/bin/python3.9 /usr/bin/python3 && \
|
||||
ln -s /usr/bin/python3 /usr/bin/python && \
|
||||
pip install --no-cache requests argparse cryptography==3.3.2 urllib3 && \
|
||||
pip install --upgrade requests && \
|
||||
pip install setuptools==58.4.0 && \
|
||||
|
|
|
|||
|
|
@ -19,28 +19,38 @@ COPY --from=key-getter /root/intel-oneapi-archive-keyring.gpg /usr/share/keyring
|
|||
RUN echo "deb [signed-by=/usr/share/keyrings/intel-oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list
|
||||
|
||||
RUN mkdir -p /ipex_llm/data && mkdir -p /ipex_llm/model && \
|
||||
# install pytorch 2.1.0
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends python3-pip python3.9-dev python3-wheel python3.9-distutils git software-properties-common && \
|
||||
# Install python 3.11.1
|
||||
apt-get update && apt-get install -y curl wget gpg gpg-agent software-properties-common git gcc g++ make libunwind8-dev zlib1g-dev libssl-dev libffi-dev && \
|
||||
mkdir -p /opt/python && \
|
||||
cd /opt/python && \
|
||||
wget https://www.python.org/ftp/python/3.11.1/Python-3.11.1.tar.xz && \
|
||||
tar -xf Python-3.11.1.tar.xz && \
|
||||
cd Python-3.11.1 && \
|
||||
./configure --enable-optimizations --with-zlib && \
|
||||
make altinstall && \
|
||||
rm /usr/bin/python3 && \
|
||||
# Create a symbolic link pointing to Python 3.11 at /usr/bin/python3
|
||||
ln -s /opt/python/Python-3.11.1/python /usr/bin/python3 && \
|
||||
# Create a symbolic link pointing to /usr/bin/python3 at /usr/bin/python
|
||||
ln -s /usr/bin/python3 /usr/bin/python && \
|
||||
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
|
||||
python3 get-pip.py && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
pip3 install --upgrade pip && \
|
||||
pip install --upgrade pip && \
|
||||
export PIP_DEFAULT_TIMEOUT=100 && \
|
||||
pip install --upgrade torch==2.1.0 && \
|
||||
# install CPU ipex-llm
|
||||
pip3 install --pre --upgrade ipex-llm[all] && \
|
||||
pip install --pre --upgrade ipex-llm[all] && \
|
||||
# install ipex and oneccl
|
||||
pip install https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/cpu/intel_extension_for_pytorch-2.1.0%2Bcpu-cp39-cp39-linux_x86_64.whl && \
|
||||
pip install https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/cpu/intel_extension_for_pytorch-2.1.0%2Bcpu-cp311-cp311-linux_x86_64.whl && \
|
||||
pip install oneccl_bind_pt -f https://developer.intel.com/ipex-whl-stable && \
|
||||
# install huggingface dependencies
|
||||
pip install datasets transformers==4.35.0 && \
|
||||
pip install fire peft==0.5.0 && \
|
||||
pip install accelerate==0.23.0 && \
|
||||
pip install bitsandbytes && \
|
||||
# install basic dependencies
|
||||
apt-get update && apt-get install -y curl wget gpg gpg-agent software-properties-common libunwind8-dev && \
|
||||
# get qlora example code
|
||||
ln -s /usr/bin/python3 /usr/bin/python && \
|
||||
cd /ipex_llm && \
|
||||
git clone https://github.com/intel-analytics/IPEX-LLM.git && \
|
||||
mv IPEX-LLM/python/llm/example/CPU/QLoRA-FineTuning/* . && \
|
||||
|
|
|
|||
|
|
@ -20,9 +20,24 @@ COPY --from=key-getter /root/intel-oneapi-archive-keyring.gpg /usr/share/keyring
|
|||
RUN echo "deb [signed-by=/usr/share/keyrings/intel-oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list
|
||||
|
||||
RUN mkdir -p /ipex_llm/data && mkdir -p /ipex_llm/model && \
|
||||
apt-get update && \
|
||||
apt install -y --no-install-recommends openssh-server openssh-client libcap2-bin gnupg2 ca-certificates \
|
||||
python3-pip python3.9-dev python3-wheel python3.9-distutils git software-properties-common && \
|
||||
# Install python 3.11.1
|
||||
apt-get update && apt-get install -y openssh-server openssh-client libcap2-bin gnupg2 ca-certificates \
|
||||
curl wget gpg gpg-agent software-properties-common git \
|
||||
gcc g++ make libunwind8-dev zlib1g-dev libssl-dev libffi-dev && \
|
||||
mkdir -p /opt/python && \
|
||||
cd /opt/python && \
|
||||
wget https://www.python.org/ftp/python/3.11.1/Python-3.11.1.tar.xz && \
|
||||
tar -xf Python-3.11.1.tar.xz && \
|
||||
cd Python-3.11.1 && \
|
||||
./configure --enable-optimizations --with-zlib && \
|
||||
make altinstall && \
|
||||
rm /usr/bin/python3 && \
|
||||
# Create a symbolic link pointing to Python 3.11 at /usr/bin/python3
|
||||
ln -s /opt/python/Python-3.11.1/python /usr/bin/python3 && \
|
||||
# Create a symbolic link pointing to /usr/bin/python3 at /usr/bin/python
|
||||
ln -s /usr/bin/python3 /usr/bin/python && \
|
||||
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
|
||||
python3 get-pip.py && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
mkdir -p /var/run/sshd && \
|
||||
|
|
@ -37,11 +52,11 @@ RUN mkdir -p /ipex_llm/data && mkdir -p /ipex_llm/model && \
|
|||
cp -r /etc/ssh/sshd_config /home/mpiuser/.sshd_config && \
|
||||
echo "Port $port" >> /home/mpiuser/.sshd_config && \
|
||||
# install pytorch 2.1.0
|
||||
pip3 install --upgrade pip && \
|
||||
pip install --upgrade pip && \
|
||||
export PIP_DEFAULT_TIMEOUT=100 && \
|
||||
pip install --upgrade torch==2.1.0 --index-url https://download.pytorch.org/whl/cpu && \
|
||||
# install CPU ipex-llm
|
||||
pip3 install --pre --upgrade ipex-llm[all] && \
|
||||
pip install --pre --upgrade ipex-llm[all] && \
|
||||
# install ipex and oneccl
|
||||
pip install intel_extension_for_pytorch==2.0.100 && \
|
||||
pip install oneccl_bind_pt -f https://developer.intel.com/ipex-whl-stable && \
|
||||
|
|
@ -50,7 +65,7 @@ RUN mkdir -p /ipex_llm/data && mkdir -p /ipex_llm/model && \
|
|||
pip install fire peft==0.5.0 && \
|
||||
pip install accelerate==0.23.0 && \
|
||||
# install basic dependencies
|
||||
apt-get update && apt-get install -y curl wget gpg gpg-agent software-properties-common libunwind8-dev && \
|
||||
apt-get update && apt-get install -y curl wget gpg gpg-agent && \
|
||||
# Install Intel oneAPI keys.
|
||||
apt remove -y gnupg2 ca-certificates && \
|
||||
apt autoremove -y && \
|
||||
|
|
@ -58,7 +73,6 @@ RUN mkdir -p /ipex_llm/data && mkdir -p /ipex_llm/model && \
|
|||
apt install -y --no-install-recommends dnsutils intel-oneapi-mpi && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
# get qlora example code
|
||||
ln -s /usr/bin/python3 /usr/bin/python && \
|
||||
cd /ipex_llm && \
|
||||
git clone https://github.com/intel-analytics/IPEX-LLM.git && \
|
||||
mv IPEX-LLM/python/llm/example/CPU/QLoRA-FineTuning/* . && \
|
||||
|
|
|
|||
|
|
@ -17,17 +17,18 @@ RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-P
|
|||
# install basic dependencies
|
||||
apt-get install -y curl wget git gnupg gpg-agent software-properties-common libunwind8-dev vim less && \
|
||||
# install Intel GPU driver
|
||||
apt-get install -y intel-opencl-icd intel-level-zero-gpu=1.3.26241.33-647~22.04 level-zero level-zero-dev --allow-downgrades && \
|
||||
apt-get install -y intel-opencl-icd intel-level-zero-gpu level-zero level-zero-dev --allow-downgrades && \
|
||||
# install python 3.9
|
||||
ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \
|
||||
env DEBIAN_FRONTEND=noninteractive apt-get update && \
|
||||
add-apt-repository ppa:deadsnakes/ppa -y && \
|
||||
apt-get install -y python3.9 && \
|
||||
apt-get install -y python3.11 && \
|
||||
rm /usr/bin/python3 && \
|
||||
ln -s /usr/bin/python3.9 /usr/bin/python3 && \
|
||||
ln -s /usr/bin/python3.11 /usr/bin/python3 && \
|
||||
ln -s /usr/bin/python3 /usr/bin/python && \
|
||||
apt-get install -y python3-pip python3.9-dev python3-wheel python3.9-distutils && \
|
||||
apt-get install -y python3-pip python3.11-dev python3-wheel python3.11-distutils && \
|
||||
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
|
||||
python3 get-pip.py && \
|
||||
# install XPU ipex-llm
|
||||
pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ && \
|
||||
# install huggingface dependencies
|
||||
|
|
|
|||
Loading…
Reference in a new issue