From 7297036c034348757c1e91a0f737c0fb6cb2f446 Mon Sep 17 00:00:00 2001 From: Shaojun Liu <61072813+liu-shaojun@users.noreply.github.com> Date: Tue, 16 Apr 2024 09:28:10 +0800 Subject: [PATCH] upgrade python (#10769) --- .../llm/finetune/lora/cpu/docker/Dockerfile | 26 +++++++++++------ .../llm/finetune/qlora/cpu/docker/Dockerfile | 28 +++++++++++++------ .../finetune/qlora/cpu/docker/Dockerfile.k8s | 28 ++++++++++++++----- .../llm/finetune/qlora/xpu/docker/Dockerfile | 9 +++--- 4 files changed, 62 insertions(+), 29 deletions(-) diff --git a/docker/llm/finetune/lora/cpu/docker/Dockerfile b/docker/llm/finetune/lora/cpu/docker/Dockerfile index 4b6f51b9..0dd34e70 100644 --- a/docker/llm/finetune/lora/cpu/docker/Dockerfile +++ b/docker/llm/finetune/lora/cpu/docker/Dockerfile @@ -19,10 +19,24 @@ COPY --from=key-getter /root/intel-oneapi-archive-keyring.gpg /usr/share/keyring RUN echo "deb [signed-by=/usr/share/keyrings/intel-oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list RUN mkdir /ipex_llm/data && mkdir /ipex_llm/model && \ +# Install python 3.11.1 + apt-get update && apt-get install -y curl wget gpg gpg-agent software-properties-common git gcc g++ make libunwind8-dev zlib1g-dev libssl-dev libffi-dev && \ + mkdir -p /opt/python && \ + cd /opt/python && \ + wget https://www.python.org/ftp/python/3.11.1/Python-3.11.1.tar.xz && \ + tar -xf Python-3.11.1.tar.xz && \ + cd Python-3.11.1 && \ + ./configure --enable-optimizations --with-zlib && \ + make altinstall && \ + rm /usr/bin/python3 && \ + # Create a symbolic link pointing to Python 3.11 at /usr/bin/python3 + ln -s /opt/python/Python-3.11.1/python /usr/bin/python3 && \ + # Create a symbolic link pointing to /usr/bin/python3 at /usr/bin/python + ln -s /usr/bin/python3 /usr/bin/python && \ + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py && \ # install pytorch 2.0.1 - apt-get update && \ - apt-get install -y python3-pip python3.9-dev python3-wheel git software-properties-common && \ - pip3 install --upgrade pip && \ + pip install --upgrade pip && \ export PIP_DEFAULT_TIMEOUT=100 && \ pip install --upgrade torch==2.1.0 --index-url https://download.pytorch.org/whl/cpu && \ # install ipex and oneccl @@ -35,12 +49,6 @@ RUN mkdir /ipex_llm/data && mkdir /ipex_llm/model && \ git reset --hard 057e1d74733f52817dc05b673a340b4e3ebea08c && \ pip install . && \ pip install -r /ipex_llm/requirements.txt && \ -# install python - add-apt-repository ppa:deadsnakes/ppa -y && \ - apt-get install -y python3.9 && \ - rm /usr/bin/python3 && \ - ln -s /usr/bin/python3.9 /usr/bin/python3 && \ - ln -s /usr/bin/python3 /usr/bin/python && \ pip install --no-cache requests argparse cryptography==3.3.2 urllib3 && \ pip install --upgrade requests && \ pip install setuptools==58.4.0 && \ diff --git a/docker/llm/finetune/qlora/cpu/docker/Dockerfile b/docker/llm/finetune/qlora/cpu/docker/Dockerfile index 2aaaa08e..15b96d6c 100644 --- a/docker/llm/finetune/qlora/cpu/docker/Dockerfile +++ b/docker/llm/finetune/qlora/cpu/docker/Dockerfile @@ -19,28 +19,38 @@ COPY --from=key-getter /root/intel-oneapi-archive-keyring.gpg /usr/share/keyring RUN echo "deb [signed-by=/usr/share/keyrings/intel-oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list RUN mkdir -p /ipex_llm/data && mkdir -p /ipex_llm/model && \ - # install pytorch 2.1.0 - apt-get update && \ - apt-get install -y --no-install-recommends python3-pip python3.9-dev python3-wheel python3.9-distutils git software-properties-common && \ + # Install python 3.11.1 + apt-get update && apt-get install -y curl wget gpg gpg-agent software-properties-common git gcc g++ make libunwind8-dev zlib1g-dev libssl-dev libffi-dev && \ + mkdir -p /opt/python && \ + cd /opt/python && \ + wget https://www.python.org/ftp/python/3.11.1/Python-3.11.1.tar.xz && \ + tar -xf Python-3.11.1.tar.xz && \ + cd Python-3.11.1 && \ + ./configure --enable-optimizations --with-zlib && \ + make altinstall && \ + rm /usr/bin/python3 && \ + # Create a symbolic link pointing to Python 3.11 at /usr/bin/python3 + ln -s /opt/python/Python-3.11.1/python /usr/bin/python3 && \ + # Create a symbolic link pointing to /usr/bin/python3 at /usr/bin/python + ln -s /usr/bin/python3 /usr/bin/python && \ + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ - pip3 install --upgrade pip && \ + pip install --upgrade pip && \ export PIP_DEFAULT_TIMEOUT=100 && \ pip install --upgrade torch==2.1.0 && \ # install CPU ipex-llm - pip3 install --pre --upgrade ipex-llm[all] && \ + pip install --pre --upgrade ipex-llm[all] && \ # install ipex and oneccl - pip install https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/cpu/intel_extension_for_pytorch-2.1.0%2Bcpu-cp39-cp39-linux_x86_64.whl && \ + pip install https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/cpu/intel_extension_for_pytorch-2.1.0%2Bcpu-cp311-cp311-linux_x86_64.whl && \ pip install oneccl_bind_pt -f https://developer.intel.com/ipex-whl-stable && \ # install huggingface dependencies pip install datasets transformers==4.35.0 && \ pip install fire peft==0.5.0 && \ pip install accelerate==0.23.0 && \ pip install bitsandbytes && \ - # install basic dependencies - apt-get update && apt-get install -y curl wget gpg gpg-agent software-properties-common libunwind8-dev && \ # get qlora example code - ln -s /usr/bin/python3 /usr/bin/python && \ cd /ipex_llm && \ git clone https://github.com/intel-analytics/IPEX-LLM.git && \ mv IPEX-LLM/python/llm/example/CPU/QLoRA-FineTuning/* . && \ diff --git a/docker/llm/finetune/qlora/cpu/docker/Dockerfile.k8s b/docker/llm/finetune/qlora/cpu/docker/Dockerfile.k8s index 71a8a5e1..7757a231 100644 --- a/docker/llm/finetune/qlora/cpu/docker/Dockerfile.k8s +++ b/docker/llm/finetune/qlora/cpu/docker/Dockerfile.k8s @@ -20,9 +20,24 @@ COPY --from=key-getter /root/intel-oneapi-archive-keyring.gpg /usr/share/keyring RUN echo "deb [signed-by=/usr/share/keyrings/intel-oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main " > /etc/apt/sources.list.d/oneAPI.list RUN mkdir -p /ipex_llm/data && mkdir -p /ipex_llm/model && \ - apt-get update && \ - apt install -y --no-install-recommends openssh-server openssh-client libcap2-bin gnupg2 ca-certificates \ - python3-pip python3.9-dev python3-wheel python3.9-distutils git software-properties-common && \ + # Install python 3.11.1 + apt-get update && apt-get install -y openssh-server openssh-client libcap2-bin gnupg2 ca-certificates \ + curl wget gpg gpg-agent software-properties-common git \ + gcc g++ make libunwind8-dev zlib1g-dev libssl-dev libffi-dev && \ + mkdir -p /opt/python && \ + cd /opt/python && \ + wget https://www.python.org/ftp/python/3.11.1/Python-3.11.1.tar.xz && \ + tar -xf Python-3.11.1.tar.xz && \ + cd Python-3.11.1 && \ + ./configure --enable-optimizations --with-zlib && \ + make altinstall && \ + rm /usr/bin/python3 && \ + # Create a symbolic link pointing to Python 3.11 at /usr/bin/python3 + ln -s /opt/python/Python-3.11.1/python /usr/bin/python3 && \ + # Create a symbolic link pointing to /usr/bin/python3 at /usr/bin/python + ln -s /usr/bin/python3 /usr/bin/python && \ + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ mkdir -p /var/run/sshd && \ @@ -37,11 +52,11 @@ RUN mkdir -p /ipex_llm/data && mkdir -p /ipex_llm/model && \ cp -r /etc/ssh/sshd_config /home/mpiuser/.sshd_config && \ echo "Port $port" >> /home/mpiuser/.sshd_config && \ # install pytorch 2.1.0 - pip3 install --upgrade pip && \ + pip install --upgrade pip && \ export PIP_DEFAULT_TIMEOUT=100 && \ pip install --upgrade torch==2.1.0 --index-url https://download.pytorch.org/whl/cpu && \ # install CPU ipex-llm - pip3 install --pre --upgrade ipex-llm[all] && \ + pip install --pre --upgrade ipex-llm[all] && \ # install ipex and oneccl pip install intel_extension_for_pytorch==2.0.100 && \ pip install oneccl_bind_pt -f https://developer.intel.com/ipex-whl-stable && \ @@ -50,7 +65,7 @@ RUN mkdir -p /ipex_llm/data && mkdir -p /ipex_llm/model && \ pip install fire peft==0.5.0 && \ pip install accelerate==0.23.0 && \ # install basic dependencies - apt-get update && apt-get install -y curl wget gpg gpg-agent software-properties-common libunwind8-dev && \ + apt-get update && apt-get install -y curl wget gpg gpg-agent && \ # Install Intel oneAPI keys. apt remove -y gnupg2 ca-certificates && \ apt autoremove -y && \ @@ -58,7 +73,6 @@ RUN mkdir -p /ipex_llm/data && mkdir -p /ipex_llm/model && \ apt install -y --no-install-recommends dnsutils intel-oneapi-mpi && \ rm -rf /var/lib/apt/lists/* && \ # get qlora example code - ln -s /usr/bin/python3 /usr/bin/python && \ cd /ipex_llm && \ git clone https://github.com/intel-analytics/IPEX-LLM.git && \ mv IPEX-LLM/python/llm/example/CPU/QLoRA-FineTuning/* . && \ diff --git a/docker/llm/finetune/qlora/xpu/docker/Dockerfile b/docker/llm/finetune/qlora/xpu/docker/Dockerfile index ea5fe693..e53d4ec9 100644 --- a/docker/llm/finetune/qlora/xpu/docker/Dockerfile +++ b/docker/llm/finetune/qlora/xpu/docker/Dockerfile @@ -17,17 +17,18 @@ RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-P # install basic dependencies apt-get install -y curl wget git gnupg gpg-agent software-properties-common libunwind8-dev vim less && \ # install Intel GPU driver - apt-get install -y intel-opencl-icd intel-level-zero-gpu=1.3.26241.33-647~22.04 level-zero level-zero-dev --allow-downgrades && \ + apt-get install -y intel-opencl-icd intel-level-zero-gpu level-zero level-zero-dev --allow-downgrades && \ # install python 3.9 ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ env DEBIAN_FRONTEND=noninteractive apt-get update && \ add-apt-repository ppa:deadsnakes/ppa -y && \ - apt-get install -y python3.9 && \ + apt-get install -y python3.11 && \ rm /usr/bin/python3 && \ - ln -s /usr/bin/python3.9 /usr/bin/python3 && \ + ln -s /usr/bin/python3.11 /usr/bin/python3 && \ ln -s /usr/bin/python3 /usr/bin/python && \ - apt-get install -y python3-pip python3.9-dev python3-wheel python3.9-distutils && \ + apt-get install -y python3-pip python3.11-dev python3-wheel python3.11-distutils && \ curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py && \ # install XPU ipex-llm pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ && \ # install huggingface dependencies