ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

[LLM] Change build system from centos7 to ubuntu18.04 (#8686)

Browse source 
* centos7 to ubuntu18
* ubuntu git version 2.17 need to update
* use almalinux8 to build avx2 binaries
This commit is contained in:
xingyuan li 2023-08-07 19:09:58 +09:00 committed by GitHub
parent ea5d7aff5b
commit 4482ccb329
1 changed files with 44 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

105
.github/workflows/llm-binary-build.yml vendored
View file

@ -34,11 +34,11 @@ jobs:
name: linux-avxvnni name: linux-avxvnni
linux-build-avxvnni: linux-build-avxvnni:
runs-on: [self-hosted, AVX2, centos7] runs-on: [self-hosted, AVX2, almalinux8]
needs: check-linux-avxvnni-artifact needs: check-linux-avxvnni-artifact
if: needs.check-linux-avxvnni-artifact.outputs.if-exists == 'false' if: needs.check-linux-avxvnni-artifact.outputs.if-exists == 'false'
steps: steps:
# todo: it seems centos7 doesn't support python 3.9 # This action only support ubuntu 16.04 to 22.04
# - name: Set up Python # - name: Set up Python
# uses: actions/setup-python@v4 # uses: actions/setup-python@v4
# with: # with:
@ -46,46 +46,22 @@ jobs:
- name: Set access token - name: Set access token
run: | run: |
echo "github_access_token=${GITHUB_ACCESS_TOKEN}" >> "$GITHUB_ENV" echo "github_access_token=${GITHUB_ACCESS_TOKEN}" >> "$GITHUB_ENV"
- name: Update Git
shell: bash
run: |
yum -y remove git
yum -y remove git-*
yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm || true
yum -y install git
- uses: actions/checkout@v3
with:
repository: "intel-analytics/llm.cpp"
token: ${{ env.github_access_token }}
submodules: 'recursive'
- name: Install Build Environment - name: Install Build Environment
shell: bash
run: |
yum update -y
yum install -y centos-release-scl
yum install -y devtoolset-11
yum install -y sshpass netcat
- name: Download cmake
shell: bash shell: bash
run: | run: |
export http_proxy=${HTTP_PROXY} export http_proxy=${HTTP_PROXY}
export https_proxy=${HTTPS_PROXY} export https_proxy=${HTTPS_PROXY}
cd .. yum install -y gcc-toolset-11 cmake git
if [ -f "cmake-3.27.1-linux-x86_64.tar.gz" ]; then - uses: actions/checkout@v3
actual_sha256=$(sha256sum "cmake-3.27.1-linux-x86_64.tar.gz" | awk '{print $1}') with:
if [ "$actual_sha256" != "9fef63e1cf87cab1153f9433045df2e43c336e462518b0f5e52d2cc91d762cff" ]; then repository: "intel-analytics/llm.cpp"
wget https://github.com/Kitware/CMake/releases/download/v3.27.1/cmake-3.27.1-linux-x86_64.tar.gz token: ${{ env.github_access_token }}
fi submodules: "recursive"
else
wget https://github.com/Kitware/CMake/releases/download/v3.27.1/cmake-3.27.1-linux-x86_64.tar.gz
fi
tar zxvf cmake-3.27.1-linux-x86_64.tar.gz
- name: Build binary - name: Build binary
shell: bash shell: bash
run: | run: |
cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake" scl enable gcc-toolset-11 "cmake -B build"
scl enable devtoolset-11 "$cmake_path -B build" scl enable gcc-toolset-11 "cmake --build build --config Release"
scl enable devtoolset-11 "$cmake_path --build build --config Release"
- name: Move release binary - name: Move release binary
shell: bash shell: bash
run: | run: |
@ -109,10 +85,9 @@ jobs:
- name: Build Chatglm - name: Build Chatglm
shell: bash shell: bash
run: | run: |
cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake"
cd src/chatglm cd src/chatglm
scl enable devtoolset-11 "$cmake_path -B build" scl enable gcc-toolset-11 "cmake -B build"
scl enable devtoolset-11 "$cmake_path --build build --config Release" scl enable gcc-toolset-11 "cmake --build build --config Release"
- name: Move Chatglm binaries - name: Move Chatglm binaries
shell: bash shell: bash
run: | run: |
@ -142,7 +117,7 @@ jobs:
name: linux-avx512 name: linux-avx512
linux-build-avx512: linux-build-avx512:
runs-on: [self-hosted, AVX512, centos7] runs-on: [self-hosted, AVX512, ubuntu-18.04-lts]
needs: check-linux-avx512-artifact needs: check-linux-avx512-artifact
if: needs.check-linux-avx512-artifact.outputs.if-exists == 'false' if: needs.check-linux-avx512-artifact.outputs.if-exists == 'false'
steps: steps:
@ -152,27 +127,23 @@ jobs:
- name: Update Git - name: Update Git
shell: bash shell: bash
run: | run: |
yum -y remove git add-apt-repository -y ppa:git-core/ppa
yum -y remove git-* apt update
yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm || true apt install -y git
yum -y install git
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
repository: "intel-analytics/llm.cpp" repository: "intel-analytics/llm.cpp"
token: ${{ env.github_access_token }} token: ${{ env.github_access_token }}
submodules: 'recursive' submodules: "recursive"
- name: Install Build Environment - name: Install Build Environment
shell: bash
run: |
yum update -y
yum install -y centos-release-scl
yum install -y devtoolset-11
yum install -y sshpass netcat
- name: Download cmake
shell: bash shell: bash
run: | run: |
export http_proxy=${HTTP_PROXY} export http_proxy=${HTTP_PROXY}
export https_proxy=${HTTPS_PROXY} export https_proxy=${HTTPS_PROXY}
add-apt-repository -y ppa:ubuntu-toolchain-r/test
apt update
apt install -y build-essential
apt install -y gcc-11 g++-11
cd .. cd ..
if [ -f "cmake-3.27.1-linux-x86_64.tar.gz" ]; then if [ -f "cmake-3.27.1-linux-x86_64.tar.gz" ]; then
actual_sha256=$(sha256sum "cmake-3.27.1-linux-x86_64.tar.gz" | awk '{print $1}') actual_sha256=$(sha256sum "cmake-3.27.1-linux-x86_64.tar.gz" | awk '{print $1}')
@ -187,8 +158,11 @@ jobs:
shell: bash shell: bash
run: | run: |
cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake" cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake"
scl enable devtoolset-11 "$cmake_path -DONLYAVX=OFF -DONLYAVX2=OFF -B build" $cmake_path -DONLYAVX=OFF -DONLYAVX2=OFF -B build
scl enable devtoolset-11 "$cmake_path --build build --config Release" $cmake_path --build build --config Release
env:
CC: gcc-11
CXX: g++-11
- name: Move avx512 release binary - name: Move avx512 release binary
shell: bash shell: bash
run: | run: |
@ -205,8 +179,11 @@ jobs:
shell: bash shell: bash
run: | run: |
cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake" cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake"
scl enable devtoolset-11 "$cmake_path -DONLYAVX=OFF -DONLYAVX2=ON -B build" $cmake_path -DONLYAVX=OFF -DONLYAVX2=ON -B build
scl enable devtoolset-11 "$cmake_path --build build --config Release" $cmake_path --build build --config Release
env:
CC: gcc-11
CXX: g++-11
- name: Move avx2 release binary - name: Move avx2 release binary
shell: bash shell: bash
run: | run: |
@ -219,8 +196,11 @@ jobs:
shell: bash shell: bash
run: | run: |
cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake" cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake"
scl enable devtoolset-11 "$cmake_path -DONLYAVX=ON -DONLYAVX2=OFF -B build" $cmake_path -DONLYAVX=ON -DONLYAVX2=OFF -B build
scl enable devtoolset-11 "$cmake_path --build build --config Release" $cmake_path --build build --config Release
env:
CC: gcc-11
CXX: g++-11
- name: Move avx release binary - name: Move avx release binary
shell: bash shell: bash
run: | run: |
@ -234,8 +214,11 @@ jobs:
# run: | # run: |
# cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake" # cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake"
# cd src/chatglm # cd src/chatglm
# scl enable devtoolset-11 "$cmake_path -B build" # $cmake_path -B build
# scl enable devtoolset-11 "$cmake_path --build build --config Release" # $cmake_path --build build --config Release
# env:
# CC: gcc-11
# CXX: g++-11
- name: Archive avx512 build files - name: Archive avx512 build files
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@ -283,7 +266,7 @@ jobs:
with: with:
repository: "intel-analytics/llm.cpp" repository: "intel-analytics/llm.cpp"
token: ${{ env.github_access_token }} token: ${{ env.github_access_token }}
submodules: 'recursive' submodules: "recursive"
- name: Add msbuild to PATH - name: Add msbuild to PATH
uses: microsoft/setup-msbuild@v1.1 uses: microsoft/setup-msbuild@v1.1
with: with:
@ -330,7 +313,7 @@ jobs:
with: with:
repository: "intel-analytics/llm.cpp" repository: "intel-analytics/llm.cpp"
token: ${{ env.github_access_token }} token: ${{ env.github_access_token }}
submodules: 'recursive' submodules: "recursive"
- name: Add msbuild to PATH - name: Add msbuild to PATH
uses: microsoft/setup-msbuild@v1.1 uses: microsoft/setup-msbuild@v1.1
with: with:
@ -404,7 +387,7 @@ jobs:
with: with:
repository: "intel-analytics/llm.cpp" repository: "intel-analytics/llm.cpp"
token: ${{ env.github_access_token }} token: ${{ env.github_access_token }}
submodules: 'recursive' submodules: "recursive"
- name: Add msbuild to PATH - name: Add msbuild to PATH
uses: microsoft/setup-msbuild@v1.1 uses: microsoft/setup-msbuild@v1.1
with: with: