[LLM] Change build system from centos7 to ubuntu18.04 (#8686)
* centos7 to ubuntu18 * ubuntu git version 2.17 need to update * use almalinux8 to build avx2 binaries
This commit is contained in:
xingyuan li
GitHub
parent
ea5d7aff5b
commit
4482ccb329
1 changed files with 44 additions and 61 deletions
105
.github/workflows/llm-binary-build.yml
vendored
105
.github/workflows/llm-binary-build.yml
vendored
|
|
@ -34,11 +34,11 @@ jobs:
|
|||
name: linux-avxvnni
|
||||
|
||||
linux-build-avxvnni:
|
||||
runs-on: [self-hosted, AVX2, centos7]
|
||||
runs-on: [self-hosted, AVX2, almalinux8]
|
||||
needs: check-linux-avxvnni-artifact
|
||||
if: needs.check-linux-avxvnni-artifact.outputs.if-exists == 'false'
|
||||
steps:
|
||||
# todo: it seems centos7 doesn't support python 3.9
|
||||
# This action only support ubuntu 16.04 to 22.04
|
||||
# - name: Set up Python
|
||||
# uses: actions/setup-python@v4
|
||||
# with:
|
||||
|
|
@ -46,46 +46,22 @@ jobs:
|
|||
- name: Set access token
|
||||
run: |
|
||||
echo "github_access_token=${GITHUB_ACCESS_TOKEN}" >> "$GITHUB_ENV"
|
||||
- name: Update Git
|
||||
shell: bash
|
||||
run: |
|
||||
yum -y remove git
|
||||
yum -y remove git-*
|
||||
yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm || true
|
||||
yum -y install git
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
repository: "intel-analytics/llm.cpp"
|
||||
token: ${{ env.github_access_token }}
|
||||
submodules: 'recursive'
|
||||
- name: Install Build Environment
|
||||
shell: bash
|
||||
run: |
|
||||
yum update -y
|
||||
yum install -y centos-release-scl
|
||||
yum install -y devtoolset-11
|
||||
yum install -y sshpass netcat
|
||||
- name: Download cmake
|
||||
shell: bash
|
||||
run: |
|
||||
export http_proxy=${HTTP_PROXY}
|
||||
export https_proxy=${HTTPS_PROXY}
|
||||
cd ..
|
||||
if [ -f "cmake-3.27.1-linux-x86_64.tar.gz" ]; then
|
||||
actual_sha256=$(sha256sum "cmake-3.27.1-linux-x86_64.tar.gz" | awk '{print $1}')
|
||||
if [ "$actual_sha256" != "9fef63e1cf87cab1153f9433045df2e43c336e462518b0f5e52d2cc91d762cff" ]; then
|
||||
wget https://github.com/Kitware/CMake/releases/download/v3.27.1/cmake-3.27.1-linux-x86_64.tar.gz
|
||||
fi
|
||||
else
|
||||
wget https://github.com/Kitware/CMake/releases/download/v3.27.1/cmake-3.27.1-linux-x86_64.tar.gz
|
||||
fi
|
||||
tar zxvf cmake-3.27.1-linux-x86_64.tar.gz
|
||||
yum install -y gcc-toolset-11 cmake git
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
repository: "intel-analytics/llm.cpp"
|
||||
token: ${{ env.github_access_token }}
|
||||
submodules: "recursive"
|
||||
- name: Build binary
|
||||
shell: bash
|
||||
run: |
|
||||
cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake"
|
||||
scl enable devtoolset-11 "$cmake_path -B build"
|
||||
scl enable devtoolset-11 "$cmake_path --build build --config Release"
|
||||
scl enable gcc-toolset-11 "cmake -B build"
|
||||
scl enable gcc-toolset-11 "cmake --build build --config Release"
|
||||
- name: Move release binary
|
||||
shell: bash
|
||||
run: |
|
||||
|
|
@ -109,10 +85,9 @@ jobs:
|
|||
- name: Build Chatglm
|
||||
shell: bash
|
||||
run: |
|
||||
cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake"
|
||||
cd src/chatglm
|
||||
scl enable devtoolset-11 "$cmake_path -B build"
|
||||
scl enable devtoolset-11 "$cmake_path --build build --config Release"
|
||||
scl enable gcc-toolset-11 "cmake -B build"
|
||||
scl enable gcc-toolset-11 "cmake --build build --config Release"
|
||||
- name: Move Chatglm binaries
|
||||
shell: bash
|
||||
run: |
|
||||
|
|
@ -142,7 +117,7 @@ jobs:
|
|||
name: linux-avx512
|
||||
|
||||
linux-build-avx512:
|
||||
runs-on: [self-hosted, AVX512, centos7]
|
||||
runs-on: [self-hosted, AVX512, ubuntu-18.04-lts]
|
||||
needs: check-linux-avx512-artifact
|
||||
if: needs.check-linux-avx512-artifact.outputs.if-exists == 'false'
|
||||
steps:
|
||||
|
|
@ -152,27 +127,23 @@ jobs:
|
|||
- name: Update Git
|
||||
shell: bash
|
||||
run: |
|
||||
yum -y remove git
|
||||
yum -y remove git-*
|
||||
yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm || true
|
||||
yum -y install git
|
||||
add-apt-repository -y ppa:git-core/ppa
|
||||
apt update
|
||||
apt install -y git
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
repository: "intel-analytics/llm.cpp"
|
||||
token: ${{ env.github_access_token }}
|
||||
submodules: 'recursive'
|
||||
submodules: "recursive"
|
||||
- name: Install Build Environment
|
||||
shell: bash
|
||||
run: |
|
||||
yum update -y
|
||||
yum install -y centos-release-scl
|
||||
yum install -y devtoolset-11
|
||||
yum install -y sshpass netcat
|
||||
- name: Download cmake
|
||||
shell: bash
|
||||
run: |
|
||||
export http_proxy=${HTTP_PROXY}
|
||||
export https_proxy=${HTTPS_PROXY}
|
||||
add-apt-repository -y ppa:ubuntu-toolchain-r/test
|
||||
apt update
|
||||
apt install -y build-essential
|
||||
apt install -y gcc-11 g++-11
|
||||
cd ..
|
||||
if [ -f "cmake-3.27.1-linux-x86_64.tar.gz" ]; then
|
||||
actual_sha256=$(sha256sum "cmake-3.27.1-linux-x86_64.tar.gz" | awk '{print $1}')
|
||||
|
|
@ -187,8 +158,11 @@ jobs:
|
|||
shell: bash
|
||||
run: |
|
||||
cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake"
|
||||
scl enable devtoolset-11 "$cmake_path -DONLYAVX=OFF -DONLYAVX2=OFF -B build"
|
||||
scl enable devtoolset-11 "$cmake_path --build build --config Release"
|
||||
$cmake_path -DONLYAVX=OFF -DONLYAVX2=OFF -B build
|
||||
$cmake_path --build build --config Release
|
||||
env:
|
||||
CC: gcc-11
|
||||
CXX: g++-11
|
||||
- name: Move avx512 release binary
|
||||
shell: bash
|
||||
run: |
|
||||
|
|
@ -205,8 +179,11 @@ jobs:
|
|||
shell: bash
|
||||
run: |
|
||||
cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake"
|
||||
scl enable devtoolset-11 "$cmake_path -DONLYAVX=OFF -DONLYAVX2=ON -B build"
|
||||
scl enable devtoolset-11 "$cmake_path --build build --config Release"
|
||||
$cmake_path -DONLYAVX=OFF -DONLYAVX2=ON -B build
|
||||
$cmake_path --build build --config Release
|
||||
env:
|
||||
CC: gcc-11
|
||||
CXX: g++-11
|
||||
- name: Move avx2 release binary
|
||||
shell: bash
|
||||
run: |
|
||||
|
|
@ -219,8 +196,11 @@ jobs:
|
|||
shell: bash
|
||||
run: |
|
||||
cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake"
|
||||
scl enable devtoolset-11 "$cmake_path -DONLYAVX=ON -DONLYAVX2=OFF -B build"
|
||||
scl enable devtoolset-11 "$cmake_path --build build --config Release"
|
||||
$cmake_path -DONLYAVX=ON -DONLYAVX2=OFF -B build
|
||||
$cmake_path --build build --config Release
|
||||
env:
|
||||
CC: gcc-11
|
||||
CXX: g++-11
|
||||
- name: Move avx release binary
|
||||
shell: bash
|
||||
run: |
|
||||
|
|
@ -234,8 +214,11 @@ jobs:
|
|||
# run: |
|
||||
# cmake_path="$(pwd)/../cmake-3.27.1-linux-x86_64/bin/cmake"
|
||||
# cd src/chatglm
|
||||
# scl enable devtoolset-11 "$cmake_path -B build"
|
||||
# scl enable devtoolset-11 "$cmake_path --build build --config Release"
|
||||
# $cmake_path -B build
|
||||
# $cmake_path --build build --config Release
|
||||
# env:
|
||||
# CC: gcc-11
|
||||
# CXX: g++-11
|
||||
- name: Archive avx512 build files
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
|
|
@ -283,7 +266,7 @@ jobs:
|
|||
with:
|
||||
repository: "intel-analytics/llm.cpp"
|
||||
token: ${{ env.github_access_token }}
|
||||
submodules: 'recursive'
|
||||
submodules: "recursive"
|
||||
- name: Add msbuild to PATH
|
||||
uses: microsoft/setup-msbuild@v1.1
|
||||
with:
|
||||
|
|
@ -330,7 +313,7 @@ jobs:
|
|||
with:
|
||||
repository: "intel-analytics/llm.cpp"
|
||||
token: ${{ env.github_access_token }}
|
||||
submodules: 'recursive'
|
||||
submodules: "recursive"
|
||||
- name: Add msbuild to PATH
|
||||
uses: microsoft/setup-msbuild@v1.1
|
||||
with:
|
||||
|
|
@ -404,7 +387,7 @@ jobs:
|
|||
with:
|
||||
repository: "intel-analytics/llm.cpp"
|
||||
token: ${{ env.github_access_token }}
|
||||
submodules: 'recursive'
|
||||
submodules: "recursive"
|
||||
- name: Add msbuild to PATH
|
||||
uses: microsoft/setup-msbuild@v1.1
|
||||
with:
|
||||
|
|
|
|||
Loading…
Reference in a new issue