ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

[PPML]CICD: build 32g bigdata image (#8205)

Browse source 
* [PPML]CICD: build 32g bigdata image

* fix
This commit is contained in:
Shaojie Cui 2023-05-17 11:30:10 +08:00 committed by GitHub
parent 21d5108f95
commit 768b15881d
2 changed files with 42 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
.github/workflows/manually_build.yml vendored
View file

@ -882,6 +882,22 @@ jobs:
cd custom-image cd custom-image
openssl genrsa -3 -out enclave-key.pem 3072 openssl genrsa -3 -out enclave-key.pem 3072
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-noattest-32g
sudo docker build \
--no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=32G \
--build-arg SGX_LOG_LEVEL=error \
--build-arg no_proxy=${NO_PROXY} \
--build-arg ENABLE_DCAP_ATTESTATION=false \
-t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile .
sudo docker push ${bigdata_custom_image}:${TAG}
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-noattest-16g export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-noattest-16g
sudo docker build \ sudo docker build \
--no-cache=true \ --no-cache=true \
@ -914,15 +930,14 @@ jobs:
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-noattest-4g export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g
sudo docker build \ sudo docker build \
--no-cache=true \ --no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \ --build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=4G \ --build-arg SGX_MEM_SIZE=32G \
--build-arg SGX_LOG_LEVEL=error \ --build-arg SGX_LOG_LEVEL=error \
--build-arg no_proxy=${NO_PROXY} \ --build-arg no_proxy=${NO_PROXY} \
--build-arg ENABLE_DCAP_ATTESTATION=false \
-t ${bigdata_custom_image}:${TAG} \ -t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile . -f ./Dockerfile .
sudo docker push ${bigdata_custom_image}:${TAG} sudo docker push ${bigdata_custom_image}:${TAG}
@ -964,13 +979,13 @@ jobs:
sudo docker push 10.239.45.10/arda/intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest sudo docker push 10.239.45.10/arda/intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-4g export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g-all
sudo docker build \ sudo docker build \
--no-cache=true \ --no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \ --build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=4G \ --build-arg SGX_MEM_SIZE=32G \
--build-arg SGX_LOG_LEVEL=error \ --build-arg SGX_LOG_LEVEL=all \
--build-arg no_proxy=${NO_PROXY} \ --build-arg no_proxy=${NO_PROXY} \
-t ${bigdata_custom_image}:${TAG} \ -t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile . -f ./Dockerfile .
@ -1009,21 +1024,6 @@ jobs:
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-4g-all
sudo docker build \
--no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=4G \
--build-arg SGX_LOG_LEVEL=all \
--build-arg no_proxy=${NO_PROXY} \
-t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile .
sudo docker push ${bigdata_custom_image}:${TAG}
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${base_image}:${TAG} sudo docker rmi -f ${base_image}:${TAG}
sudo docker rmi -f ${bigdata_base_image}:${TAG} 10.239.45.10/arda/${bigdata_base_image}:${TAG} sudo docker rmi -f ${bigdata_base_image}:${TAG} 10.239.45.10/arda/${bigdata_base_image}:${TAG}

42
.github/workflows/manually_build_for_testing.yml vendored
View file

@ -739,6 +739,22 @@ jobs:
cd custom-image cd custom-image
openssl genrsa -3 -out enclave-key.pem 3072 openssl genrsa -3 -out enclave-key.pem 3072
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-noattest-32g
sudo docker build \
--no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=32G \
--build-arg SGX_LOG_LEVEL=error \
--build-arg no_proxy=${NO_PROXY} \
--build-arg ENABLE_DCAP_ATTESTATION=false \
-t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile .
sudo docker push ${bigdata_custom_image}:${TAG}
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-noattest-16g export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-noattest-16g
sudo docker build \ sudo docker build \
--no-cache=true \ --no-cache=true \
@ -771,15 +787,14 @@ jobs:
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-noattest-4g export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g
sudo docker build \ sudo docker build \
--no-cache=true \ --no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \ --build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=4G \ --build-arg SGX_MEM_SIZE=32G \
--build-arg SGX_LOG_LEVEL=error \ --build-arg SGX_LOG_LEVEL=error \
--build-arg no_proxy=${NO_PROXY} \ --build-arg no_proxy=${NO_PROXY} \
--build-arg ENABLE_DCAP_ATTESTATION=false \
-t ${bigdata_custom_image}:${TAG} \ -t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile . -f ./Dockerfile .
sudo docker push ${bigdata_custom_image}:${TAG} sudo docker push ${bigdata_custom_image}:${TAG}
@ -817,13 +832,13 @@ jobs:
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-4g export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g-all
sudo docker build \ sudo docker build \
--no-cache=true \ --no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \ --build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=4G \ --build-arg SGX_MEM_SIZE=32G \
--build-arg SGX_LOG_LEVEL=error \ --build-arg SGX_LOG_LEVEL=all \
--build-arg no_proxy=${NO_PROXY} \ --build-arg no_proxy=${NO_PROXY} \
-t ${bigdata_custom_image}:${TAG} \ -t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile . -f ./Dockerfile .
@ -862,20 +877,5 @@ jobs:
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-4g-all
sudo docker build \
--no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=4G \
--build-arg SGX_LOG_LEVEL=all \
--build-arg no_proxy=${NO_PROXY} \
-t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile .
sudo docker push ${bigdata_custom_image}:${TAG}
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${base_image}:${TAG} sudo docker rmi -f ${base_image}:${TAG}
sudo docker rmi -f ${bigdata_base_image}:${TAG} 10.239.45.10/arda/${bigdata_base_image}:${TAG} sudo docker rmi -f ${bigdata_base_image}:${TAG} 10.239.45.10/arda/${bigdata_base_image}:${TAG}