[PPML]refactor bigdata build (#7088)
This commit is contained in:
Shaojie Cui
GitHub
parent
14cf4aff72
commit
3e040d7b28
2 changed files with 118 additions and 60 deletions
113
.github/workflows/manually_build.yml
vendored
113
.github/workflows/manually_build.yml
vendored
|
|
@ -67,58 +67,6 @@ jobs:
|
|||
sudo docker push ${base_image}:${TAG}
|
||||
sudo docker tag ${base_image}:${TAG} 10.239.45.10/arda/${base_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${base_image}:${TAG}
|
||||
echo "########################################################"
|
||||
echo "######### bigdl-ppml-trusted-bigdata-gramine ###########"
|
||||
echo "########################################################"
|
||||
cd ../trusted-bigdata
|
||||
export bigdata_base_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-base
|
||||
sudo docker build \
|
||||
--build-arg http_proxy=${HTTP_PROXY} \
|
||||
--build-arg https_proxy=${HTTPS_PROXY} \
|
||||
--build-arg HTTP_PROXY_HOST=${HTTP_PROXY_HOST_2} \
|
||||
--build-arg HTTP_PROXY_PORT=${HTTP_PROXY_PORT_2} \
|
||||
--build-arg HTTPS_PROXY_HOST=${HTTP_PROXY_HOST_2} \
|
||||
--build-arg HTTPS_PROXY_PORT=${HTTP_PROXY_PORT_3} \
|
||||
--build-arg JDK_VERSION=8u192 \
|
||||
--build-arg JDK_URL=${JDK_URL} \
|
||||
--build-arg SPARK_JAR_REPO_URL=${SPARK_JAR_REPO_URL} \
|
||||
--build-arg no_proxy=${LOCAL_IP} \
|
||||
--build-arg BASE_IMAGE_NAME=${base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
-t ${bigdata_base_image}:${TAG} -f ./Dockerfile .
|
||||
sudo docker push ${bigdata_base_image}:${TAG}
|
||||
sudo docker tag ${bigdata_base_image}:${TAG} 10.239.45.10/arda/${bigdata_base_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_base_image}:${TAG}
|
||||
cd custom-image
|
||||
openssl genrsa -3 -out enclave-key.pem 3072
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g
|
||||
sudo docker build \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=32G \
|
||||
--build-arg SGX_LOG_LEVEL=error \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:${TAG}
|
||||
sudo docker push intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG}
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g
|
||||
sudo docker build \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=64G \
|
||||
--build-arg SGX_LOG_LEVEL=error \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${base_image}:${TAG} 10.239.45.10/arda/${base_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_base_image}:${TAG} 10.239.45.10/arda/${bigdata_base_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
bigdl-ppml-trusted-deep-learning-gramine-base:
|
||||
if: ${{ github.event.inputs.artifact == 'bigdl-ppml-trusted-deep-learning-gramine-base' || github.event.inputs.artifact == 'all' }}
|
||||
|
|
@ -816,6 +764,20 @@ jobs:
|
|||
sudo docker push 10.239.45.10/arda/${bigdata_base_image}:${TAG}
|
||||
cd custom-image
|
||||
openssl genrsa -3 -out enclave-key.pem 3072
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-16g
|
||||
sudo docker build \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=16G \
|
||||
--build-arg SGX_LOG_LEVEL=error \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g
|
||||
sudo docker build \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
|
|
@ -827,9 +789,10 @@ jobs:
|
|||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:${TAG}
|
||||
sudo docker push intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest
|
||||
sudo docker push intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g
|
||||
sudo docker build \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
|
|
@ -841,6 +804,46 @@ jobs:
|
|||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-16g-all
|
||||
sudo docker build \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=16G \
|
||||
--build-arg SGX_LOG_LEVEL=all \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g-all
|
||||
sudo docker build \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=32G \
|
||||
--build-arg SGX_LOG_LEVEL=all \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g-all
|
||||
sudo docker build \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=64G \
|
||||
--build-arg SGX_LOG_LEVEL=all \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
sudo docker rmi -f ${base_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_base_image}:${TAG} 10.239.45.10/arda/${bigdata_base_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
|
|
|||
65
.github/workflows/manually_build_for_testing.yml
vendored
65
.github/workflows/manually_build_for_testing.yml
vendored
|
|
@ -671,6 +671,20 @@ jobs:
|
|||
sudo docker push 10.239.45.10/arda/${bigdata_base_image}:${TAG}
|
||||
cd custom-image
|
||||
openssl genrsa -3 -out enclave-key.pem 3072
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-16g
|
||||
sudo docker build \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=16G \
|
||||
--build-arg SGX_LOG_LEVEL=error \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g
|
||||
sudo docker build \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
|
|
@ -682,9 +696,10 @@ jobs:
|
|||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:${TAG}
|
||||
sudo docker push intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest
|
||||
sudo docker push intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g
|
||||
sudo docker build \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
|
|
@ -696,6 +711,46 @@ jobs:
|
|||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-16g-all
|
||||
sudo docker build \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=16G \
|
||||
--build-arg SGX_LOG_LEVEL=all \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g-all
|
||||
sudo docker build \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=32G \
|
||||
--build-arg SGX_LOG_LEVEL=all \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g-all
|
||||
sudo docker build \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=64G \
|
||||
--build-arg SGX_LOG_LEVEL=all \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
sudo docker rmi -f ${base_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_base_image}:${TAG} 10.239.45.10/arda/${bigdata_base_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_base_image}:${TAG} 10.239.45.10/arda/${bigdata_base_image}:${TAG}
|
||||
Loading…
Reference in a new issue