From 3e040d7b2847ab593940dd1b626e3374d508bebc Mon Sep 17 00:00:00 2001 From: Shaojie Cui Date: Mon, 26 Dec 2022 13:49:59 +0800 Subject: [PATCH] [PPML]refactor bigdata build (#7088) --- .github/workflows/manually_build.yml | 113 +++++++++--------- .../workflows/manually_build_for_testing.yml | 65 +++++++++- 2 files changed, 118 insertions(+), 60 deletions(-) diff --git a/.github/workflows/manually_build.yml b/.github/workflows/manually_build.yml index 28c3e0cb..965148af 100644 --- a/.github/workflows/manually_build.yml +++ b/.github/workflows/manually_build.yml @@ -67,58 +67,6 @@ jobs: sudo docker push ${base_image}:${TAG} sudo docker tag ${base_image}:${TAG} 10.239.45.10/arda/${base_image}:${TAG} sudo docker push 10.239.45.10/arda/${base_image}:${TAG} - echo "########################################################" - echo "######### bigdl-ppml-trusted-bigdata-gramine ###########" - echo "########################################################" - cd ../trusted-bigdata - export bigdata_base_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-base - sudo docker build \ - --build-arg http_proxy=${HTTP_PROXY} \ - --build-arg https_proxy=${HTTPS_PROXY} \ - --build-arg HTTP_PROXY_HOST=${HTTP_PROXY_HOST_2} \ - --build-arg HTTP_PROXY_PORT=${HTTP_PROXY_PORT_2} \ - --build-arg HTTPS_PROXY_HOST=${HTTP_PROXY_HOST_2} \ - --build-arg HTTPS_PROXY_PORT=${HTTP_PROXY_PORT_3} \ - --build-arg JDK_VERSION=8u192 \ - --build-arg JDK_URL=${JDK_URL} \ - --build-arg SPARK_JAR_REPO_URL=${SPARK_JAR_REPO_URL} \ - --build-arg no_proxy=${LOCAL_IP} \ - --build-arg BASE_IMAGE_NAME=${base_image} \ - --build-arg BASE_IMAGE_TAG=${TAG} \ - -t ${bigdata_base_image}:${TAG} -f ./Dockerfile . - sudo docker push ${bigdata_base_image}:${TAG} - sudo docker tag ${bigdata_base_image}:${TAG} 10.239.45.10/arda/${bigdata_base_image}:${TAG} - sudo docker push 10.239.45.10/arda/${bigdata_base_image}:${TAG} - cd custom-image - openssl genrsa -3 -out enclave-key.pem 3072 - export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g - sudo docker build \ - --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ - --build-arg BASE_IMAGE_TAG=${TAG} \ - --build-arg SGX_MEM_SIZE=32G \ - --build-arg SGX_LOG_LEVEL=error \ - -t ${bigdata_custom_image}:${TAG} \ - -f ./Dockerfile . - sudo docker push ${bigdata_custom_image}:${TAG} - sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} - sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} - sudo docker tag ${bigdata_custom_image}:${TAG} intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:${TAG} - sudo docker push intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:${TAG} - sudo docker rmi -f ${bigdata_custom_image}:${TAG} - export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g - sudo docker build \ - --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ - --build-arg BASE_IMAGE_TAG=${TAG} \ - --build-arg SGX_MEM_SIZE=64G \ - --build-arg SGX_LOG_LEVEL=error \ - -t ${bigdata_custom_image}:${TAG} \ - -f ./Dockerfile . - sudo docker push ${bigdata_custom_image}:${TAG} - sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} - sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} - sudo docker rmi -f ${base_image}:${TAG} 10.239.45.10/arda/${base_image}:${TAG} - sudo docker rmi -f ${bigdata_base_image}:${TAG} 10.239.45.10/arda/${bigdata_base_image}:${TAG} - sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} bigdl-ppml-trusted-deep-learning-gramine-base: if: ${{ github.event.inputs.artifact == 'bigdl-ppml-trusted-deep-learning-gramine-base' || github.event.inputs.artifact == 'all' }} @@ -816,6 +764,20 @@ jobs: sudo docker push 10.239.45.10/arda/${bigdata_base_image}:${TAG} cd custom-image openssl genrsa -3 -out enclave-key.pem 3072 + + export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-16g + sudo docker build \ + --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ + --build-arg BASE_IMAGE_TAG=${TAG} \ + --build-arg SGX_MEM_SIZE=16G \ + --build-arg SGX_LOG_LEVEL=error \ + -t ${bigdata_custom_image}:${TAG} \ + -f ./Dockerfile . + sudo docker push ${bigdata_custom_image}:${TAG} + sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g sudo docker build \ --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ @@ -827,9 +789,10 @@ jobs: sudo docker push ${bigdata_custom_image}:${TAG} sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} - sudo docker tag ${bigdata_custom_image}:${TAG} intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:${TAG} - sudo docker push intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:${TAG} + sudo docker tag ${bigdata_custom_image}:${TAG} intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest + sudo docker push intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g sudo docker build \ --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ @@ -841,6 +804,46 @@ jobs: sudo docker push ${bigdata_custom_image}:${TAG} sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + + export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-16g-all + sudo docker build \ + --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ + --build-arg BASE_IMAGE_TAG=${TAG} \ + --build-arg SGX_MEM_SIZE=16G \ + --build-arg SGX_LOG_LEVEL=all \ + -t ${bigdata_custom_image}:${TAG} \ + -f ./Dockerfile . + sudo docker push ${bigdata_custom_image}:${TAG} + sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + + export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g-all + sudo docker build \ + --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ + --build-arg BASE_IMAGE_TAG=${TAG} \ + --build-arg SGX_MEM_SIZE=32G \ + --build-arg SGX_LOG_LEVEL=all \ + -t ${bigdata_custom_image}:${TAG} \ + -f ./Dockerfile . + sudo docker push ${bigdata_custom_image}:${TAG} + sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + + export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g-all + sudo docker build \ + --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ + --build-arg BASE_IMAGE_TAG=${TAG} \ + --build-arg SGX_MEM_SIZE=64G \ + --build-arg SGX_LOG_LEVEL=all \ + -t ${bigdata_custom_image}:${TAG} \ + -f ./Dockerfile . + sudo docker push ${bigdata_custom_image}:${TAG} + sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker rmi -f ${base_image}:${TAG} sudo docker rmi -f ${bigdata_base_image}:${TAG} 10.239.45.10/arda/${bigdata_base_image}:${TAG} - sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} diff --git a/.github/workflows/manually_build_for_testing.yml b/.github/workflows/manually_build_for_testing.yml index 5f806738..d7a66883 100644 --- a/.github/workflows/manually_build_for_testing.yml +++ b/.github/workflows/manually_build_for_testing.yml @@ -671,6 +671,20 @@ jobs: sudo docker push 10.239.45.10/arda/${bigdata_base_image}:${TAG} cd custom-image openssl genrsa -3 -out enclave-key.pem 3072 + + export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-16g + sudo docker build \ + --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ + --build-arg BASE_IMAGE_TAG=${TAG} \ + --build-arg SGX_MEM_SIZE=16G \ + --build-arg SGX_LOG_LEVEL=error \ + -t ${bigdata_custom_image}:${TAG} \ + -f ./Dockerfile . + sudo docker push ${bigdata_custom_image}:${TAG} + sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g sudo docker build \ --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ @@ -682,9 +696,10 @@ jobs: sudo docker push ${bigdata_custom_image}:${TAG} sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} - sudo docker tag ${bigdata_custom_image}:${TAG} intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:${TAG} - sudo docker push intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:${TAG} - sudo docker rmi -f ${bigdata_custom_image}:${TAG} + sudo docker tag ${bigdata_custom_image}:${TAG} intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest + sudo docker push intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest + sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g sudo docker build \ --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ @@ -696,6 +711,46 @@ jobs: sudo docker push ${bigdata_custom_image}:${TAG} sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + + export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-16g-all + sudo docker build \ + --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ + --build-arg BASE_IMAGE_TAG=${TAG} \ + --build-arg SGX_MEM_SIZE=16G \ + --build-arg SGX_LOG_LEVEL=all \ + -t ${bigdata_custom_image}:${TAG} \ + -f ./Dockerfile . + sudo docker push ${bigdata_custom_image}:${TAG} + sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + + export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g-all + sudo docker build \ + --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ + --build-arg BASE_IMAGE_TAG=${TAG} \ + --build-arg SGX_MEM_SIZE=32G \ + --build-arg SGX_LOG_LEVEL=all \ + -t ${bigdata_custom_image}:${TAG} \ + -f ./Dockerfile . + sudo docker push ${bigdata_custom_image}:${TAG} + sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + + export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g-all + sudo docker build \ + --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ + --build-arg BASE_IMAGE_TAG=${TAG} \ + --build-arg SGX_MEM_SIZE=64G \ + --build-arg SGX_LOG_LEVEL=all \ + -t ${bigdata_custom_image}:${TAG} \ + -f ./Dockerfile . + sudo docker push ${bigdata_custom_image}:${TAG} + sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} + sudo docker rmi -f ${base_image}:${TAG} - sudo docker rmi -f ${bigdata_base_image}:${TAG} 10.239.45.10/arda/${bigdata_base_image}:${TAG} - sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} \ No newline at end of file + sudo docker rmi -f ${bigdata_base_image}:${TAG} 10.239.45.10/arda/${bigdata_base_image}:${TAG} \ No newline at end of file