[PPML]Reduce the epc usage of bigdata (#7606)
* build bigdata with smaller epc * reduce epc in script * fix action * install pyarrow * delete unused part * reduce epc * fix * malloc_arena_max * set 16g to default and delete 4g * unused example * spark simplequery * pyspark simplequery * fix * fix * fix * code style fix
This commit is contained in:
Shaojie Cui
GitHub
parent
3a57b97c2b
commit
f0fa26a8a8
2 changed files with 8 additions and 64 deletions
36
.github/workflows/manually_build.yml
vendored
36
.github/workflows/manually_build.yml
vendored
|
|
@ -862,32 +862,18 @@ jobs:
|
|||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g
|
||||
sudo docker build \
|
||||
--no-cache=true \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=32G \
|
||||
--build-arg SGX_LOG_LEVEL=error \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest
|
||||
sudo docker push intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest
|
||||
sudo docker push 10.239.45.10/arda/intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-8g
|
||||
sudo docker build \
|
||||
--no-cache=true \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=64G \
|
||||
--build-arg SGX_MEM_SIZE=8G \
|
||||
--build-arg SGX_LOG_LEVEL=error \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
|
|
@ -910,26 +896,12 @@ jobs:
|
|||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g-all
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-8g-all
|
||||
sudo docker build \
|
||||
--no-cache=true \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=32G \
|
||||
--build-arg SGX_LOG_LEVEL=all \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g-all
|
||||
sudo docker build \
|
||||
--no-cache=true \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=64G \
|
||||
--build-arg SGX_MEM_SIZE=8G \
|
||||
--build-arg SGX_LOG_LEVEL=all \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
|
|
|
|||
36
.github/workflows/manually_build_for_testing.yml
vendored
36
.github/workflows/manually_build_for_testing.yml
vendored
|
|
@ -752,26 +752,12 @@ jobs:
|
|||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-8g
|
||||
sudo docker build \
|
||||
--no-cache=true \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=32G \
|
||||
--build-arg SGX_LOG_LEVEL=error \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g
|
||||
sudo docker build \
|
||||
--no-cache=true \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=64G \
|
||||
--build-arg SGX_MEM_SIZE=8G \
|
||||
--build-arg SGX_LOG_LEVEL=error \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
|
|
@ -794,26 +780,12 @@ jobs:
|
|||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g-all
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-8g-all
|
||||
sudo docker build \
|
||||
--no-cache=true \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=32G \
|
||||
--build-arg SGX_LOG_LEVEL=all \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
sudo docker push ${bigdata_custom_image}:${TAG}
|
||||
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
|
||||
|
||||
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g-all
|
||||
sudo docker build \
|
||||
--no-cache=true \
|
||||
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
|
||||
--build-arg BASE_IMAGE_TAG=${TAG} \
|
||||
--build-arg SGX_MEM_SIZE=64G \
|
||||
--build-arg SGX_MEM_SIZE=8G \
|
||||
--build-arg SGX_LOG_LEVEL=all \
|
||||
-t ${bigdata_custom_image}:${TAG} \
|
||||
-f ./Dockerfile .
|
||||
|
|
|
|||
Loading…
Reference in a new issue