ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

[PPML]Reduce the epc usage of bigdata (#7606)

Browse source 
* build bigdata with smaller epc

* reduce epc in script

* fix action

* install pyarrow

* delete unused part

* reduce epc

* fix

* malloc_arena_max

* set 16g to default and delete 4g

* unused example

* spark simplequery

* pyspark simplequery

* fix

* fix

* fix

* code style fix
This commit is contained in:
Shaojie Cui 2023-03-01 14:59:27 +08:00 committed by GitHub
parent 3a57b97c2b
commit f0fa26a8a8
2 changed files with 8 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
.github/workflows/manually_build.yml vendored
View file

@ -862,32 +862,18 @@ jobs:
sudo docker push ${bigdata_custom_image}:${TAG} sudo docker push ${bigdata_custom_image}:${TAG}
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g
sudo docker build \
--no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=32G \
--build-arg SGX_LOG_LEVEL=error \
-t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile .
sudo docker push ${bigdata_custom_image}:${TAG}
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker tag ${bigdata_custom_image}:${TAG} intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest sudo docker tag ${bigdata_custom_image}:${TAG} intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest
sudo docker push intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest sudo docker push intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest
sudo docker push 10.239.45.10/arda/intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest sudo docker push 10.239.45.10/arda/intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference:latest
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-8g
sudo docker build \ sudo docker build \
--no-cache=true \ --no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \ --build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=64G \ --build-arg SGX_MEM_SIZE=8G \
--build-arg SGX_LOG_LEVEL=error \ --build-arg SGX_LOG_LEVEL=error \
-t ${bigdata_custom_image}:${TAG} \ -t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile . -f ./Dockerfile .
@ -910,26 +896,12 @@ jobs:
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g-all export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-8g-all
sudo docker build \ sudo docker build \
--no-cache=true \ --no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \ --build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=32G \ --build-arg SGX_MEM_SIZE=8G \
--build-arg SGX_LOG_LEVEL=all \
-t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile .
sudo docker push ${bigdata_custom_image}:${TAG}
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g-all
sudo docker build \
--no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=64G \
--build-arg SGX_LOG_LEVEL=all \ --build-arg SGX_LOG_LEVEL=all \
-t ${bigdata_custom_image}:${TAG} \ -t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile . -f ./Dockerfile .

36
.github/workflows/manually_build_for_testing.yml vendored
View file

@ -752,26 +752,12 @@ jobs:
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-8g
sudo docker build \ sudo docker build \
--no-cache=true \ --no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \ --build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=32G \ --build-arg SGX_MEM_SIZE=8G \
--build-arg SGX_LOG_LEVEL=error \
-t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile .
sudo docker push ${bigdata_custom_image}:${TAG}
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g
sudo docker build \
--no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=64G \
--build-arg SGX_LOG_LEVEL=error \ --build-arg SGX_LOG_LEVEL=error \
-t ${bigdata_custom_image}:${TAG} \ -t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile . -f ./Dockerfile .
@ -794,26 +780,12 @@ jobs:
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG} sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-32g-all export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-8g-all
sudo docker build \ sudo docker build \
--no-cache=true \ --no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \ --build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \ --build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=32G \ --build-arg SGX_MEM_SIZE=8G \
--build-arg SGX_LOG_LEVEL=all \
-t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile .
sudo docker push ${bigdata_custom_image}:${TAG}
sudo docker tag ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker push 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
sudo docker rmi -f ${bigdata_custom_image}:${TAG} 10.239.45.10/arda/${bigdata_custom_image}:${TAG}
export bigdata_custom_image=intelanalytics/bigdl-ppml-trusted-bigdata-gramine-reference-64g-all
sudo docker build \
--no-cache=true \
--build-arg BASE_IMAGE_NAME=${bigdata_base_image} \
--build-arg BASE_IMAGE_TAG=${TAG} \
--build-arg SGX_MEM_SIZE=64G \
--build-arg SGX_LOG_LEVEL=all \ --build-arg SGX_LOG_LEVEL=all \
-t ${bigdata_custom_image}:${TAG} \ -t ${bigdata_custom_image}:${TAG} \
-f ./Dockerfile . -f ./Dockerfile .