diff --git a/.github/workflows/manually_build.yml b/.github/workflows/manually_build.yml
index fc6ddfe5..8f9955ff 100644
--- a/.github/workflows/manually_build.yml
+++ b/.github/workflows/manually_build.yml
@@ -58,7 +58,6 @@ env:
 
 permissions:
   contents: read
-  packages: write
 
 jobs:
   bigdl-llm-finetune-lora-cpu:
diff --git a/.github/workflows/manually_build_for_testing.yml b/.github/workflows/manually_build_for_testing.yml
index 5cdff203..3af4bc9a 100644
--- a/.github/workflows/manually_build_for_testing.yml
+++ b/.github/workflows/manually_build_for_testing.yml
@@ -55,7 +55,6 @@ env:
 
 permissions:
   contents: read
-  packages: write
 
 jobs:
   bigdl-llm-finetune-lora-cpu: