diff --git a/python/llm/test/benchmark/arc-perf-test-batch2.yaml b/python/llm/test/benchmark/arc-perf-test-batch2.yaml
index 00b2e4c1..55e14d33 100644
--- a/python/llm/test/benchmark/arc-perf-test-batch2.yaml
+++ b/python/llm/test/benchmark/arc-perf-test-batch2.yaml
@@ -9,7 +9,7 @@ repo_id:
   - 'bigcode/starcoder-15.5b-4bit'
   - 'databricks/dolly-v1-6b'
   - 'databricks/dolly-v2-7b'
-  - 'databricks/dolly-v2-12b'
+  # - 'databricks/dolly-v2-12b'
   - 'internlm/internlm-chat-7b'
   - 'Qwen/Qwen-7B-Chat'
   - 'BAAI/AquilaChat-7B'
@@ -33,6 +33,6 @@ test_api:
 cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
 exclude:
   - 'bigcode/starcoder-15.5b-4bit:2048'
-  - 'databricks/dolly-v2-12b:2048'
+  # - 'databricks/dolly-v2-12b:2048'
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
   - 'bigscience/bloomz-7b1:2048'
\ No newline at end of file
diff --git a/python/llm/test/benchmark/arc-perf-test.yaml b/python/llm/test/benchmark/arc-perf-test.yaml
index 895588ce..021af908 100644
--- a/python/llm/test/benchmark/arc-perf-test.yaml
+++ b/python/llm/test/benchmark/arc-perf-test.yaml
@@ -9,7 +9,7 @@ repo_id:
   - 'bigcode/starcoder-15.5b-4bit'
   - 'databricks/dolly-v1-6b'
   - 'databricks/dolly-v2-7b'
-  - 'databricks/dolly-v2-12b'
+  # - 'databricks/dolly-v2-12b'
   - 'internlm/internlm-chat-7b'
   - 'Qwen/Qwen-7B-Chat'
   - 'BAAI/AquilaChat-7B'