From f5ef94046ea03e67ea27edb3d69ad813f2a2047e Mon Sep 17 00:00:00 2001
From: Shaojun Liu <61072813+liu-shaojun@users.noreply.github.com>
Date: Fri, 14 Jun 2024 15:35:56 +0800
Subject: [PATCH] exclude dolly-v2-12b for arc perf test (#11315)

* test arc perf

* test

* test

* exclude dolly-v2-12b:2048

* revert changes
---
 python/llm/test/benchmark/arc-perf-test-batch2.yaml | 4 ++--
 python/llm/test/benchmark/arc-perf-test.yaml        | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/llm/test/benchmark/arc-perf-test-batch2.yaml b/python/llm/test/benchmark/arc-perf-test-batch2.yaml
index 00b2e4c1..55e14d33 100644
--- a/python/llm/test/benchmark/arc-perf-test-batch2.yaml
+++ b/python/llm/test/benchmark/arc-perf-test-batch2.yaml
@@ -9,7 +9,7 @@ repo_id:
   - 'bigcode/starcoder-15.5b-4bit'
   - 'databricks/dolly-v1-6b'
   - 'databricks/dolly-v2-7b'
-  - 'databricks/dolly-v2-12b'
+  # - 'databricks/dolly-v2-12b'
   - 'internlm/internlm-chat-7b'
   - 'Qwen/Qwen-7B-Chat'
   - 'BAAI/AquilaChat-7B'
@@ -33,6 +33,6 @@ test_api:
 cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
 exclude:
   - 'bigcode/starcoder-15.5b-4bit:2048'
-  - 'databricks/dolly-v2-12b:2048'
+  # - 'databricks/dolly-v2-12b:2048'
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
   - 'bigscience/bloomz-7b1:2048'
\ No newline at end of file
diff --git a/python/llm/test/benchmark/arc-perf-test.yaml b/python/llm/test/benchmark/arc-perf-test.yaml
index 895588ce..021af908 100644
--- a/python/llm/test/benchmark/arc-perf-test.yaml
+++ b/python/llm/test/benchmark/arc-perf-test.yaml
@@ -9,7 +9,7 @@ repo_id:
   - 'bigcode/starcoder-15.5b-4bit'
   - 'databricks/dolly-v1-6b'
   - 'databricks/dolly-v2-7b'
-  - 'databricks/dolly-v2-12b'
+  # - 'databricks/dolly-v2-12b'
   - 'internlm/internlm-chat-7b'
   - 'Qwen/Qwen-7B-Chat'
   - 'BAAI/AquilaChat-7B'