ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

remove models we no longer need in benchmark. (#11492)

Browse source 
Co-authored-by: ATMxsp01 <shou.xu@intel.com>
This commit is contained in:
Xu, Shuo 2024-07-02 17:20:48 +08:00 committed by GitHub
parent 6a0134a9b2
commit 52519e07df
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 0 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
python/llm/test/benchmark/arc-perf-test-batch2.yaml
View file

@ -1,21 +1,9 @@
repo_id: repo_id:
- 'meta-llama/Llama-2-7b-chat-hf' - 'meta-llama/Llama-2-7b-chat-hf'
- 'meta-llama/Llama-2-13b-chat-hf' - 'meta-llama/Llama-2-13b-chat-hf'
- 'THUDM/chatglm2-6b'
- 'THUDM/chatglm3-6b-4bit' - 'THUDM/chatglm3-6b-4bit'
- 'tiiuae/falcon-7b-instruct-with-patch'
- 'mosaicml/mpt-7b-chat'
- 'redpajama/gptneox-7b-redpajama-bf16'
- 'bigcode/starcoder-15.5b-4bit'
- 'databricks/dolly-v1-6b'
- 'databricks/dolly-v2-7b'
# - 'databricks/dolly-v2-12b'
- 'internlm/internlm-chat-7b'
- 'Qwen/Qwen-7B-Chat'
- 'BAAI/AquilaChat-7B'
- 'baichuan-inc/Baichuan2-7B-Chat' - 'baichuan-inc/Baichuan2-7B-Chat'
- 'baichuan-inc/Baichuan2-13B-Chat-4bit' - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
- 'bigscience/bloomz-7b1'
# - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+ # - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+
- 'mistralai/Mistral-7B-v0.1' - 'mistralai/Mistral-7B-v0.1'
local_model_hub: '/mnt/disk1/models' local_model_hub: '/mnt/disk1/models'
@ -32,8 +20,5 @@ test_api:
- "transformer_int4_fp16_gpu" # on Intel GPU - "transformer_int4_fp16_gpu" # on Intel GPU
cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api) cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
exclude: exclude:
- 'bigcode/starcoder-15.5b-4bit:2048'
# - 'databricks/dolly-v2-12b:2048'
- 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048' - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
- 'bigscience/bloomz-7b1:2048'
task: 'continuation' # task can be 'continuation', 'QA' and 'summarize' task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'

26
python/llm/test/benchmark/arc-perf-test-batch4.yaml
View file

@ -1,21 +1,9 @@
repo_id: repo_id:
- 'meta-llama/Llama-2-7b-chat-hf' - 'meta-llama/Llama-2-7b-chat-hf'
- 'meta-llama/Llama-2-13b-chat-hf' - 'meta-llama/Llama-2-13b-chat-hf'
- 'THUDM/chatglm2-6b'
- 'THUDM/chatglm3-6b-4bit' - 'THUDM/chatglm3-6b-4bit'
- 'tiiuae/falcon-7b-instruct-with-patch'
- 'mosaicml/mpt-7b-chat'
- 'redpajama/gptneox-7b-redpajama-bf16'
- 'bigcode/starcoder-15.5b-4bit'
- 'databricks/dolly-v1-6b'
- 'databricks/dolly-v2-7b'
- 'databricks/dolly-v2-12b'
- 'internlm/internlm-chat-7b'
- 'Qwen/Qwen-7B-Chat'
- 'BAAI/AquilaChat-7B'
- 'baichuan-inc/Baichuan2-7B-Chat' - 'baichuan-inc/Baichuan2-7B-Chat'
- 'baichuan-inc/Baichuan2-13B-Chat-4bit' - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
- 'bigscience/bloomz-7b1'
# - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+ # - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+
- 'mistralai/Mistral-7B-v0.1' #mwj: need to check - 'mistralai/Mistral-7B-v0.1' #mwj: need to check
local_model_hub: '/mnt/disk1/models' local_model_hub: '/mnt/disk1/models'
@ -33,23 +21,9 @@ test_api:
cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api) cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
exclude: exclude:
- 'meta-llama/Llama-2-13b-chat-hf:2048' - 'meta-llama/Llama-2-13b-chat-hf:2048'
- 'tiiuae/falcon-7b-instruct-with-patch:2048'
- 'mosaicml/mpt-7b-chat:2048'
- 'redpajama/gptneox-7b-redpajama-bf16:2048'
- 'bigcode/starcoder-15.5b-4bit:1024'
- 'bigcode/starcoder-15.5b-4bit:2048'
- 'databricks/dolly-v1-6b:2048'
- 'databricks/dolly-v2-7b:2048'
- 'databricks/dolly-v2-12b:1024'
- 'databricks/dolly-v2-12b:2048'
- 'internlm/internlm-chat-7b:2048'
- 'Qwen/Qwen-7B-Chat:2048'
- 'BAAI/AquilaChat-7B:2048'
- 'baichuan-inc/Baichuan2-7B-Chat:2048' - 'baichuan-inc/Baichuan2-7B-Chat:2048'
- 'baichuan-inc/Baichuan2-13B-Chat-4bit:1024' - 'baichuan-inc/Baichuan2-13B-Chat-4bit:1024'
- 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048' - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
- 'bigscience/bloomz-7b1:1024'
- 'bigscience/bloomz-7b1:2048'
# - 'fnlp/moss-moon-003-sft-4bit:1024' # - 'fnlp/moss-moon-003-sft-4bit:1024'
# - 'fnlp/moss-moon-003-sft-4bit:2048' # - 'fnlp/moss-moon-003-sft-4bit:2048'
task: 'continuation' # task can be 'continuation', 'QA' and 'summarize' task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'

13
python/llm/test/benchmark/arc-perf-test.yaml
View file

@ -1,21 +1,9 @@
repo_id: repo_id:
- 'meta-llama/Llama-2-7b-chat-hf' - 'meta-llama/Llama-2-7b-chat-hf'
- 'meta-llama/Llama-2-13b-chat-hf' - 'meta-llama/Llama-2-13b-chat-hf'
- 'THUDM/chatglm2-6b'
- 'THUDM/chatglm3-6b-4bit' - 'THUDM/chatglm3-6b-4bit'
- 'tiiuae/falcon-7b-instruct-with-patch'
- 'mosaicml/mpt-7b-chat'
- 'redpajama/gptneox-7b-redpajama-bf16'
- 'bigcode/starcoder-15.5b-4bit'
- 'databricks/dolly-v1-6b'
- 'databricks/dolly-v2-7b'
# - 'databricks/dolly-v2-12b'
- 'internlm/internlm-chat-7b'
- 'Qwen/Qwen-7B-Chat'
- 'BAAI/AquilaChat-7B'
- 'baichuan-inc/Baichuan2-7B-Chat' - 'baichuan-inc/Baichuan2-7B-Chat'
- 'baichuan-inc/Baichuan2-13B-Chat-4bit' - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
- 'bigscience/bloomz-7b1'
# - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+ # - 'fnlp/moss-moon-003-sft-4bit' # moss-moon-003-sft cannot work on transformers 4.34+
- 'mistralai/Mistral-7B-v0.1' - 'mistralai/Mistral-7B-v0.1'
local_model_hub: '/mnt/disk1/models' local_model_hub: '/mnt/disk1/models'
@ -35,5 +23,4 @@ exclude:
# - 'fnlp/moss-moon-003-sft-4bit:1024' # - 'fnlp/moss-moon-003-sft-4bit:1024'
# - 'fnlp/moss-moon-003-sft-4bit:2048' # - 'fnlp/moss-moon-003-sft-4bit:2048'
- 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048' - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
- 'bigscience/bloomz-7b1:2048'
task: 'continuation' # task can be 'continuation', 'QA' and 'summarize' task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'

1
python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml
View file

@ -1,7 +1,6 @@
# For the models that require transformers 4.37.0 # For the models that require transformers 4.37.0
repo_id: repo_id:
- 'Qwen/Qwen1.5-7B-Chat' - 'Qwen/Qwen1.5-7B-Chat'
- 'microsoft/phi-2'
- 'microsoft/Phi-3-mini-4k-instruct' - 'microsoft/Phi-3-mini-4k-instruct'
- 'meta-llama/Meta-Llama-3-8B-Instruct' - 'meta-llama/Meta-Llama-3-8B-Instruct'
local_model_hub: '/mnt/disk1/models' local_model_hub: '/mnt/disk1/models'

1
python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml
View file

@ -1,7 +1,6 @@
# For the models that require transformers 4.37.0 # For the models that require transformers 4.37.0
repo_id: repo_id:
- 'Qwen/Qwen1.5-7B-Chat' - 'Qwen/Qwen1.5-7B-Chat'
- 'microsoft/phi-2'
- 'microsoft/Phi-3-mini-4k-instruct' - 'microsoft/Phi-3-mini-4k-instruct'
- 'meta-llama/Meta-Llama-3-8B-Instruct' # mwj: need to test - 'meta-llama/Meta-Llama-3-8B-Instruct' # mwj: need to test
local_model_hub: '/mnt/disk1/models' local_model_hub: '/mnt/disk1/models'

1
python/llm/test/benchmark/arc-perf-transformers-437.yaml
View file

@ -1,7 +1,6 @@
# For the models that require transformers 4.37.0 # For the models that require transformers 4.37.0
repo_id: repo_id:
- 'Qwen/Qwen1.5-7B-Chat' - 'Qwen/Qwen1.5-7B-Chat'
- 'microsoft/phi-2'
- 'microsoft/Phi-3-mini-4k-instruct' - 'microsoft/Phi-3-mini-4k-instruct'
- 'meta-llama/Meta-Llama-3-8B-Instruct' - 'meta-llama/Meta-Llama-3-8B-Instruct'
local_model_hub: '/mnt/disk1/models' local_model_hub: '/mnt/disk1/models'