ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

LLM: Compress some models to save space (#10315)

Browse source 
* LLM: compress some models to save space

* add deleted comments
This commit is contained in:
WeiguangHan 2024-03-04 17:53:03 +08:00 committed by GitHub
parent bab2ee5f9e
commit fd81d66047
2 changed files with 19 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
python/llm/dev/benchmark/all-in-one/run.py
View file

@ -398,15 +398,19 @@ def run_transformer_int4_gpu(repo_id,
tokenizer = LlamaTokenizer.from_pretrained(model_path, trust_remote_code=True)
model = model.to('xpu')
else:
if 'starcoder' in repo_id:
# Load starcoder-15.5b model in bf16 format to avoid CPU OOM.
model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_low_bit=low_bit,
trust_remote_code=True, use_cache=True, torch_dtype=torch.bfloat16).eval()
# Convert the low-bit model back to fp32 for performance considerations.
model = model.float()
if "4bit" in repo_id:
model = AutoModelForCausalLM.load_low_bit(model_path, optimize_model=True,
trust_remote_code=True, use_cache=True).eval()
else:
model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_low_bit=low_bit,
trust_remote_code=True, use_cache=True).eval()
if 'starcoder' in repo_id:
# Load starcoder-15.5b model in bf16 format to avoid CPU OOM.
model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_low_bit=low_bit,
trust_remote_code=True, use_cache=True, torch_dtype=torch.bfloat16).eval()
# Convert the low-bit model back to fp32 for performance considerations.
model = model.float()
else:
model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_low_bit=low_bit,
trust_remote_code=True, use_cache=True).eval()
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
model = model.to('xpu')
if isinstance(model, GPTJForCausalLM):

14
python/llm/test/benchmark/arc-perf-test.yaml
View file

@ -6,7 +6,7 @@ repo_id:
- 'tiiuae/falcon-7b-instruct-with-patch'
- 'mosaicml/mpt-7b-chat'
- 'redpajama/gptneox-7b-redpajama-bf16'
- 'bigcode/starcoder-15.5b'
- 'bigcode/starcoder-15.5b-4bit'
- 'databricks/dolly-v1-6b'
- 'databricks/dolly-v2-7b'
- 'databricks/dolly-v2-12b'
@ -14,9 +14,9 @@ repo_id:
- 'Qwen/Qwen-7B-Chat'
- 'BAAI/AquilaChat-7B'
- 'baichuan-inc/Baichuan2-7B-Chat'
- 'baichuan-inc/Baichuan2-13B-Chat'
- 'baichuan-inc/Baichuan2-13B-Chat-4bit'
- 'bigscience/bloomz-7b1'
- 'fnlp/moss-moon-003-sft'
- 'fnlp/moss-moon-003-sft-4bit'
local_model_hub: '/mnt/disk1/models'
warm_up: 1
num_trials: 3
@ -31,8 +31,8 @@ test_api:
- "transformer_int4_gpu" # on Intel GPU
cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
exclude:
- 'fnlp/moss-moon-003-sft:1024'
- 'fnlp/moss-moon-003-sft:2048'
- 'baichuan-inc/Baichuan2-13B-Chat:1024'
- 'baichuan-inc/Baichuan2-13B-Chat:2048'
- 'fnlp/moss-moon-003-sft-4bit:1024'
- 'fnlp/moss-moon-003-sft-4bit:2048'
- 'baichuan-inc/Baichuan2-13B-Chat-4bit:1024'
- 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
- 'bigscience/bloomz-7b1:2048'