ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix NPU load error message and add minicpm npu lowbit feat (#12064)

Browse source 
* fix npu_model raise sym_int4 error

* add load_lowbit

* remove print&perf
This commit is contained in:
Jinhe 2024-09-11 16:56:35 +08:00 committed by GitHub
parent 32e8362da7
commit 4ca330da15
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 37 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
python/llm/example/NPU/HF-Transformers-AutoModels/LLM/minicpm.py
View file

@ -37,6 +37,12 @@ if __name__ == "__main__":
help="The huggingface repo id for the Llama2 model to be downloaded" help="The huggingface repo id for the Llama2 model to be downloaded"
", or the path to the huggingface checkpoint folder", ", or the path to the huggingface checkpoint folder",
) )
parser.add_argument("--lowbit-path", type=str,
default="",
help="The path to the lowbit model folder, leave blank if you do not want to save. \
If path not exists, lowbit model will be saved there. \
Else, lowbit model will be loaded.",
)
parser.add_argument('--prompt', type=str, default="What is AI?", parser.add_argument('--prompt', type=str, default="What is AI?",
help='Prompt to infer') help='Prompt to infer')
parser.add_argument("--n-predict", type=int, default=32, help="Max tokens to predict") parser.add_argument("--n-predict", type=int, default=32, help="Max tokens to predict")
@ -48,7 +54,7 @@ if __name__ == "__main__":
args = parser.parse_args() args = parser.parse_args()
model_path = args.repo_id_or_model_path model_path = args.repo_id_or_model_path
if not args.lowbit_path or not os.path.exists(args.lowbit_path):
model = AutoModelForCausalLM.from_pretrained( model = AutoModelForCausalLM.from_pretrained(
model_path, model_path,
torch_dtype=torch.float16, torch_dtype=torch.float16,
@ -62,9 +68,24 @@ if __name__ == "__main__":
inter_pp=args.inter_pp, inter_pp=args.inter_pp,
transpose_value_cache=not args.disable_transpose_value_cache, transpose_value_cache=not args.disable_transpose_value_cache,
) )
else:
model = AutoModelForCausalLM.load_low_bit(
args.lowbit_path,
attn_implementation="eager",
torch_dtype=torch.float16,
optimize_model=True,
max_output_len=args.max_output_len,
max_prompt_len=args.max_prompt_len,
intra_pp=args.intra_pp,
inter_pp=args.inter_pp,
transpose_value_cache=not args.disable_transpose_value_cache,
trust_remote_code=True,
)
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
if args.lowbit_path and not os.path.exists(args.lowbit_path):
model.save_low_bit(args.lowbit_path)
print("-" * 80) print("-" * 80)
print("done") print("done")
with torch.inference_mode(): with torch.inference_mode():

2
python/llm/src/ipex_llm/transformers/npu_model.py
View file

@ -270,7 +270,7 @@ class _BaseAutoModelClass:
invalidInputError( invalidInputError(
qtype in ["sym_int8_rtn", "sym_int4_rtn"], qtype in ["sym_int8_rtn", "sym_int4_rtn"],
f"Unknown bigdl_transformers_low_bit value: {qtype}," f"Unknown bigdl_transformers_low_bit value: {qtype},"
f" expected: sym_int4, asym_int4, sym_int5, asym_int5 or sym_int8.", f" expected: sym_int8_rtn, sym_int4_rtn. "
) )
has_remote_code = hasattr(config, "auto_map") and cls.HF_Model.__name__ in config.auto_map has_remote_code = hasattr(config, "auto_map") and cls.HF_Model.__name__ in config.auto_map