ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

[LLM] Correct prompt format of Yi, Llama2 and Qwen in generate.py (#9786)

Browse source 
* correct prompt format of Yi

* correct prompt format of llama2 in cpu generate.py

* correct prompt format of Qwen in GPU example
This commit is contained in:
Ziteng Zhang 2023-12-26 16:57:55 +08:00 committed by GitHub
parent 0ea842231e
commit 44b4a0c9c5
4 changed files with 31 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
python/llm/example/CPU/HF-Transformers-AutoModels/Model/llama2/generate.py
View file

@ -22,11 +22,12 @@ from bigdl.llm.transformers import AutoModelForCausalLM
from transformers import LlamaTokenizer
# you could tune the prompt based on your own model,
# here the prompt tuning refers to https://huggingface.co/georgesung/llama2_7b_chat_uncensored#prompt-style
LLAMA2_PROMPT_FORMAT = """### HUMAN:
{prompt}
### RESPONSE:
# Refer to https://huggingface.co/TheBloke/Llama-2-70B-Chat-GGML#prompt-template-llama-2-chat
LLAMA2_PROMPT_FORMAT = """
[INST] <<SYS>>
You are a helpful assistant.
<</SYS>>
{prompt}[/INST]
"""
if __name__ == '__main__':

10
python/llm/example/CPU/HF-Transformers-AutoModels/Model/yi/generate.py
View file

@ -21,8 +21,14 @@ import argparse
from bigdl.llm.transformers import AutoModelForCausalLM
from transformers import AutoTokenizer
# you could tune the prompt based on your own model,
YI_PROMPT_FORMAT = "{prompt}"
# Refer to https://huggingface.co/01-ai/Yi-6B-Chat#31-use-the-chat-model
YI_PROMPT_FORMAT = """
<|im_start|>system
You are a helpful assistant. If you don't understand what the user means, ask the user to provide more information.<|im_end|>
<|im_start|>user
{prompt}<|im_end|>
<|im_start|>assistant
"""
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Predict Tokens using `generate()` API for Yi model')

10
python/llm/example/GPU/HF-Transformers-AutoModels/Model/qwen/generate.py
View file

@ -23,7 +23,15 @@ from bigdl.llm.transformers import AutoModelForCausalLM
from transformers import AutoTokenizer
# you could tune the prompt based on your own model
QWEN_PROMPT_FORMAT = "<human>{prompt} <bot>"
QWEN_PROMPT_FORMAT = """
<|im_start|>system
You are a helpful assistant.
<|im_end|>
<|im_start|>user
{prompt}
<|im_end|>
<|im_start|>assistant
"""
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Predict Tokens using `generate()` API for Qwen model')

10
python/llm/example/GPU/HF-Transformers-AutoModels/Model/yi/generate.py
View file

@ -22,8 +22,14 @@ import argparse
from bigdl.llm.transformers import AutoModelForCausalLM
from transformers import AutoTokenizer
# you could tune the prompt based on your own model
YI_PROMPT_FORMAT = "{prompt}"
# Refer to https://huggingface.co/01-ai/Yi-6B-Chat#31-use-the-chat-model
YI_PROMPT_FORMAT = """
<|im_start|>system
You are a helpful assistant. If you don't understand what the user means, ask the user to provide more information.<|im_end|>
<|im_start|>user
{prompt}<|im_end|>
<|im_start|>assistant
"""
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Predict Tokens using `generate()` API for Yi model')