diff --git a/python/llm/example/transformers/transformers_int4/falcon/README.md b/python/llm/example/transformers/transformers_int4/falcon/README.md index 76dc960d..eb4aacb5 100644 --- a/python/llm/example/transformers/transformers_int4/falcon/README.md +++ b/python/llm/example/transformers/transformers_int4/falcon/README.md @@ -29,8 +29,8 @@ from huggingface_hub import snapshot_download # for tiiuae/falcon-7b-instruct model_path = snapshot_download(repo_id='tiiuae/falcon-7b-instruct', - revision="c7f670a03d987254220f343c6b026ea0c5147185", - cache_dir="dir/path/where/model/files/are/downloaded") + revision="c7f670a03d987254220f343c6b026ea0c5147185", + cache_dir="dir/path/where/model/files/are/downloaded") print(f'tiiuae/falcon-7b-instruct checkpoint is downloaded to {model_path}') # for tiiuae/falcon-40b-instruct diff --git a/python/llm/example/transformers/transformers_int4/mpt/README.md b/python/llm/example/transformers/transformers_int4/mpt/README.md index 2a74c964..b082bde7 100644 --- a/python/llm/example/transformers/transformers_int4/mpt/README.md +++ b/python/llm/example/transformers/transformers_int4/mpt/README.md @@ -22,7 +22,7 @@ python ./generate.py --repo-id-or-model-path REPO_ID_OR_MODEL_PATH --prompt PROM ``` Arguments info: -- `--repo-id-or-model-path REPO_ID_OR_MODEL_PATH`: argument defining the huggingface repo id for the MPT model (e.g. `mosaicml/mpt-7b-chat` and `mosaicml/mpt-7b-chat`) to be downloaded, or the path to the huggingface checkpoint folder. It is default to be `'mosaicml/mpt-7b-chat'`. +- `--repo-id-or-model-path REPO_ID_OR_MODEL_PATH`: argument defining the huggingface repo id for the MPT model (e.g. `mosaicml/mpt-7b-chat` and `mosaicml/mpt-30b-chat`) to be downloaded, or the path to the huggingface checkpoint folder. It is default to be `'mosaicml/mpt-7b-chat'`. - `--prompt PROMPT`: argument defining the prompt to be infered (with integrated prompt format for chat). It is default to be `'What is AI?'`. - `--n-predict N_PREDICT`: argument defining the max number of tokens to predict. It is default to be `32`. @@ -67,6 +67,7 @@ AI, or artificial intelligence, is the simulation of human intelligence in machi #### [mosaicml/mpt-30b-chat](https://huggingface.co/mosaicml/mpt-30b-chat) ```log +Inference time: xxxx s -------------------- Prompt -------------------- <|im_start|>user What is AI?<|im_end|> diff --git a/python/llm/example/transformers/transformers_int4/mpt/generate.py b/python/llm/example/transformers/transformers_int4/mpt/generate.py index d63fc7b0..eb70ea70 100644 --- a/python/llm/example/transformers/transformers_int4/mpt/generate.py +++ b/python/llm/example/transformers/transformers_int4/mpt/generate.py @@ -29,7 +29,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description='Predict Tokens using `generate()` API for MPT model') parser.add_argument('--repo-id-or-model-path', type=str, default="mosaicml/mpt-7b-chat", help='The huggingface repo id for the MPT models' - '(e.g. `mosaicml/mpt-7b-chat` and `mosaicml/mpt-7b-chat`) to be downloaded' + '(e.g. `mosaicml/mpt-7b-chat` and `mosaicml/mpt-30b-chat`) to be downloaded' ', or the path to the huggingface checkpoint folder') parser.add_argument('--prompt', type=str, default="What is AI?", help='Prompt to infer')