From 89069d61739a7f0e774bb9ab617474c8c1e0153f Mon Sep 17 00:00:00 2001 From: dingbaorong Date: Wed, 6 Dec 2023 15:17:54 +0800 Subject: [PATCH] Add gpu gguf example (#9603) * add gpu gguf example * some fixes * address kai's comments * address json's comments --- README.md | 2 +- .../Advanced-Quantizations/GGUF}/README.md | 6 +- .../Advanced-Quantizations/GGUF}/generate.py | 0 .../Advanced-Quantizations/GGUF/README.md | 77 +++++++++++++++++++ .../Advanced-Quantizations/GGUF/generate.py | 63 +++++++++++++++ 5 files changed, 146 insertions(+), 2 deletions(-) rename python/llm/example/CPU/{GGUF-Models/llama2 => HF-Transformers-AutoModels/Advanced-Quantizations/GGUF}/README.md (95%) rename python/llm/example/CPU/{GGUF-Models/llama2 => HF-Transformers-AutoModels/Advanced-Quantizations/GGUF}/generate.py (100%) create mode 100644 python/llm/example/GPU/HF-Transformers-AutoModels/Advanced-Quantizations/GGUF/README.md create mode 100644 python/llm/example/GPU/HF-Transformers-AutoModels/Advanced-Quantizations/GGUF/generate.py diff --git a/README.md b/README.md index 75798b92..83640750 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ ### Latest update - [2023/12] `bigdl-llm` now supports [FP8 and FP4 inference](python/llm/example/GPU/HF-Transformers-AutoModels/More-Data-Types) on Intel ***GPU***. -- [2023/11] Initial support for directly loading [GGUF](python/llm/example/CPU/GGUF-Models/llama2), [AWQ](python/llm/example/GPU/HF-Transformers-AutoModels/Advanced-Quantizations/AWQ) and [GPTQ](python/llm/example/GPU/HF-Transformers-AutoModels/Advanced-Quantizations/GPTQ) models in to `bigdl-llm` is available. +- [2023/11] Initial support for directly loading [GGUF](python/llm/example/GPU/HF-Transformers-AutoModels/Advanced-Quantizations/GGUF), [AWQ](python/llm/example/GPU/HF-Transformers-AutoModels/Advanced-Quantizations/AWQ) and [GPTQ](python/llm/example/GPU/HF-Transformers-AutoModels/Advanced-Quantizations/GPTQ) models in to `bigdl-llm` is available. - [2023/11] Initial support for [vLLM continuous batching](python/llm/example/CPU/vLLM-Serving) is availabe on Intel ***CPU***. - [2023/11] Initial support for [vLLM continuous batching](python/llm/example/GPU/vLLM-Serving) is availabe on Intel ***GPU***. - [2023/10] [QLoRA finetuning](python/llm/example/CPU/QLoRA-FineTuning) on Intel ***CPU*** is available. diff --git a/python/llm/example/CPU/GGUF-Models/llama2/README.md b/python/llm/example/CPU/HF-Transformers-AutoModels/Advanced-Quantizations/GGUF/README.md similarity index 95% rename from python/llm/example/CPU/GGUF-Models/llama2/README.md rename to python/llm/example/CPU/HF-Transformers-AutoModels/Advanced-Quantizations/GGUF/README.md index 962389ba..2725c669 100644 --- a/python/llm/example/CPU/GGUF-Models/llama2/README.md +++ b/python/llm/example/CPU/HF-Transformers-AutoModels/Advanced-Quantizations/GGUF/README.md @@ -5,6 +5,9 @@ In this directory, you will find examples on how to load GGUF model into `bigdl- ## Requirements To run these examples with BigDL-LLM, we have some recommended requirements for your machine, please refer to [here](../README.md#recommended-requirements) for more information. +**Important: Please make sure you have installed `transformers==4.33.0` to run the example.** + + ## Example: Load gguf model using `from_gguf()` API In the example [generate.py](./generate.py), we show a basic use case to load a GGUF LLaMA2 model into `bigdl-llm` using `from_gguf()` API, with BigDL-LLM optimizations. @@ -17,6 +20,7 @@ conda create -n llm python=3.9 # recommend to use Python 3.9 conda activate llm pip install --pre --upgrade bigdl-llm[all] # install the latest bigdl-llm nightly build with 'all' option +pip install transformers==4.33.0 # upgrade transformers ``` ### 2. Run @@ -50,7 +54,7 @@ In the example, several arguments can be passed to satisfy your requirements: - `--prompt PROMPT`: argument defining the prompt to be infered (with integrated prompt format for chat). It is default to be `'What is AI?'`. - `--n-predict N_PREDICT`: argument defining the max number of tokens to predict. It is default to be `32`. -#### 2.3 Sample Output +#### 2.4 Sample Output #### [llama-2-7b-chat.Q4_0.gguf](https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/tree/main) ```log Inference time: xxxx s diff --git a/python/llm/example/CPU/GGUF-Models/llama2/generate.py b/python/llm/example/CPU/HF-Transformers-AutoModels/Advanced-Quantizations/GGUF/generate.py similarity index 100% rename from python/llm/example/CPU/GGUF-Models/llama2/generate.py rename to python/llm/example/CPU/HF-Transformers-AutoModels/Advanced-Quantizations/GGUF/generate.py diff --git a/python/llm/example/GPU/HF-Transformers-AutoModels/Advanced-Quantizations/GGUF/README.md b/python/llm/example/GPU/HF-Transformers-AutoModels/Advanced-Quantizations/GGUF/README.md new file mode 100644 index 00000000..77c5e0dc --- /dev/null +++ b/python/llm/example/GPU/HF-Transformers-AutoModels/Advanced-Quantizations/GGUF/README.md @@ -0,0 +1,77 @@ +# Loading GGUF models +In this directory, you will find examples on how to load GGUF model into `bigdl-llm`. For illustration purposes, we utilize the [llama-2-7b-chat.Q4_0.gguf](https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/tree/main) and [llama-2-7b-chat.Q4_1.gguf](https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/tree/main) as reference LLaMA2 GGUF models. +>Note: Only LLaMA2 family models are currently supported + +## Requirements +To run these examples with BigDL-LLM, we have some recommended requirements for your machine, please refer to [here](../README.md#recommended-requirements) for more information. + +**Important: Please make sure you have installed `transformers==4.33.0` to run the example.** + +## Example: Load gguf model using `from_gguf()` API +In the example [generate.py](./generate.py), we show a basic use case to load a GGUF LLaMA2 model into `bigdl-llm` using `from_gguf()` API, with BigDL-LLM optimizations. + +### 1. Install +We suggest using conda to manage the Python environment. For more information about conda installation, please refer to [here](https://docs.conda.io/en/latest/miniconda.html#). + +After installing conda, create a Python environment for BigDL-LLM: +```bash +conda create -n llm python=3.9 # recommend to use Python 3.9 +conda activate llm + +# below command will install intel_extension_for_pytorch==2.0.110+xpu as default +# you can install specific ipex/torch version for your need +pip install --pre --upgrade bigdl-llm[xpu] -f https://developer.intel.com/ipex-whl-stable-xpu +pip install transformers==4.33.0 # upgrade transformers +``` + +### 2. Configures OneAPI environment variables +```bash +source /opt/intel/oneapi/setvars.sh +``` + +### 3. Run + +For optimal performance on Arc, it is recommended to set several environment variables. + +```bash +export USE_XETLA=OFF +export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 +``` + +``` +python ./generate.py --model --prompt 'What is AI?' +``` + +More information about arguments can be found in [Arguments Info](#33-arguments-info) section. The expected output can be found in [Sample Output](#34-sample-output) section. + +#### 3.3 Arguments Info +In the example, several arguments can be passed to satisfy your requirements: + +- `--model`: path to GGUF model, it should be a file with name like `llama-2-7b-chat.Q4_0.gguf` +- `--prompt PROMPT`: argument defining the prompt to be infered (with integrated prompt format for chat). It is default to be `'What is AI?'`. +- `--n-predict N_PREDICT`: argument defining the max number of tokens to predict. It is default to be `32`. + +#### 3.4 Sample Output +#### [llama-2-7b-chat.Q4_0.gguf](https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/tree/main) +```log +Inference time: xxxx s +-------------------- Output -------------------- +### HUMAN: +What is AI? + +### RESPONSE: + +AI is a term used to describe a type of computer software that is designed to perform tasks that typically require human intelligence, such as visual perception, speech +``` + +#### [llama-2-7b-chat.Q4_1.gguf](https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/tree/main) +```log +Inference time: xxxx s +-------------------- Output -------------------- +### HUMAN: +What is AI? + +### RESPONSE: + +Artificial intelligence (AI) is the field of study focused on creating machines that can perform tasks that typically require human intelligence, such as understanding language, +``` diff --git a/python/llm/example/GPU/HF-Transformers-AutoModels/Advanced-Quantizations/GGUF/generate.py b/python/llm/example/GPU/HF-Transformers-AutoModels/Advanced-Quantizations/GGUF/generate.py new file mode 100644 index 00000000..14a61fd8 --- /dev/null +++ b/python/llm/example/GPU/HF-Transformers-AutoModels/Advanced-Quantizations/GGUF/generate.py @@ -0,0 +1,63 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +import intel_extension_for_pytorch as ipex +import time +import argparse + +from transformers import LlamaTokenizer +from bigdl.llm.transformers import AutoModelForCausalLM + +# you could tune the prompt based on your own model, +# here the prompt tuning refers to https://huggingface.co/georgesung/llama2_7b_chat_uncensored#prompt-style +LLAMA2_PROMPT_FORMAT = """### HUMAN: +{prompt} + +### RESPONSE: +""" + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Predict Tokens using `generate()` API for Llama2 model') + parser.add_argument('--model', type=str, required=True, + help='Path to a gguf model') + parser.add_argument('--prompt', type=str, default="What is AI?", + help='Prompt to infer') + parser.add_argument('--n-predict', type=int, default=32, + help='Max tokens to predict') + + args = parser.parse_args() + + model_path = args.model + + # Load gguf model and vocab, then convert them to bigdl-llm model and huggingface tokenizer + model, tokenizer = AutoModelForCausalLM.from_gguf(model_path) + model = model.to('xpu') + + # Generate predicted tokens + with torch.inference_mode(): + prompt = LLAMA2_PROMPT_FORMAT.format(prompt=args.prompt) + input_ids = tokenizer.encode(prompt, return_tensors="pt").to('xpu') + st = time.time() + output = model.generate(input_ids, + max_new_tokens=args.n_predict) + torch.xpu.synchronize() + end = time.time() + output = output.cpu() + output_str = tokenizer.decode(output[0], skip_special_tokens=True) + print(f'Inference time: {end-st} s') + print('-'*20, 'Output', '-'*20) + print(output_str)