From f053688cada3c8dd964f0d2a5be2a11f64cc9b01 Mon Sep 17 00:00:00 2001 From: dingbaorong Date: Fri, 27 Oct 2023 18:59:20 +0800 Subject: [PATCH] add cpu example of LLaVA (#9269) * add LLaVA cpu example * Small text updates * update link --------- Co-authored-by: Yuwen Hu --- README.md | 2 + python/llm/README.md | 1 + .../CPU/PyTorch-Models/Model/README.md | 1 + .../CPU/PyTorch-Models/Model/llava/README.md | 86 +++++ .../PyTorch-Models/Model/llava/generate.py | 337 ++++++++++++++++++ 5 files changed, 427 insertions(+) create mode 100644 python/llm/example/CPU/PyTorch-Models/Model/llava/README.md create mode 100644 python/llm/example/CPU/PyTorch-Models/Model/llava/generate.py diff --git a/README.md b/README.md index ff02fb83..58b26bf7 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,8 @@ Over 20 models have been optimized/verified on `bigdl-llm`, including *LLaMA/LLa | Phi-1_5 | [link](python/llm/example/CPU/HF-Transformers-AutoModels/Model/phi-1_5) | [link](python/llm/example/GPU/HF-Transformers-AutoModels/Model/phi-1_5) | | Flan-t5 | [link](python/llm/example/CPU/HF-Transformers-AutoModels/Model/flan-t5) | [link](python/llm/example/GPU/HF-Transformers-AutoModels/Model/flan-t5) | | Qwen-VL | [link](python/llm/example/CPU/HF-Transformers-AutoModels/Model/qwen-vl) | | +| LLaVA | [link](python/llm/example/CPU/PyTorch-Models/Model/llava) | | + ***For more details, please refer to the `bigdl-llm` [Document](https://test-bigdl-llm.readthedocs.io/en/main/doc/LLM/index.html), [Readme](python/llm), [Tutorial](https://github.com/intel-analytics/bigdl-llm-tutorial) and [API Doc](https://bigdl.readthedocs.io/en/latest/doc/PythonAPI/LLM/index.html).*** diff --git a/python/llm/README.md b/python/llm/README.md index 6032a992..628496e6 100644 --- a/python/llm/README.md +++ b/python/llm/README.md @@ -61,6 +61,7 @@ Over 20 models have been optimized/verified on `bigdl-llm`, including *LLaMA/LLa | Phi-1_5 | [link](example/CPU/HF-Transformers-AutoModels/Model/phi-1_5) | [link](example/GPU/HF-Transformers-AutoModels/Model/phi-1_5) | | Flan-t5 | [link](example/CPU/HF-Transformers-AutoModels/Model/flan-t5) | [link](example/GPU/HF-Transformers-AutoModels/Model/flan-t5) | | Qwen-VL | [link](example/CPU/HF-Transformers-AutoModels/Model/qwen-vl) | | +| LLaVA | [link](example/CPU/PyTorch-Models/Model/llava) | | ### Working with `bigdl-llm` diff --git a/python/llm/example/CPU/PyTorch-Models/Model/README.md b/python/llm/example/CPU/PyTorch-Models/Model/README.md index 40288dd4..090e3dc0 100644 --- a/python/llm/example/CPU/PyTorch-Models/Model/README.md +++ b/python/llm/example/CPU/PyTorch-Models/Model/README.md @@ -13,6 +13,7 @@ You can use `optimize_model` API to accelerate general PyTorch models on Intel s | Flan-t5 | [link](flan-t5) | | Phi-1_5 | [link](phi-1_5) | | Qwen-VL | [link](qwen-vl) | +| LLaVA | [link](llava) | ## Recommended Requirements To run the examples, we recommend using Intel® Xeon® processors (server), or >= 12th Gen Intel® Core™ processor (client). diff --git a/python/llm/example/CPU/PyTorch-Models/Model/llava/README.md b/python/llm/example/CPU/PyTorch-Models/Model/llava/README.md new file mode 100644 index 00000000..9f88b14e --- /dev/null +++ b/python/llm/example/CPU/PyTorch-Models/Model/llava/README.md @@ -0,0 +1,86 @@ +# LLaVA + +In this directory, you will find examples on how you could apply BigDL-LLM INT4 optimizations on LLaVA models. For illustration purposes, we utilize the [liuhaotian/llava-v1.5-13b](https://huggingface.co/liuhaotian/llava-v1.5-13b) as a reference LLaVA model. + +## 0. Requirements +To run these examples with BigDL-LLM, we have some recommended requirements for your machine, please refer to [here](../README.md#recommended-requirements) for more information. + +## Example: Multi-turn chat centered around an image using `generate()` API +In the example [generate.py](./generate.py), we show a basic use case for a LLaVA model to start a multi-trun chat centered around an image using `generate()` API, with BigDL-LLM INT4 optimizations. +### 1. Install +We suggest using conda to manage the Python environment. For more information about conda installation, please refer to [here](https://docs.conda.io/en/latest/miniconda.html#). + +After installing conda, create a Python environment for BigDL-LLM: +```bash +conda create -n llm python=3.9 # recommend to use Python 3.9 +conda activate llm + +pip install --pre --upgrade bigdl-llm[all] # install the latest bigdl-llm nightly build with 'all' option + +git clone -b v1.1.1 --depth=1 https://github.com/haotian-liu/LLaVA.git # clone the llava libary +pip install einops # install dependencies required by llava +cp generate.py ./LLaVA/ # copy our example to the LLaVA folder +cd LLaVA # change the working directory to the LLaVA folder +``` + +### 2. Run +After setting up the Python environment, you could run the example by following steps. + +> **Note**: When loading the model in 4-bit, BigDL-LLM converts linear layers in the model into INT4 format. In theory, a *X*B model saved in 16-bit will requires approximately 2*X* GB of memory for loading, and ~0.5*X* GB memory for further inference. +> +> Please select the appropriate size of the LLaVA model based on the capabilities of your machine. + +#### 2.1 Client +On client Windows machines, it is recommended to run directly with full utilization of all cores: +```powershell +python ./generate.py --image-path-or-url 'https://llava-vl.github.io/static/images/monalisa.jpg' +``` +More information about arguments can be found in [Arguments Info](#23-arguments-info) section. The expected output can be found in [Sample Output](#24-sample-output) section. + +If you encounter some network error (which means your machine is unable to access huggingface.co) when running this example, refer to [Trouble Shooting](#3-trouble-shooting) section. + +#### 2.2 Server +For optimal performance on server, it is recommended to set several environment variables (refer to [here](../README.md#best-known-configuration-on-linux) for more information), and run the example with all the physical cores of a single socket. + +E.g. on Linux, +```bash +# set BigDL-Nano env variables +source bigdl-nano-init + +# e.g. for a server with 48 cores per socket +export OMP_NUM_THREADS=48 +numactl -C 0-47 -m 0 python ./generate.py --image-path-or-url 'https://llava-vl.github.io/static/images/monalisa.jpg' +``` +More information about arguments can be found in [Arguments Info](#23-arguments-info) section. The expected output can be found in [Sample Output](#24-sample-output) section. + +#### 2.3 Arguments Info +In the example, several arguments can be passed to satisfy your requirements: + +- `--repo-id-or-model-path REPO_ID_OR_MODEL_PATH`: argument defining the huggingface repo id for the LLaVA model (e.g. `liuhaotian/llava-v1.5-13b`) to be downloaded, or the path to the huggingface checkpoint folder. It is default to be `'liuhaotian/llava-v1.5-13b'`. +- `--image-path-or-url IMAGE_PATH_OR_URL`: argument defining the input image that the chat will focus on. It is required. +- `--n-predict N_PREDICT`: argument defining the max number of tokens to predict. It is default to be `512`. + + +#### 2.4 Sample Output +#### [liuhaotian/llava-v1.5-13b](https://huggingface.co/liuhaotian/llava-v1.5-13b) + +```log +USER: 你知道这幅画是谁画的吗？ +ASSISTANT: 这幅画是由著名的文艺复兴画家达芬奇（Leonardo da Vinci）画的。该画是他的代表作之一，是出自意大利佛罗伦萨的博物馆。画中的女子被认为是一位不为人知的模特，而且画作可能还有一个人物底版，这可能使得这幅画的价值更高。 +``` + +The sample inpuit image is: + + + +### 3. Trouble shooting + +#### 3.1 SSLError +If you encounter the following output, it means your machine has some trouble accessing huggingface.co. +```log +requests.exceptions.SSLError: (MaxRetryError("HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /openai/clip-vit-large-patch14-336/resolve/main/config.json (Caused by SSLError(SSLZeroReturnError(6, 'TLS/SSL connection has been closed (EOF) (_ssl.c:1129)')))"), +``` + +You can resolve this problem with the following steps: +1. Download https://huggingface.co/openai/clip-vit-large-patch14-336 on some machine that can access huggingface.co, and put it in huggingface's local cache (default to be `~/.cache/huggingface/hub`) on the machine that you are going to run this example. +2. Set the environment variable (`export TRANSFORMERS_OFFLINE=1`) before you run the example. \ No newline at end of file diff --git a/python/llm/example/CPU/PyTorch-Models/Model/llava/generate.py b/python/llm/example/CPU/PyTorch-Models/Model/llava/generate.py new file mode 100644 index 00000000..dab360b0 --- /dev/null +++ b/python/llm/example/CPU/PyTorch-Models/Model/llava/generate.py @@ -0,0 +1,337 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Some parts of this file is adapted from +# https://github.com/haotian-liu/LLaVA/blob/v1.1.1/llava/model/builder.py +# and +# https://github.com/haotian-liu/LLaVA/blob/v1.1.1/llava/serve/cli.py +# +# Copyright 2023 Haotian Liu +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import torch +import time + +from transformers import AutoModelForCausalLM +from llava.model.language_model.llava_llama import LlavaLlamaForCausalLM +from transformers import AutoTokenizer + +from llava.constants import ( + DEFAULT_IMAGE_PATCH_TOKEN, + IMAGE_TOKEN_INDEX, + DEFAULT_IMAGE_TOKEN, + DEFAULT_IM_START_TOKEN, + DEFAULT_IM_END_TOKEN +) +from llava.conversation import conv_templates, SeparatorStyle +from llava.utils import disable_torch_init +from llava.mm_utils import ( + process_images, + tokenizer_image_token, + get_model_name_from_path, + KeywordsStoppingCriteria +) + +from bigdl.llm import optimize_model + +# Lod the pretained model. +# Adapted from llava.model.builder.load_pretrained_model. +def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, + device_map="auto", device="cpu"): + kwargs = {"device_map": device_map} + kwargs['torch_dtype'] = torch.float32 + + if 'llava' in model_name.lower(): + # Load LLaVA model + if 'lora' in model_name.lower() and model_base is None: + warnings.warn('There is `lora` in model name but no `model_base` is provided.' + 'If you are loading a LoRA model, please provide the `model_base` argument' + '. Detailed instruction:' + 'https://github.com/haotian-liu/LLaVA#launch-a-model-worker-lora-weights-unmerged.') + if 'lora' in model_name.lower() and model_base is not None: + lora_cfg_pretrained = AutoConfig.from_pretrained(model_path) + tokenizer = AutoTokenizer.from_pretrained( + model_base, use_fast=False) + print('Loading LLaVA from base model...') + model = LlavaLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, + config=lora_cfg_pretrained, **kwargs) + token_num, tokem_dim = model.lm_head.out_features, model.lm_head.in_features + if model.lm_head.weight.shape[0] != token_num: + model.lm_head.weight = torch.nn.Parameter(torch.empty( + token_num, tokem_dim, device=model.device, dtype=model.dtype)) + model.model.embed_tokens.weight = torch.nn.Parameter(torch.empty( + token_num, tokem_dim, device=model.device, dtype=model.dtype)) + + print('Loading additional LLaVA weights...') + if os.path.exists(os.path.join(model_path, 'non_lora_trainables.bin')): + non_lora_trainables = torch.load(os.path.join(model_path, + 'non_lora_trainables.bin'), + map_location='cpu') + else: + # this is probably from HF Hub + from huggingface_hub import hf_hub_download + + def load_from_hf(repo_id, filename, subfolder=None): + cache_file = hf_hub_download( + repo_id=repo_id, + filename=filename, + subfolder=subfolder) + return torch.load(cache_file, map_location='cpu') + non_lora_trainables = load_from_hf( + model_path, 'non_lora_trainables.bin') + non_lora_trainables = {(k[11:] if k.startswith( + 'base_model.') else k): v for k, v in non_lora_trainables.items()} + if any(k.startswith('model.model.') for k in non_lora_trainables): + non_lora_trainables = {(k[6:] if k.startswith( + 'model.') else k): v for k, v in non_lora_trainables.items()} + model.load_state_dict(non_lora_trainables, strict=False) + + from peft import PeftModel + print('Loading LoRA weights...') + model = PeftModel.from_pretrained(model, model_path) + print('Merging LoRA weights...') + model = model.merge_and_unload() + print('Model is loaded...') + elif model_base is not None: + # this may be mm projector only + print('Loading LLaVA from base model...') + if 'mpt' in model_name.lower(): + if not os.path.isfile(os.path.join(model_path, 'configuration_mpt.py')): + shutil.copyfile(os.path.join(model_base, 'configuration_mpt.py'), os.path.join( + model_path, 'configuration_mpt.py')) + tokenizer = AutoTokenizer.from_pretrained( + model_base, use_fast=True) + cfg_pretrained = AutoConfig.from_pretrained( + model_path, trust_remote_code=True) + model = LlavaMPTForCausalLM.from_pretrained( + model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs) + else: + tokenizer = AutoTokenizer.from_pretrained( + model_base, use_fast=False) + cfg_pretrained = AutoConfig.from_pretrained(model_path) + model = LlavaLlamaForCausalLM.from_pretrained( + model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs) + + mm_projector_weights = torch.load(os.path.join( + model_path, 'mm_projector.bin'), map_location='cpu') + mm_projector_weights = {k: v.to(torch.float32) + for k, v in mm_projector_weights.items()} + model.load_state_dict(mm_projector_weights, strict=False) + else: + if 'mpt' in model_name.lower(): + tokenizer = AutoTokenizer.from_pretrained( + model_path, use_fast=True) + model = LlavaMPTForCausalLM.from_pretrained( + model_path, low_cpu_mem_usage=True, **kwargs) + else: + tokenizer = AutoTokenizer.from_pretrained( + model_path, use_fast=False) + model = LlavaLlamaForCausalLM.from_pretrained( + model_path, low_cpu_mem_usage=True, **kwargs) + else: + # Load language model + if model_base is not None: + # PEFT model + from peft import PeftModel + tokenizer = AutoTokenizer.from_pretrained( + model_base, use_fast=False) + model = AutoModelForCausalLM.from_pretrained( + model_base, torch_dtype=torch.float32, low_cpu_mem_usage=True, device_map="auto") + print(f"Loading LoRA weights from {model_path}") + model = PeftModel.from_pretrained(model, model_path) + print(f"Merging weights") + model = model.merge_and_unload() + print('Convert to FP32...') + model.to(torch.float32) + else: + use_fast = False + if 'mpt' in model_name.lower(): + tokenizer = AutoTokenizer.from_pretrained( + model_path, use_fast=True) + model = AutoModelForCausalLM.from_pretrained( + model_path, low_cpu_mem_usage=True, trust_remote_code=True, **kwargs) + else: + tokenizer = AutoTokenizer.from_pretrained( + model_path, use_fast=False) + model = AutoModelForCausalLM.from_pretrained( + model_path, low_cpu_mem_usage=True, **kwargs) + + image_processor = None + + if 'llava' in model_name.lower(): + mm_use_im_start_end = getattr( + model.config, "mm_use_im_start_end", False) + mm_use_im_patch_token = getattr( + model.config, "mm_use_im_patch_token", True) + if mm_use_im_patch_token: + tokenizer.add_tokens( + [DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True) + if mm_use_im_start_end: + tokenizer.add_tokens( + [DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True) + model.resize_token_embeddings(len(tokenizer)) + + vision_tower = model.get_vision_tower() + if not vision_tower.is_loaded: + vision_tower.load_model() + vision_tower.to(device=device, dtype=torch.float32) + image_processor = vision_tower.image_processor + + if hasattr(model.config, "max_sequence_length"): + context_len = model.config.max_sequence_length + else: + context_len = 2048 + + return tokenizer, model, image_processor, context_len + +# Initialize conversation from templates and get conversation roles. +def get_conv_and_role(model_name): + if 'llama-2' in model_name.lower(): + conv_mode = "llava_llama_2" + elif "v1" in model_name.lower(): + conv_mode = "llava_v1" + elif "mpt" in model_name.lower(): + conv_mode = "mpt" + else: + conv_mode = "llava_v0" + + conv = conv_templates[conv_mode].copy() + if "mpt" in model_name.lower(): + roles = ('user', 'assistant') + else: + roles = conv.roles + + return conv, roles + +# Load image from a url or path. +def load_image(image_file): + import requests + from PIL import Image + from io import BytesIO + + if image_file.startswith('http://') or image_file.startswith('https://'): + response = requests.get(image_file) + image = Image.open(BytesIO(response.content)).convert('RGB') + else: + image = Image.open(image_file).convert('RGB') + return image + +def generate_image_tensor(image_file): + image = load_image(image_file) + model_cfg = {"image_aspect_ratio": 'pad'} + image_tensor = process_images([image], image_processor, model_cfg) + return image_tensor + +# Generate input prompt with user input. +def get_prompt(mm_use_im_start_end, first_round, conv, user_input): + if first_round: + # first message + if mm_use_im_start_end: + user_input = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + \ + DEFAULT_IM_END_TOKEN + '\n' + user_input + else: + user_input = DEFAULT_IMAGE_TOKEN + '\n' + user_input + conv.append_message(conv.roles[0], user_input) + else: + # later messages + conv.append_message(conv.roles[0], user_input) + conv.append_message(conv.roles[1], None) + return conv.get_prompt() + +def get_stopping_criteria(conv, tokenizer, input_ids): + stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2 + keywords = [stop_str] + stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids) + return stopping_criteria + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Predict Tokens using `generate()` API for LLaVA model') + parser.add_argument('--repo-id-or-model-path', type=str, default="liuhaotian/llava-v1.5-13b", + help='The huggingface repo id for the LLaVA model to be downloaded' + ', or the path to the huggingface checkpoint folder') + parser.add_argument('--image-path-or-url', type=str, + required=True, help='Image path or url for the input image that the chat will focus on') + parser.add_argument('--n-predict', type=int, default=512, + help='Max tokens to predict') + + args = parser.parse_args() + model_path = args.repo_id_or_model_path + model_name = get_model_name_from_path(model_path) + + # Disable the redundant torch default initialization to accelerate model creation. + disable_torch_init() + + # Load model + tokenizer, model, image_processor, _ = load_pretrained_model(model_path=model_path, + model_base=None, + model_name=model_name) + + # With only one line to enable BigDL-LLM optimization on model + model = optimize_model(model) + + # Genereate imgea tensor + image_tensor = generate_image_tensor(args.image_path_or_url) + + # Get conversation template and roles + conv, roles = get_conv_and_role(model_name) + + first_round = True + while True: + try: + user_input = input(f"{roles[0]}: ") + except EOFError: + user_input = "" + if not user_input: + print("exit...") + break + + prompt = get_prompt(model.config.mm_use_im_start_end, first_round, conv, user_input) + first_round = False + input_ids = tokenizer_image_token( + prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0) + stopping_criteria = get_stopping_criteria(conv, tokenizer, input_ids) + + # Generate predicted tokens + with torch.inference_mode(): + st = time.time() + output_ids = model.generate( + input_ids, + images=image_tensor, + do_sample=True, + max_new_tokens=args.n_predict, + use_cache=True, + stopping_criteria=[stopping_criteria]) + end = time.time() + #print(f'Inference time: {end-st} s') + + outputs = tokenizer.decode( + output_ids[0, input_ids.shape[1]:], skip_special_tokens=True).strip() + conv.messages[-1][-1] = outputs + print(f"{roles[1]}: ", end="") + print(outputs)