From 8bc1d8a17cfa5342a62804e71b409c2dc488a8eb Mon Sep 17 00:00:00 2001 From: Zhao Changmin Date: Thu, 7 Sep 2023 10:35:59 +0800 Subject: [PATCH] LLM: Fix discards in `optimize_model` with non-hf models and add openai whisper example (#8877) * openai-whisper --- .../pytorch-model/openai-whisper/readme.md | 74 +++++++++++++++++++ .../pytorch-model/openai-whisper/recognize.py | 58 +++++++++++++++ .../llm/src/bigdl/llm/transformers/convert.py | 9 +++ .../llm/src/bigdl/llm/transformers/model.py | 5 +- .../llm/src/bigdl/llm/transformers/utils.py | 4 + 5 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 python/llm/example/pytorch-model/openai-whisper/readme.md create mode 100644 python/llm/example/pytorch-model/openai-whisper/recognize.py diff --git a/python/llm/example/pytorch-model/openai-whisper/readme.md b/python/llm/example/pytorch-model/openai-whisper/readme.md new file mode 100644 index 00000000..148f603a --- /dev/null +++ b/python/llm/example/pytorch-model/openai-whisper/readme.md @@ -0,0 +1,74 @@ +# Whisper + +In this directory, you will find examples on how you could apply BigDL-LLM INT4 optimizations on general pytorch models, for example Openai Whisper models. For illustration purposes, we utilize the [whisper-tiny](https://github.com/openai/whisper/blob/main/model-card.md) as a reference Whisper model. + +## 0. Requirements +To run these examples with BigDL-LLM, we have some recommended requirements for your machine, please refer to [here](../README.md#recommended-requirements) for more information. + +## Example: Recognize Tokens using `transcribe()` API +In the example [recognize.py](./recognize.py), we show a basic use case for a Whisper model to conduct transcription using `transcribe()` API, with BigDL-LLM INT4 optimizations. +### 1. Install +We suggest using conda to manage environment: +```bash +conda create -n llm python=3.9 +conda activate llm + +pip install bigdl-llm[all] # install bigdl-llm with 'all' option +pip install -U openai-whisper +pip install librosa # required by audio processing +``` + +### 2. Run +``` +python ./recognize.py --repo-id-or-model-path REPO_ID_OR_MODEL_PATH --repo-id-or-data-path REPO_ID_OR_DATA_PATH --language LANGUAGE +``` + +Arguments info: +- `--model-name MODEL_NAME`: argument defining the model name(tiny, medium, base, etc.) for the Whisper model to be downloaded. It is one of the official model names listed by `whisper.available_models()`, or path to a model checkpoint containing the model dimensions and the model state_dict. It is default to be `'tiny'`. +- `--audio-file AUDIO_FILE`: argument defining the path of the audio file to be recognized. +- `--language LANGUAGE`: argument defining language to be transcribed. It is default to be `english`. + +> **Note**: When loading the model in 4-bit, BigDL-LLM converts linear layers in the model into INT4 format. In theory, a *X*B model saved in 16-bit will requires approximately 2*X* GB of memory for loading, and ~0.5*X* GB memory for further inference. +> +> Please select the appropriate size of the Whisper model based on the capabilities of your machine. + + +#### 2.1 Client +On client Windows machine, it is recommended to run directly with full utilization of all cores: +```powershell +python ./recognize.py --audio-file /PATH/TO/AUDIO_FILE +``` + +#### 2.2 Server +For optimal performance on server, it is recommended to set several environment variables (refer to [here](../README.md#best-known-configuration-on-linux) for more information), and run the example with all the physical cores of a single socket. + +E.g. on Linux, +```bash +# set BigDL-Nano env variables +source bigdl-nano-init + +# e.g. for a server with 48 cores per socket +export OMP_NUM_THREADS=48 +numactl -C 0-47 -m 0 python ./recognize.py +``` + +#### 2.3 Sample Output +#### [whisper-tiny](https://github.com/openai/whisper/blob/main/model-card.md) + +For audio file(.wav) download from https://www.youtube.com/watch?v=-LIIf7E-qFI, it should be extracted as: +```log +[00:00.000 --> 00:10.000] I don't know who you are. +[00:10.000 --> 00:15.000] I don't know what you want. +[00:15.000 --> 00:21.000] If you're looking for ransom, I can tell you I don't know money, but what I do have. +[00:21.000 --> 00:24.000] I'm a very particular set of skills. +[00:24.000 --> 00:27.000] The skills I have acquired are very long career. +[00:27.000 --> 00:31.000] The skills that make me a nightmare for people like you. +[00:31.000 --> 00:35.000] If you let my daughter go now, that'll be the end of it. +[00:35.000 --> 00:39.000] I will not look for you. I will not pursue you. +[00:39.000 --> 00:45.000] But if you don't, I will look for you. I will find you. +[00:45.000 --> 00:48.000] And I will kill you. +[00:48.000 --> 00:53.000] Good luck. +Inference time: xxxx s +-------------------- Output -------------------- + I don't know who you are. I don't know what you want. If you're looking for ransom, I can tell you I don't know money, but what I do have. I'm a very particular set of skills. The skills I have acquired are very long career. The skills that make me a nightmare for people like you. If you let my daughter go now, that'll be the end of it. I will not look for you. I will not pursue you. But if you don't, I will look for you. I will find you. And I will kill you. Good luck. +``` diff --git a/python/llm/example/pytorch-model/openai-whisper/recognize.py b/python/llm/example/pytorch-model/openai-whisper/recognize.py new file mode 100644 index 00000000..ca2c5dc4 --- /dev/null +++ b/python/llm/example/pytorch-model/openai-whisper/recognize.py @@ -0,0 +1,58 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import whisper +import time +import librosa +import argparse +from bigdl.llm import optimize_model + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Recognize Tokens using `transcribe()` API for Openai Whisper model') + parser.add_argument('--model-name', type=str, default="tiny", + help="The model name(tiny, medium, base, etc.) for the Whisper model to be downloaded." + "It is one of the official model names listed by `whisper.available_models()`, or" + "path to a model checkpoint containing the model dimensions and the model state_dict.") + parser.add_argument('--audio-file', type=str, required=True, + help='The path of the audio file to be recognized.') + parser.add_argument('--language', type=str, default="English", + help='language to be transcribed') + args = parser.parse_args() + + # Load the input audio + y, sr = librosa.load(args.audio_file) + + # Downsample the audio to 16kHz + target_sr = 16000 + audio = librosa.resample(y, + orig_sr=sr, + target_sr=target_sr) + + # Load whisper model under pytorch framework + model = whisper.load_model(args.model_name) + + # With only one line to enable bigdl optimize on a pytorch model + model = optimize_model(model) + + st = time.time() + result = model.transcribe(audio, verbose=True, language=args.language) + end = time.time() + print(f'Inference time: {end-st} s') + + print('-'*20, 'Output', '-'*20) + print(result["text"]) diff --git a/python/llm/src/bigdl/llm/transformers/convert.py b/python/llm/src/bigdl/llm/transformers/convert.py index ae758b99..6fd3d406 100644 --- a/python/llm/src/bigdl/llm/transformers/convert.py +++ b/python/llm/src/bigdl/llm/transformers/convert.py @@ -41,6 +41,7 @@ from accelerate import init_empty_weights import warnings import transformers import importlib +from .utils import logger def _replace_with_low_bit_linear(model, qtype, modules_to_not_convert=None, @@ -129,6 +130,14 @@ def convert_forward(m, target_m, new_forward): def optimize(model): from packaging import version from bigdl.llm.transformers.models.llama import llama_attention_forward_4_31 + from transformers.modeling_utils import PreTrainedModel + + # All huggingface format models are inherited from `PreTrainedModel` + if not isinstance(model, PreTrainedModel): + logger.info("It is not supported that optimizing a model isn't belong to" + "huggingface transformers with `optimize_model=True` for now.") + return model + trans_version = transformers.__version__ if version.parse(trans_version) >= version.parse("4.31.0"): convert_forward( diff --git a/python/llm/src/bigdl/llm/transformers/model.py b/python/llm/src/bigdl/llm/transformers/model.py index 180d2916..18eeb693 100644 --- a/python/llm/src/bigdl/llm/transformers/model.py +++ b/python/llm/src/bigdl/llm/transformers/model.py @@ -23,10 +23,7 @@ from bigdl.llm.ggml.quantize import ggml_tensor_qtype from bigdl.llm.utils.common import invalidInputError import torch import copy -import logging - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO) -logger = logging.getLogger(__name__) +from .utils import logger def save_low_bit(self, *args, **kwargs): diff --git a/python/llm/src/bigdl/llm/transformers/utils.py b/python/llm/src/bigdl/llm/transformers/utils.py index 7be59e4f..c0fa9d2a 100644 --- a/python/llm/src/bigdl/llm/transformers/utils.py +++ b/python/llm/src/bigdl/llm/transformers/utils.py @@ -45,6 +45,10 @@ from ..utils.common import invalidInputError from typing import Union import torch from torch import nn +import logging + +logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO) +logger = logging.getLogger(__name__) WEIGHTS_NAME = "pytorch_model.bin"