ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

LLM: avoid unnecessary import torch except converting process (#8297)

Browse source
This commit is contained in:
binbin Deng 2023-06-08 14:24:58 +08:00 committed by GitHub
parent f9e2bda04a
commit 08bdfce2d8
1 changed files with 4 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
python/llm/src/bigdl/llm/utils/convert_util.py
View file

@ -64,9 +64,7 @@ from pathlib import Path
from typing import (IO, TYPE_CHECKING, Any, Callable, Dict, Iterable, List, from typing import (IO, TYPE_CHECKING, Any, Callable, Dict, Iterable, List,
Literal, Optional, Sequence, Tuple, TypeVar, Union) Literal, Optional, Sequence, Tuple, TypeVar, Union)
import numpy as np import numpy as np
import torch
from sentencepiece import SentencePieceProcessor from sentencepiece import SentencePieceProcessor
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
from bigdl.llm.utils.common import invalidInputError from bigdl.llm.utils.common import invalidInputError
if TYPE_CHECKING: if TYPE_CHECKING:
@ -1238,6 +1236,8 @@ def bytes_to_unicode():
def _convert_gptneox_hf_to_ggml(model_path, outfile_dir, outtype): def _convert_gptneox_hf_to_ggml(model_path, outfile_dir, outtype):
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch
tokenizer = AutoTokenizer.from_pretrained(model_path) tokenizer = AutoTokenizer.from_pretrained(model_path)
model = AutoModelForCausalLM.from_pretrained(model_path, model = AutoModelForCausalLM.from_pretrained(model_path,
torch_dtype=torch.float16 torch_dtype=torch.float16
@ -1317,6 +1317,8 @@ def _convert_gptneox_hf_to_ggml(model_path, outfile_dir, outtype):
def _convert_bloom_hf_to_ggml(model_path, outfile_dir, outtype): def _convert_bloom_hf_to_ggml(model_path, outfile_dir, outtype):
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
import torch
conv_map = {'word_embeddings': 'tok_embeddings', conv_map = {'word_embeddings': 'tok_embeddings',
'word_embeddings_layernorm': 'norm', 'word_embeddings_layernorm': 'norm',
'input_layernorm': 'attention_norm', 'input_layernorm': 'attention_norm',