ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

fix gptq of llama (#11749)

Browse source 
* fix gptq of llama

* small fix
This commit is contained in:
Ruonan Wang 2024-08-09 11:39:25 +03:00 committed by GitHub
parent dd46c141bd
commit 7e917d6cfb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 20 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
python/llm/example/GPU/HuggingFace/Advanced-Quantizations/GPTQ/generate.py
View file

@ -49,10 +49,7 @@ if __name__ == '__main__':
trust_remote_code=True,).to("xpu")
# Load tokenizer
if "qwen" in model_path.lower():
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
else:
tokenizer = LlamaTokenizer.from_pretrained(model_path, trust_remote_code=True)
# Generate predicted tokens
with torch.inference_mode():

5
python/llm/src/ipex_llm/transformers/models/common.py
View file

@ -19,6 +19,8 @@ from typing import List
def merge_linear(linears: List[torch.nn.Linear]) -> torch.nn.Linear:
if hasattr(linears[0], "weight"):
# For GPTQ model, it might be qweight
new_weight = torch.cat(list(linear.weight.data for linear in linears), dim=0)
if linears[0].bias is not None:
new_linear = torch.nn.Linear(0, 0, bias=True)
@ -30,6 +32,8 @@ def merge_linear(linears: List[torch.nn.Linear]) -> torch.nn.Linear:
new_linear.in_features = new_weight.size(1)
new_linear.out_features = new_weight.size(0)
return new_linear
else:
return None
def merge_qkv_base(module: torch.nn.Module, attention_class):
@ -39,6 +43,7 @@ def merge_qkv_base(module: torch.nn.Module, attention_class):
module.k_proj,
module.v_proj,
])
if qkv_proj is not None:
module.qkv_proj = qkv_proj
del module.q_proj, module.k_proj, module.v_proj