ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

glm 4v 1st sdp for vision (#12904)

Browse source 
* glm4v 1st sdp

* update glm4v example

* meet code review

* fix style
This commit is contained in:
Xin Qiu 2025-02-28 13:23:27 +08:00 committed by GitHub
parent 5c100ac105
commit e946127613
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 14 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
python/llm/example/GPU/HuggingFace/Multimodal/glm-4v/generate.py
View file

@ -61,7 +61,7 @@ if __name__ == '__main__':
trust_remote_code=True,
use_cache=True,
model_hub=model_hub)
model = model.half().to('xpu')
model = model.to('xpu')
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)

34
python/llm/src/ipex_llm/transformers/models/chatglm4v.py
View file

@ -19,7 +19,7 @@
import torch
from typing import Optional, Tuple, Union
from ipex_llm.transformers.models.common import merge_qkv_base
from ipex_llm.transformers.models.common import merge_qkv_base, padding_qkv_hd
from ipex_llm.transformers.models.common import scaled_dot_product_attention
from ipex_llm.transformers.models.utils import update_past_key_value
from ipex_llm.transformers.models.utils import use_quantize_kv_cache, use_sdp
@ -265,26 +265,18 @@ def visual_attention_forward(self, x: "tensor(B, L, D)") -> "tensor(B, L, D)":
q, k, v = qkv[0], qkv[1], qkv[2]
bsz, q_len, kv_seq_len, head_dim = q.shape
if use_sdp(q_len, kv_seq_len, head_dim, q):
import xe_addons
out = xe_addons.sdp(q, k, v, None)
elif q.device.type == "cpu":
out = torch.nn.functional.scaled_dot_product_attention(q, k, v,
attn_mask=None,
dropout_p=0.,
is_causal=False)
else:
attn_weights = torch.matmul(q / math.sqrt(head_dim),
k.transpose(2, 3)).to(v.dtype)
if kv_seq_len >= 2048 or bsz >= 64:
# for memory considerations, do not upcast attention to fp32
# for long sequences or large batches
attn_weights = torch.nn.functional.softmax(attn_weights, dim=-1)
else:
# upcast attention to fp32
attn_weights = torch.nn.functional.softmax(attn_weights, dim=-1,
dtype=torch.float32).to(v.dtype)
out = torch.matmul(attn_weights, v)
q, k, v = padding_qkv_hd(
q, k, v,
head_dim, 128
)
attn_weights = None
attn_output = scaled_dot_product_attention(
q, k.contiguous(), v.contiguous(),
None, False, 1 / math.sqrt(head_dim)
)
out = attn_output[:, :, :, :head_dim]
output = self.dense(out.transpose(1, 2).reshape(B, L, -1))
output = self.output_dropout(output)
return output