ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

disable fused layer norm on UHD (#10130)

Browse source
This commit is contained in:
Yishuo Wang 2024-02-08 10:20:01 +08:00 committed by GitHub
parent a8450fc300
commit 1aa0c623ce
1 changed files with 3 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
python/llm/src/bigdl/llm/transformers/models/utils.py
View file

@ -309,12 +309,13 @@ def use_xmx(x: torch.Tensor, qtype: int):
def use_fused_layer_norm(x: torch.Tensor, training: bool): def use_fused_layer_norm(x: torch.Tensor, training: bool):
device = get_xpu_device_type(x)
return ( return (
not training not training
and not x.requires_grad and not x.requires_grad
and x.device.type == 'xpu' and device in ["arc", "flex", "pvc", "mtl"] # fused layer norm cannot run on UHD
and ( and (
get_xpu_device_type(x) not in ["arc", "flex"] device == "mtl" # fused layer norm conflicts with XMX, so disable it when using XMX
or x.numel() // x.size(-1) == 1 or x.numel() // x.size(-1) == 1
) )
) )