diff --git a/python/llm/src/ipex_llm/transformers/convert.py b/python/llm/src/ipex_llm/transformers/convert.py
index e60d5e1d..c1a3d5e0 100644
--- a/python/llm/src/ipex_llm/transformers/convert.py
+++ b/python/llm/src/ipex_llm/transformers/convert.py
@@ -1533,8 +1533,6 @@ def _optimize_post(model, lightweight_bmm=False):
             model,
             module.Phi3RMSNorm,
             phi3_rms_norm_forward)
-        # Empty cache after the first attention to run long context.
-        model.model.layers[0].self_attn.register_forward_hook(empty_cache_post)
     elif model.config.model_type == 'yuan':
         modeling_module_name = model.__class__.__module__
         module = importlib.import_module(modeling_module_name)