diff --git a/python/llm/src/ipex_llm/utils/ipex_importer.py b/python/llm/src/ipex_llm/utils/ipex_importer.py
index 0b60e48c..3fb543af 100644
--- a/python/llm/src/ipex_llm/utils/ipex_importer.py
+++ b/python/llm/src/ipex_llm/utils/ipex_importer.py
@@ -27,37 +27,6 @@ ipex_duplicate_import_error = "intel_extension_for_pytorch has already been auto
     "imported. Please avoid importing it again!"
 
 
-def get_calling_package():
-    """
-    Return calling package name, e.g., ipex_llm.transformers
-    """
-    # Get the current stack frame
-    frame = inspect.currentframe()
-    # Get the caller's frame
-    caller_frame = frame.f_back.f_back
-    # Get the caller's module
-    module = inspect.getmodule(caller_frame)
-    if module:
-        # Return the module's package name
-        return module.__package__
-    return None
-
-
-def custom_ipex_import(name, globals=None, locals=None, fromlist=(), level=0):
-    """
-    Custom import function to avoid importing ipex again
-    """
-    # check import calling pacage
-    calling_package = get_calling_package()
-    if calling_package is not None:
-        return original_import(name, globals, locals, fromlist, level)
-    # Only check ipex for main thread
-    if name == "ipex" or name == "intel_extension_for_pytorch":
-        log4Error.invalidInputError(False,
-                                    ipex_duplicate_import_error)
-    return original_import(name, globals, locals, fromlist, level)
-
-
 class IPEXImporter:
     """
     Auto import Intel Extension for PyTorch as ipex,
@@ -102,8 +71,6 @@ class IPEXImporter:
                                             ipex_duplicate_import_error)
             self.directly_import_ipex()
             self.ipex_version = ipex.__version__
-            # Replace default importer
-            builtins.__import__ = custom_ipex_import
             logging.info("intel_extension_for_pytorch auto imported")
 
     def directly_import_ipex(self):
@@ -127,13 +94,6 @@ class IPEXImporter:
 
         Raises ImportError if cannot import Intel Extension for PyTorch
         """
-        if self.ipex_version is not None:
-            return self.ipex_version
-        # try to import Intel Extension for PyTorch and get version
-        self.directly_import_ipex()
-        self.ipex_version = ipex.__version__
-        # Replace default importer
-        builtins.__import__ = custom_ipex_import
         return self.ipex_version