diff --git a/python/llm/setup.py b/python/llm/setup.py
index 9bf0ec75..34724a69 100644
--- a/python/llm/setup.py
+++ b/python/llm/setup.py
@@ -73,6 +73,8 @@ lib_urls["Linux"] = [
     "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/libgptneox_avx2.so",
     "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/libgptneox_avx512.so",
     "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/quantize-gptneox",
+    "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/libbloom_avx2.so",
+    "https://sourceforge.net/projects/analytics-zoo/files/bigdl-llm/libbloom_avx512.so"
 ]
 
 
@@ -102,6 +104,8 @@ def setup_package():
         "libs/libgptneox_avx2.so",
         "libs/libgptneox_avx512.so",
         "libs/quantize-gptneox",
+        "libs/libbloom_avx2.so",
+        "libs/libbloom_avx512.so",
     ]
 
     platform_name = None
diff --git a/python/llm/src/bigdl/llm/ggml/model/bloom/bloom_cpp.py b/python/llm/src/bigdl/llm/ggml/model/bloom/bloom_cpp.py
index e46cc352..f04a8529 100644
--- a/python/llm/src/bigdl/llm/ggml/model/bloom/bloom_cpp.py
+++ b/python/llm/src/bigdl/llm/ggml/model/bloom/bloom_cpp.py
@@ -61,19 +61,19 @@ from ctypes import (
     c_size_t,
 )
 import pathlib
-
+from bigdl.llm.utils import get_avx_flags
 
 # Load the library
 def _load_shared_library(lib_base_name: str):
     # Determine the file extension based on the platform
-    if sys.platform.startswith("linux"):
-        lib_ext = ".so"
-    elif sys.platform == "darwin":
+    if sys.platform.startswith("linux") or sys.platform == "darwin":
         lib_ext = ".so"
     elif sys.platform == "win32":
         lib_ext = ".dll"
     else:
         raise RuntimeError("Unsupported platform")
+    
+    avx = get_avx_flags()
 
     # Construct the paths to the possible shared library names (python/llm/src/bigdl/llm/libs)
     _base_path = pathlib.Path(__file__).parent.parent.parent.parent.resolve()
@@ -81,8 +81,8 @@ def _load_shared_library(lib_base_name: str):
     # Searching for the library in the current directory under the name "libbloom" (default name
     # for bloomcpp) and "bloom" (default name for this repo)
     _lib_paths = [
-        _base_path / f"lib{lib_base_name}{lib_ext}",
-        _base_path / f"{lib_base_name}{lib_ext}",
+        _base_path / f"lib{lib_base_name}{avx}{lib_ext}",
+        _base_path / f"{lib_base_name}{avx}{lib_ext}",
     ]
 
     if "BLOOM_CPP_LIB" in os.environ: