diff --git a/.github/workflows/llm_unit_tests.yml b/.github/workflows/llm_unit_tests.yml
index 1c6dab55..c8db1b7d 100644
--- a/.github/workflows/llm_unit_tests.yml
+++ b/.github/workflows/llm_unit_tests.yml
@@ -363,6 +363,7 @@ jobs:
           pip install -U langchain==0.0.184
           pip install -U chromadb==0.3.25
           pip install -U pandas==2.0.3
+          pip install -U langchain-community==0.0.27
           # Specific oneapi position on arc ut test machines
           if [[ '${{ matrix.pytorch-version }}' == '2.1' ]]; then
             source /opt/intel/oneapi/setvars.sh
diff --git a/python/llm/test/run-llm-langchain-tests-gpu.sh b/python/llm/test/run-llm-langchain-tests-gpu.sh
index bff28198..130fd400 100644
--- a/python/llm/test/run-llm-langchain-tests-gpu.sh
+++ b/python/llm/test/run-llm-langchain-tests-gpu.sh
@@ -15,6 +15,12 @@ start=$(date "+%s")
 
 python -m pytest -s ${LLM_INFERENCE_TEST_DIR}
 
+mkdir ${LLM_INFERENCE_TEST_DIR}/tmp_wget_dir
+wget https://raw.githubusercontent.com/langchain-ai/langchain/master/libs/community/tests/integration_tests/llms/test_bigdl.py -P ${LLM_INFERENCE_TEST_DIR}/tmp_wget_dir
+sed -i "s,model_id=\"[^\"]*\",model_id=\"$LLAMA2_7B_ORIGIN_PATH\",g" ${LLM_INFERENCE_TEST_DIR}/tmp_wget_dir/test_bigdl.py
+python -m pytest -s ${LLM_INFERENCE_TEST_DIR}/tmp_wget_dir
+rm -rf ${LLM_INFERENCE_TEST_DIR}/tmp_wget_dir
+
 now=$(date "+%s")
 time=$((now-start))