From ca3e59a1dc2e8529e8c618b882393597c78d6455 Mon Sep 17 00:00:00 2001
From: Ruonan Wang <105281011+rnwang04@users.noreply.github.com>
Date: Fri, 11 Aug 2023 14:51:30 +0800
Subject: [PATCH] LLM: support stop for starcoder native int4 stream (#8734)

---
 python/llm/src/bigdl/llm/ggml/model/starcoder/starcoder.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/python/llm/src/bigdl/llm/ggml/model/starcoder/starcoder.py b/python/llm/src/bigdl/llm/ggml/model/starcoder/starcoder.py
index e54392d7..3432a180 100644
--- a/python/llm/src/bigdl/llm/ggml/model/starcoder/starcoder.py
+++ b/python/llm/src/bigdl/llm/ggml/model/starcoder/starcoder.py
@@ -295,6 +295,10 @@ class Starcoder(GenerationMixin):
                 text = self.detokenize([token]).decode("utf-8", errors="ignore")
                 if text.endswith("<|endoftext|>"):
                     print('\n')
+                    return
+                elif text is not None and text in stop:
+                    print('\n')
+                    return
                 else:
                     yield {
                         "id": completion_id,