From 0a552d5bdc15654e385d5eab9a8a81b4f9bec4a6 Mon Sep 17 00:00:00 2001
From: binbin Deng <108676127+plusbang@users.noreply.github.com>
Date: Mon, 18 Sep 2023 11:14:54 +0800
Subject: [PATCH] LLM: fix installation on windows (#8989)

---
 python/llm/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/llm/setup.py b/python/llm/setup.py
index 292996c2..ebdc4b66 100644
--- a/python/llm/setup.py
+++ b/python/llm/setup.py
@@ -50,7 +50,7 @@ llm_home = os.path.join(os.path.dirname(os.path.abspath(__file__)), "src")
 github_artifact_dir = os.path.join(llm_home, '../llm-binary')
 libs_dir = os.path.join(llm_home, "bigdl", "llm", "libs")
 CONVERT_DEP = ['numpy >= 1.22', 'torch',
-               'transformers == 4.31.0', 'sentencepiece',
+               'transformers == 4.31.0', 'sentencepiece', 'tokenizers == 0.13.3',
                # TODO: Support accelerate 0.22.0
                'accelerate == 0.21.0', 'tabulate']
 SERVING_DEP = ['fschat[model_worker, webui] == 0.2.28', 'protobuf']