ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

[LLM] vLLM: Fix kv cache init (#9771)

Browse source 
Fix kv cache init
This commit is contained in:
Xiangyu Tian 2023-12-25 14:17:06 +08:00 committed by GitHub
parent 449b387125
commit 30dab36f76
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
python/llm/src/bigdl/llm/vllm/engine/llm_engine.py
View file

@ -127,7 +127,7 @@ class LLMEngine:
# self.parallel_config = parallel_config # self.parallel_config = parallel_config
self.scheduler_config = scheduler_config self.scheduler_config = scheduler_config
self.log_stats = log_stats self.log_stats = log_stats
self.kv_cache = [[dict() for _ in range(2)] for _ in range(32)] self.kv_cache = [[dict(), dict()] for _ in range(model_config.hf_config.num_hidden_layers)]
# self._verify_args() # self._verify_args()
self.tokenizer = get_tokenizer( self.tokenizer = get_tokenizer(