From b06a3146c87bd7b6eb27c75e1edb05c02b7a66b4 Mon Sep 17 00:00:00 2001 From: "Chen, Zhentao" Date: Thu, 21 Dec 2023 10:40:52 +0800 Subject: [PATCH] Fix 70b oom (#9738) * add default value to bigdl llm * fix model oom --- python/llm/dev/benchmark/harness/bigdl_llm.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/llm/dev/benchmark/harness/bigdl_llm.py b/python/llm/dev/benchmark/harness/bigdl_llm.py index c6bd843d..f14d680a 100644 --- a/python/llm/dev/benchmark/harness/bigdl_llm.py +++ b/python/llm/dev/benchmark/harness/bigdl_llm.py @@ -45,9 +45,12 @@ class BigDLLM(AutoCausalLM): keys = list(kwargs.keys()) for k in keys: if k not in self.AutoCausalLM_ARGS: - self.bigdl_llm_kwargs[k] = kwargs[k] - kwargs.pop(k) + self.bigdl_llm_kwargs[k] = kwargs.pop(k) + + self.bigdl_llm_kwargs['use_cache'] = kwargs.get('use_cache', True) + self.bigdl_llm_kwargs['optimize_model'] = kwargs.get('optimize_model', True) AutoModelForCausalLM.from_pretrained = partial(AutoModelForCausalLM.from_pretrained, **self.bigdl_llm_kwargs) + kwargs['trust_remote_code'] = kwargs.get('trust_remote_code', True) super().__init__(*args, **kwargs)