ayo/ipex-llm
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

[LLM] change default n_threads of native int4 langchain API (#8779)

Browse source
This commit is contained in:
Yishuo Wang 2023-08-21 13:30:12 +08:00 committed by GitHub
parent 3d1f2b44f8
commit 611c1fb628
1 changed files with 3 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
python/llm/src/bigdl/llm/langchain/llms/bigdlllm.py
View file

@ -110,7 +110,7 @@ class BigdlNativeLLM(LLM):
use_mlock: bool = Field(False, alias="use_mlock") use_mlock: bool = Field(False, alias="use_mlock")
"""Force system to keep model in RAM.""" """Force system to keep model in RAM."""
n_threads: Optional[int] = Field(2, alias="n_threads") n_threads: Optional[int] = Field(-1, alias="n_threads")
"""Number of threads to use.""" """Number of threads to use."""
n_batch: Optional[int] = Field(512, alias="n_batch") n_batch: Optional[int] = Field(512, alias="n_batch")
@ -158,7 +158,7 @@ class BigdlNativeLLM(LLM):
@root_validator() @root_validator()
def validate_environment(cls, values: Dict) -> Dict: def validate_environment(cls, values: Dict) -> Dict:
"""Validate that bigdl-llm is installed, family is supported""" """Validate that bigdl-llm is installed, family is supported"""
model_path = values["model_path"] model_path = values["model_path"]
model_param_names = [ model_param_names = [
@ -230,7 +230,7 @@ class BigdlNativeLLM(LLM):
def _identifying_params(self) -> Dict[str, Any]: def _identifying_params(self) -> Dict[str, Any]:
"""Get the identifying parameters.""" """Get the identifying parameters."""
return {**{"model_path": self.model_path, return {**{"model_path": self.model_path,
"model_family": self.model_family}, "model_family": self.model_family},
**self._default_params} **self._default_params}
@property @property