LLM: fix get env KV_CACHE_ALLOC_BLOCK_LENGTH type. (#10771)

python/llm/src/ipex_llm/transformers/models/aquila.py

@@ -50,7 +50,7 @@ from ipex_llm.utils.common import log4Error
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def aquila_attention_forward(

python/llm/src/ipex_llm/transformers/models/baichuan.py

@@ -37,7 +37,7 @@ from ipex_llm.transformers.models.utils import apply_rotary_pos_emb_no_cache_xpu
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def baichuan_attention_forward_7b(

python/llm/src/ipex_llm/transformers/models/baichuan2.py

@@ -46,7 +46,7 @@ except ImportError:
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def baichuan_13b_rms_norm_forward(self, hidden_states):

python/llm/src/ipex_llm/transformers/models/bloom.py

@@ -42,7 +42,7 @@ from ipex_llm.transformers.models.utils import init_kv_cache, extend_kv_cache, a
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def dropout_add(x: torch.Tensor, residual: torch.Tensor, prob: float, training: bool):

python/llm/src/ipex_llm/transformers/models/chatglm.py

@@ -40,7 +40,7 @@ def apply_rotary_pos_emb_index(q, k, cos, sin, position_id):
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 KV_CACHE_ALLOC_MIN_LENGTH = 512
 
 

python/llm/src/ipex_llm/transformers/models/chatglm2.py

@@ -30,7 +30,7 @@ from ipex_llm.transformers.models.utils import use_esimd_sdp
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 KV_CACHE_ALLOC_MIN_LENGTH = 512
 
 

python/llm/src/ipex_llm/transformers/models/chatglm2_32k.py

@@ -25,7 +25,7 @@ from ipex_llm.transformers.models.utils import init_kv_cache, extend_kv_cache, a
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 KV_CACHE_ALLOC_MIN_LENGTH = 512
 
 

python/llm/src/ipex_llm/transformers/models/decilm.py

@@ -43,7 +43,7 @@ from ipex_llm.utils.common import invalidInputError
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def decilm_attention_forward_4_35_2(

python/llm/src/ipex_llm/transformers/models/falcon.py

@@ -43,7 +43,7 @@ import warnings
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 # Copied from transformers.models.llama.modeling_llama.rotate_half

python/llm/src/ipex_llm/transformers/models/gemma.py

@@ -45,7 +45,7 @@ from ipex_llm.transformers.models.utils import decoding_fast_path_qtype_check
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):

python/llm/src/ipex_llm/transformers/models/gptj.py

@@ -28,7 +28,7 @@ from ipex_llm.utils.common import invalidInputError
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def _get_embed_positions(self, position_ids):

python/llm/src/ipex_llm/transformers/models/gptneox.py

@@ -40,7 +40,7 @@ from ipex_llm.transformers.models.utils import apply_rotary_pos_emb_no_cache_xpu
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def gptneox_attention_forward(

python/llm/src/ipex_llm/transformers/models/internlm.py

@@ -50,7 +50,7 @@ from ipex_llm.transformers.models.utils import apply_rotary_pos_emb_no_cache_xpu
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def internlm_attention_forward(

python/llm/src/ipex_llm/transformers/models/llama.py

@@ -83,7 +83,7 @@ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
                                                            n_rep, slen, head_dim)
     return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 _ipex_version = None

python/llm/src/ipex_llm/transformers/models/mistral.py

@@ -63,7 +63,7 @@ except ImportError:
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:

python/llm/src/ipex_llm/transformers/models/mixtral.py

@@ -60,7 +60,7 @@ from ipex_llm.transformers.low_bit_linear import IQ2_XXS
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:

python/llm/src/ipex_llm/transformers/models/mpt.py

@@ -27,7 +27,7 @@ from ipex_llm.transformers.models.utils import extend_kv_cache, init_kv_cache, a
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def mpt_multihead_attention_forward(self, x, past_key_value=None, attn_bias=None,

python/llm/src/ipex_llm/transformers/models/phixtral.py

@@ -54,7 +54,7 @@ from ipex_llm.transformers.models.utils import mlp_fusion_check
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:

python/llm/src/ipex_llm/transformers/models/qwen.py

@@ -56,7 +56,7 @@ logger = logging.get_logger(__name__)
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 SUPPORT_TORCH2 = hasattr(torch, '__version__') and int(torch.__version__.split(".")[0]) >= 2
 
 

python/llm/src/ipex_llm/transformers/models/qwen2.py

@@ -71,7 +71,7 @@ logger = logging.get_logger(__name__)
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def should_use_fuse_rope(self, query_states, position_ids):

python/llm/src/ipex_llm/transformers/models/qwen_vl.py

@@ -37,7 +37,7 @@ from ipex_llm.transformers.models.utils import decoding_fast_path_qtype_check
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def apply_rotary_pos_emb(t, freqs):

python/llm/src/ipex_llm/transformers/models/stablelm.py

@@ -62,7 +62,7 @@ except ImportError:
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def merge_qkv(module: torch.nn.Module):

python/llm/src/ipex_llm/transformers/models/yuan.py

@@ -40,7 +40,7 @@ from ipex_llm.transformers.models.utils import decoding_fast_path_qtype_check
 
 import os
 
-KV_CACHE_ALLOC_BLOCK_LENGTH = os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256)
+KV_CACHE_ALLOC_BLOCK_LENGTH = int(os.environ.get("KV_CACHE_ALLOC_BLOCK_LENGTH", 256))
 
 
 def use_decoding_fast_path(proj, use_fuse_rope, enough_kv_room, bs):