From 1a885020ee4e22451196d2b90602ac5fe6ce49d2 Mon Sep 17 00:00:00 2001
From: Ovo233 <76120304+Mingyu-Wei@users.noreply.github.com>
Date: Fri, 19 Apr 2024 15:34:39 +0800
Subject: [PATCH] Updated importing of top_k_top_p_filtering for
 transformers>=4.39.0 (#10794)

* In transformers>=4.39.0, the top_k_top_p_filtering function has been deprecated and moved to the hugging face package trl. Thus, for versions >= 4.39.0, import this function from trl.
---
 python/llm/src/ipex_llm/transformers/speculative.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/python/llm/src/ipex_llm/transformers/speculative.py b/python/llm/src/ipex_llm/transformers/speculative.py
index 20540c56..9d8880cc 100644
--- a/python/llm/src/ipex_llm/transformers/speculative.py
+++ b/python/llm/src/ipex_llm/transformers/speculative.py
@@ -27,8 +27,19 @@ import logging
 import transformers
 from packaging import version
 from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
-from transformers import top_k_top_p_filtering, GenerationConfig, \
+from transformers import GenerationConfig, \
     LogitsProcessorList, StoppingCriteriaList
+from ipex_llm.utils.common import log4Error
+
+trans_version = transformers.__version__
+if version.parse(trans_version) >= version.parse("4.39.0"):
+    try:
+        from trl.core import top_k_top_p_filtering
+    except ModuleNotFoundError:
+        log4Error.invalidInputError(False, "For transformers version >= 4.39.0, pip install trl")
+else:
+    from transformers import top_k_top_p_filtering
+
 from ipex_llm.utils.common import invalidInputError
 from transformers.modeling_outputs import CausalLMOutputWithPast