Add lookup generate in load_low_bit (#12243)

* add lookup generate in load_low_bit

* update comment
This commit is contained in:
Yina Chen 2024-10-22 10:51:52 +03:00 committed by GitHub
parent d8c1287335
commit ec465fbcd7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -797,6 +797,13 @@ class _BaseAutoModelClass:
model) model)
torch.distributed.barrier() torch.distributed.barrier()
# add lookup_generate to loaded model
from .lookup import lookup_generate
import types
model.lookup_generate = types.MethodType(lookup_generate, model)
if model.config.model_type == "minicpmv" and hasattr(model, 'llm'):
model.llm.lookup_generate = types.MethodType(lookup_generate, model.llm)
return model return model