Add padding_token=eos_token for GPU trl QLora example (#12398)

* Avoid tokenizer doesn't have a padding token error.
This commit is contained in:
Qiyuan Gong 2024-11-14 10:51:30 +08:00 committed by GitHub
parent d2cbcb060c
commit 7e50ff113c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -45,6 +45,9 @@ if __name__ == "__main__":
model_path = args.repo_id_or_model_path
dataset_path = args.dataset
tokenizer = LlamaTokenizer.from_pretrained(model_path, trust_remote_code=True)
# Avoid tokenizer doesn't have a padding token
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
if dataset_path.endswith(".json") or dataset_path.endswith(".jsonl"):
data = load_dataset("json", data_files=dataset_path)