Update local import for ppl (#11866)

Co-authored-by: jenniew <jenniewang123@gmail.com>
This commit is contained in:
RyuKosei 2024-08-20 18:50:00 +08:00 committed by GitHub
parent c3c058373f
commit 5df00869de
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -21,7 +21,6 @@ import argparse
import torch
from tqdm import tqdm
from datasets import load_dataset
from ipex_llm.utils.common import invalidInputError
parser = argparse.ArgumentParser()
@ -63,6 +62,7 @@ elif args.data_path:
data = f.read()
encodings = tokenizer(data.decode("utf-8").strip("\n"), return_tensors="pt")
else:
from ipex_llm.utils.common import invalidInputError
raise invalidInputError(False, "Must specify either dataset or datapath.")
if not args.max_length: