Update tasks/text.py
Browse files- tasks/text.py +1 -1
tasks/text.py
CHANGED
@@ -73,7 +73,7 @@ async def evaluate_text(request: TextEvaluationRequest):
|
|
73 |
|
74 |
def preprocess_function(df):
|
75 |
return tokenizer(df["quote"], truncation=True)
|
76 |
-
tokenized_test = test_dataset.map(preprocess_function, batched=True)
|
77 |
|
78 |
# training_args = torch.load("training_args.bin")
|
79 |
# training_args.eval_strategy='no'
|
|
|
73 |
|
74 |
def preprocess_function(df):
|
75 |
return tokenizer(df["quote"], truncation=True)
|
76 |
+
tokenized_test = test_dataset.map(preprocess_function, batched=True,padding='longest')
|
77 |
|
78 |
# training_args = torch.load("training_args.bin")
|
79 |
# training_args.eval_strategy='no'
|