MatthiasPicard commited on
Commit
8a614e0
·
verified ·
1 Parent(s): be34aad

Update tasks/text.py

Browse files
Files changed (1) hide show
  1. tasks/text.py +1 -1
tasks/text.py CHANGED
@@ -73,7 +73,7 @@ async def evaluate_text(request: TextEvaluationRequest):
73
 
74
  def preprocess_function(df):
75
  return tokenizer(df["quote"], truncation=True)
76
- tokenized_test = test_dataset.map(preprocess_function, batched=True)
77
 
78
  # training_args = torch.load("training_args.bin")
79
  # training_args.eval_strategy='no'
 
73
 
74
  def preprocess_function(df):
75
  return tokenizer(df["quote"], truncation=True)
76
+ tokenized_test = test_dataset.map(preprocess_function, batched=True,padding='longest')
77
 
78
  # training_args = torch.load("training_args.bin")
79
  # training_args.eval_strategy='no'