Vishwas1 commited on
Commit
ec09650
·
verified ·
1 Parent(s): 3a4bc44

Update train_model.py

Browse files
Files changed (1) hide show
  1. train_model.py +1 -1
train_model.py CHANGED
@@ -74,7 +74,7 @@ def load_and_prepare_dataset(task, dataset_name, tokenizer, sequence_length):
74
  logging.info("Dataset loaded successfully.")
75
 
76
  def tokenize_function(examples):
77
- return tokenizer(examples['text'], truncation=True, max_length=sequence_length)
78
 
79
  # Tokenize the dataset
80
  tokenized_datasets = dataset.shuffle(seed=42).select(range(500)).map(tokenize_function, batched=True)
 
74
  logging.info("Dataset loaded successfully.")
75
 
76
  def tokenize_function(examples):
77
+ return tokenizer(examples['text'], truncation=True, padding='max_length', max_length=sequence_length)
78
 
79
  # Tokenize the dataset
80
  tokenized_datasets = dataset.shuffle(seed=42).select(range(500)).map(tokenize_function, batched=True)