Spaces:
Sleeping
Sleeping
Update train_model.py
Browse files- train_model.py +1 -1
train_model.py
CHANGED
@@ -74,7 +74,7 @@ def load_and_prepare_dataset(task, dataset_name, tokenizer, sequence_length):
|
|
74 |
logging.info("Dataset loaded successfully.")
|
75 |
|
76 |
def tokenize_function(examples):
|
77 |
-
return tokenizer(examples['text'], truncation=True, max_length=sequence_length)
|
78 |
|
79 |
# Tokenize the dataset
|
80 |
tokenized_datasets = dataset.shuffle(seed=42).select(range(500)).map(tokenize_function, batched=True)
|
|
|
74 |
logging.info("Dataset loaded successfully.")
|
75 |
|
76 |
def tokenize_function(examples):
|
77 |
+
return tokenizer(examples['text'], truncation=True, padding='max_length', max_length=sequence_length)
|
78 |
|
79 |
# Tokenize the dataset
|
80 |
tokenized_datasets = dataset.shuffle(seed=42).select(range(500)).map(tokenize_function, batched=True)
|