Spaces:
Sleeping
Sleeping
Update train_model.py
Browse files- train_model.py +1 -1
train_model.py
CHANGED
@@ -65,7 +65,7 @@ def load_and_prepare_dataset(task, dataset_name, tokenizer, sequence_length):
|
|
65 |
logging.info(f"Loading dataset '{dataset_name}' for task '{task}'...")
|
66 |
try:
|
67 |
if task == "generation":
|
68 |
-
dataset = load_dataset(dataset_name, split='train')
|
69 |
logging.info("Dataset loaded successfully for generation task.")
|
70 |
def tokenize_function(examples):
|
71 |
return tokenizer(examples['text'], truncation=True, max_length=sequence_length)
|
|
|
65 |
logging.info(f"Loading dataset '{dataset_name}' for task '{task}'...")
|
66 |
try:
|
67 |
if task == "generation":
|
68 |
+
dataset = load_dataset(dataset_name, split='train',use_auth_token=True)
|
69 |
logging.info("Dataset loaded successfully for generation task.")
|
70 |
def tokenize_function(examples):
|
71 |
return tokenizer(examples['text'], truncation=True, max_length=sequence_length)
|