Vishwas1 commited on
Commit
27f2ab5
·
verified ·
1 Parent(s): 4cea20b

Update train_model.py

Browse files
Files changed (1) hide show
  1. train_model.py +1 -1
train_model.py CHANGED
@@ -65,7 +65,7 @@ def load_and_prepare_dataset(task, dataset_name, tokenizer, sequence_length):
65
  logging.info(f"Loading dataset '{dataset_name}' for task '{task}'...")
66
  try:
67
  if task == "generation":
68
- dataset = load_dataset(dataset_name, split='train')
69
  logging.info("Dataset loaded successfully for generation task.")
70
  def tokenize_function(examples):
71
  return tokenizer(examples['text'], truncation=True, max_length=sequence_length)
 
65
  logging.info(f"Loading dataset '{dataset_name}' for task '{task}'...")
66
  try:
67
  if task == "generation":
68
+ dataset = load_dataset(dataset_name, split='train',use_auth_token=True)
69
  logging.info("Dataset loaded successfully for generation task.")
70
  def tokenize_function(examples):
71
  return tokenizer(examples['text'], truncation=True, max_length=sequence_length)