class_docstr: The dataset class for prompt-tuning or p-tuning pretrained T5 models.
class_docstr_tok: ["The","dataset","class","for","prompt-tuning","or","p-tuning","pretrained","T5","models","."]
docstr: Loads a dataset by filling in the task templates specified in the config file
  with the information from each training/inference example. Converts all input 
  text into token ids. Also replaces the <|VIRTUAL_PROMPT_#|> placeholders in 
  the task templates with the actual virtual prompt token ids. 
  
  params:
      dataset: A list of json objects or a dictionary objects each
               containing the information needed for a training example
docstr_tok: ["Loads","a","dataset","by","filling","in","the","task","templates","specified","in","the","config","file","with","the","information","from","each","training\/inference","example",".","Converts","all","input","text","into","token","ids",".","Also","replaces","the","<","|VIRTUAL_PROMPT_","#","|",">","placeholders","in","the","task","templates","with","the","actual","virtual","prompt","token","ids",".","params",":","dataset",":","A","list","of","json","objects","or","a","dictionary","objects","each","containing","the","information","needed","for","a","training","example"]
code: def load_data(self, dataset):
      """
      Loads a dataset by filling in the task templates specified in the config file
      with the information from each training/inference example. Converts all input
      text into token ids. Also replaces the <|VIRTUAL_PROMPT_#|> placeholders in
      the task templates with the actual virtual prompt token ids.
  
      params:
          dataset: A list of json objects or a dictionary objects each
                   containing the information needed for a training example
      """
      skipped = 0
      for json_line in tqdm(dataset):
          # Read example dict or load the information for a single example from .json file
          if type(json_line) == dict:
              doc = json_line
          else:
              doc = json.loads(json_line)
  
          taskname = doc["taskname"]
          prompt_template = self.task_templates[taskname][
              "prompt_template"
          ]
          prompt_template_fields = self.task_templates[taskname][
              "prompt_template_fields"
          ]
          total_virtual_tokens = self.task_templates[taskname][
              "total_virtual_tokens"
          ]
          virtual_token_splits = self.task_templates[taskname][
              "virtual_token_splits"
          ]
          truncation_field = self.task_templates[taskname][
              "truncate_field"
          ]
          answer_field = self.task_templates[taskname]["answer_field"]
  
          input_example = prompt_template
  
          self._input_sanity_checks(
              total_virtual_tokens=total_virtual_tokens,
              virtual_token_splits=virtual_token_splits,
              prompt_template=prompt_template,
              prompt_template_fields=prompt_template_fields,
              truncation_field=truncation_field,
              answer_field=answer_field,
              doc=doc,
          )
  
          # Format the input example according to the template
          input_example = self._insert_text_in_template(
              input_example, prompt_template_fields, doc, answer_field
          )
          input_example = self._insert_virtual_token_placeholders(
              input_example, virtual_token_splits
          )
  
          # a trick to align with the data format in t5 pretraining
          input_ids = self.tokenizer.text_to_ids(input_example)
          if self.add_sentinel_to_input:
              input_ids = input_ids + self.tokenizer.text_to_ids(
                  T5Sentinel.FIRST.value
              )
  
          # Add BOS/EOS to the input of encoder if desired, adds EOS by default
          if self.ul2_prompt_token is not None:
              ul2_prompt_token_id = self.tokenizer.text_to_ids(
                  self.ul2_prompt_token
              )
              assert len(ul2_prompt_token_id) == 1
              input_ids = ul2_prompt_token_id + input_ids
          if self.add_bos:
              input_ids = [self.tokenizer.bos_id] + input_ids
          if self.add_eos:
              input_ids = input_ids + [self.tokenizer.eos_id]
  
          # Try to truncate input text to fit into the max sequence length
          if len(input_ids) > self.max_seq_length:
              input_ids = self._truncate_input(
                  truncation_field,
                  input_ids,
                  taskname,
                  doc,
                  total_virtual_tokens,
              )
  
          # get answer ids
          if answer_field in doc.keys():  # training and validation
              answer_text = doc[answer_field]
  
              if self.decoder_starts_with_pad:
                  answer_text_ids = [self.tokenizer.pad_id]
              else:
                  answer_text_ids = [self.tokenizer.bos_id]
              # a trick to align with the data format in t5 pretraining
              if self.add_sentinel_to_input:
                  answer_text_ids += self.tokenizer.text_to_ids(
                      T5Sentinel.FIRST.value
                  )
              answer_text_ids += self.tokenizer.text_to_ids(answer_text)
              if self.add_eos_to_decoder_output:
                  answer_text_ids += [self.tokenizer.eos_id]
              else:
                  answer_text_ids += self.tokenizer.text_to_ids(
                      T5Sentinel.END.value
                  )
  
          # Skip example if the final length doesn't fit length requirements even after truncation
          if (
              self.min_seq_length
              <= len(input_ids)
              <= self.max_seq_length
          ):
              if (
                  self.virtual_prompt_source
                  == VirtualPromptSource.PROMPT_ENCODER
              ):
                  taskname_id = self.tokenizer.text_to_ids(taskname)
              elif (
                  self.virtual_prompt_source
                  == VirtualPromptSource.NO_PROMPT
              ):  # TODO (@adithyare) this class and GPTPromptLearningDataset should be merged.
                  taskname_id = -1
              else:
                  raise ValueError(
                      "Invalid virtual prompt source specified"
                  )
  
              dec_input = None
              dec_labels = None
  
              if answer_field in doc.keys():  # training and validation
                  dec_input = answer_text_ids[:-1]
                  dec_labels = answer_text_ids[1:]
  
              self.examples.append(
                  (taskname_id, input_ids, dec_input, dec_labels)
              )
          else:
              skipped += 1
  
      logging.info(
          f"Skipped {skipped} sentences, sequence length too short or too long even after truncation"
      )
  
file_path: NeMo/nemo/collections/nlp/data/language_modeling/megatron/t5_prompt_learning_dataset.py
imports: import json
  import torch
  from tqdm.auto import tqdm
  from nemo.collections.nlp.data.language_modeling.megatron.base_prompt_learning_dataset import BasePromptLearningDataset
  from nemo.collections.nlp.models.language_modeling.megatron_t5_model import T5Sentinel
  from nemo.collections.nlp.modules.common import VirtualPromptSource
  from nemo.collections.nlp.modules.common.megatron.utils import build_position_ids
  from nemo.utils import logging
label_desc: Use image node_id 2 for calling the T5PromptLearningDataset obj's underlying member method code with example usage: obj.load_data(dataset) without return types