docstr: Process a single example from the dataset into IDs and other T0-related metadata.
docstr_tok: ["Process","a","single","example","from","the","dataset","into","IDs","and","other","T0-related","metadata","."]
code: def _process_example(self, example):
      """
      Process a single example from the dataset into IDs and other T0-related metadata.
      """
      tokenized_input = self.tokenizer.text_to_ids(example["input"])
      tokenized_output = self.tokenizer.text_to_ids(example["output"])
      offset = 0
      if self.add_bos_to_input:
          offset += 1
      if self.add_eos_to_input:
          offset += 1
  
      if len(tokenized_input) > self.max_src_seq_length - offset:
          tokenized_input = tokenized_input[
              : self.max_src_seq_length - offset
          ]
  
      if len(tokenized_output) > self.max_tgt_seq_length - 2:
          tokenized_output = tokenized_output[
              : self.max_tgt_seq_length - 2
          ]
  
      bos_id = (
          self.tokenizer.pad_id
          if self.replace_bos_with_pad
          else self.tokenizer.bos_id
      )
      if self.add_bos_to_input:
          tokenized_input = [bos_id] + tokenized_input
      if self.add_eos_to_input:
          tokenized_input = tokenized_input + [self.tokenizer.eos_id]
      target = [bos_id] + tokenized_output + [self.tokenizer.eos_id]
  
      processed_example = {
          "text_enc": tokenized_input,
          "text_dec": target[:-1],
          "labels": target[1:],
      }
  
      # Process optional args:
      if "chunked_idx" in example:
          original = ""
          template = ""
          for item in example["chunked_idx"].split(", "):
              item = item.split("-")
              if item[0] == "original_text":
                  original += example["input"][
                      int(item[1]) : int(item[2])
                  ]
              elif item[0] == "template":
                  template += example["input"][
                      int(item[1]) : int(item[2])
                  ]
              else:
                  raise ValueError(f"Unknown chunk type: {item[0]}")
  
          additional_args = {
              "original": self.tokenizer.text_to_ids(original),
              "template": self.tokenizer.text_to_ids(template),
              "prompt": self.tokenizer.text_to_ids(example["prompt"]),
          }
          processed_example.update(additional_args)
  
      if "choices" in example:
          additional_args = {
              "choices": [
                  self.tokenizer.text_to_ids(choice)
                  for choice in example["choices"]
              ],
          }
          processed_example.update(additional_args)
  
      if "task_name_with_prompt" in example:
          additional_args = {
              "task_name_with_prompt": self.tokenizer.text_to_ids(
                  example["task_name_with_prompt"]
              ),
          }
          processed_example.update(additional_args)
  
      return processed_example
  
file_path: NeMo/nemo/collections/nlp/data/language_modeling/t0_dataset.py
imports: import numpy
  import torch
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.collections.nlp.data.language_modeling.megatron.dataset_utils import get_samples_mapping
  from nemo.collections.nlp.data.language_modeling.text_memmap_dataset import JSONLMemMapDataset, TextMemMapDataset
  from nemo.core.classes import Dataset
label_desc: Use image node_id 5 for calling the T0Dataset obj's underlying member method code with example usage: obj._process_example(example) and returns: processed_example