class_docstr: The base dataset class for prompt-tuning or p-tuning.
  TODO: (@adithyare) should be merged into GPTPromptLearningDataset
class_docstr_tok: ["The","base","dataset","class","for","prompt-tuning","or","p-tuning",".","TODO",":","(","@","adithyare",")","should","be","merged","into","GPTPromptLearningDataset"]
code: def __init__(
      self,
      datasets,
      tokenizer,
      virtual_prompt_source: VirtualPromptSource,
      task_templates: dict,
      pseudo_tokens,
      pad_token_id: str,
      max_seq_length: int,
      min_seq_length: int = 1,
      add_bos: bool = False,
      add_eos: bool = True,
      for_train: bool = True,
  ):
      self.tokenizer = tokenizer
      self.virtual_prompt_source = virtual_prompt_source
      self.task_templates = task_templates
      self.pseudo_tokens = pseudo_tokens
      self.pseudo_token_ids = set(
          self.tokenizer.tokens_to_ids(self.pseudo_tokens)
      )
      self.pad_token_id = pad_token_id
      self.max_seq_length = max_seq_length
      self.min_seq_length = min_seq_length
      self.add_bos = add_bos
      self.add_eos = add_eos
      self.for_train = for_train
      self.examples = []
  
      assert (
          self.min_seq_length <= max_seq_length
      ), "Min sequence length should be less than or equal to max"
      assert (
          self.max_seq_length > 0
      ), "Max sequence length should be greater than 0"
  
      logging.info("Loading and tokenizing dataset ... ")
  
      # Datasets is just a list of json dicts
      if isinstance(datasets[0], dict):
          self.load_data(datasets)
  
      # Datasets are a list of file path strings to .json or .jsonl files
      elif isinstance(datasets[0], str):
          for path in datasets:
              dataset = open(path, "r", encoding="utf-8")
              self.load_data(dataset)
      else:
          raise ValueError(
              "Datasets must be a list of dicts or a list of filepath strings"
          )
  
file_path: NeMo/nemo/collections/nlp/data/language_modeling/megatron/base_prompt_learning_dataset.py
imports: import torch
  from nemo.collections.nlp.modules.common import VirtualPromptSource
  from nemo.core import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new BasePromptLearningDataset object from inherited base classes: Dataset with example: obj = BasePromptLearningDataset(datasets, tokenizer, virtual_prompt_source, task_templates, pseudo_tokens, pad_token_id, max_seq_length, min_seq_length, add_bos, add_eos, for_train)