docstr: Dataset for training information retrieval models.
  
  Args:
      tokenizer: tokenizer
      passages: path to tsv with [psg_id, psg_text] entries
      queries: path to tsv with [query_id, query_text] entries
      query_to_passages: path to tsv with
          [query_id, pos_psg_id, neg_psg_id_1, ..., neg_psg_id_k] entries
      max_query_length: maximum length of query in tokens
      max_passage_length: maximum length of passage in tokens
      num_negatives: number of negative passages per positive to use for training
      preprocess_fn: either preprocess_bert or preprocess_dpr
          preprocess_bert: joint input: [CLS] query [SEP] passage [SEP]
          preprocess_dpr: separate inputs: [CLS] query [SEP], [CLS] passage [SEP]
      psg_cache_format: either pkl or npz
docstr_tok: ["Dataset","for","training","information","retrieval","models",".","Args",":","tokenizer",":","tokenizer","passages",":","path","to","tsv","with","[","psg_id",",","psg_text","]","entries","queries",":","path","to","tsv","with","[","query_id",",","query_text","]","entries","query_to_passages",":","path","to","tsv","with","[","query_id",",","pos_psg_id",",","neg_psg_id_1",",","...",",","neg_psg_id_k","]","entries","max_query_length",":","maximum","length","of","query","in","tokens","max_passage_length",":","maximum","length","of","passage","in","tokens","num_negatives",":","number","of","negative","passages","per","positive","to","use","for","training","preprocess_fn",":","either","preprocess_bert","or","preprocess_dpr","preprocess_bert",":","joint","input",":","[","CLS","]","query","[","SEP","]","passage","[","SEP","]","preprocess_dpr",":","separate","inputs",":","[","CLS","]","query","[","SEP","]",",","[","CLS","]","passage","[","SEP","]","psg_cache_format",":","either","pkl","or","npz"]
code: def __init__(
      self,
      tokenizer: TokenizerSpec,
      passages: str,
      queries: str,
      query_to_passages: str,
      max_query_length: Optional[int] = 31,
      max_passage_length: Optional[int] = 190,
      num_negatives: Optional[int] = 10,
      preprocess_fn: Optional[str] = "preprocess_bert",
      psg_cache_format: Optional[str] = "npz",
  ):
      """
      Dataset for training information retrieval models.
  
      Args:
          tokenizer: tokenizer
          passages: path to tsv with [psg_id, psg_text] entries
          queries: path to tsv with [query_id, query_text] entries
          query_to_passages: path to tsv with
              [query_id, pos_psg_id, neg_psg_id_1, ..., neg_psg_id_k] entries
          max_query_length: maximum length of query in tokens
          max_passage_length: maximum length of passage in tokens
          num_negatives: number of negative passages per positive to use for training
          preprocess_fn: either preprocess_bert or preprocess_dpr
              preprocess_bert: joint input: [CLS] query [SEP] passage [SEP]
              preprocess_dpr: separate inputs: [CLS] query [SEP], [CLS] passage [SEP]
          psg_cache_format: either pkl or npz
      """
  
      super().__init__(tokenizer, max_query_length, max_passage_length)
      self.num_negatives = num_negatives
  
      self.passages = getattr(self, f"parse_{psg_cache_format}")(
          passages, max_passage_length
      )
      self._psgid2tokens = getattr(
          self, f"psgid2tokens_{psg_cache_format}"
      )
      self.queries = self.parse_pkl(queries, max_query_length)
      self.idx2psgs = self.parse_query_to_passages(query_to_passages)
      self._preprocess_fn = getattr(self, preprocess_fn)
  
imports: import multiprocessing
  import os
  import pickle
  import random
  from typing import Optional
  import numpy
  from torch.utils.data import Dataset
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
label_desc: Use image node_id 1 to create a new BertInformationRetrievalDataset object from inherited base classes: BaseInformationRetrievalDataset with example: obj = BertInformationRetrievalDataset(tokenizer, passages, queries, query_to_passages, max_query_length, max_passage_length, num_negatives, preprocess_fn, psg_cache_format)