class_docstr: Data Processor for MS Marco dialogues. (https://github.com/microsoft/MSMARCO-Question-Answering)
  Please agree to the Terms of Use before downloading data at 
  https://msmarco.blob.core.windows.net/msmarco/train_v2.1.json.gz
  https://msmarco.blob.core.windows.net/msmarco/dev_v2.1.json.gz
class_docstr_tok: ["Data","Processor","for","MS","Marco","dialogues",".","(","https",":","\/\/github.com\/microsoft\/MSMARCO-Question-Answering",")","Please","agree","to","the","Terms","of","Use","before","downloading","data","at","https",":","\/\/msmarco.blob.core.windows.net\/msmarco\/train_v2.1.json.gz","https",":","\/\/msmarco.blob.core.windows.net\/msmarco\/dev_v2.1.json.gz"]
docstr: Constructs DialogueMSMarcoDataProcessor
  Args:
      data_dir: path to data directory
      tokenizer: tokenizer object
      debug_mode: reduce number of samples to load in order to increase speed of processing
      cfg: cfg container for dataset
docstr_tok: ["Constructs","DialogueMSMarcoDataProcessor","Args",":","data_dir",":","path","to","data","directory","tokenizer",":","tokenizer","object","debug_mode",":","reduce","number","of","samples","to","load","in","order","to","increase","speed","of","processing","cfg",":","cfg","container","for","dataset"]
code: def __init__(self, data_dir: str, tokenizer: object, cfg=None):
      """
      Constructs DialogueMSMarcoDataProcessor
      Args:
          data_dir: path to data directory
          tokenizer: tokenizer object
          debug_mode: reduce number of samples to load in order to increase speed of processing
          cfg: cfg container for dataset
      """
      self.data_dir = data_dir
      self._tokenizer = tokenizer
      self.cfg = cfg
  
file_path: NeMo/nemo/collections/nlp/data/dialogue/data_processor/ms_marco_data_processor.py
imports: import json
  import os
  from ast import literal_eval
  from nemo.collections.nlp.data.dialogue.data_processor.data_processor import DialogueDataProcessor
  from nemo.collections.nlp.data.dialogue.input_example.input_example import DialogueInputExample
label_desc: Use image node_id 1 to create a new DialogueMSMarcoDataProcessor object from inherited base classes: DialogueDataProcessor with example: obj = DialogueMSMarcoDataProcessor(data_dir, tokenizer, cfg)