class_docstr: Data Processor for MS Marco dialogues. (https://github.com/microsoft/MSMARCO-Question-Answering)
  Please agree to the Terms of Use before downloading data at 
  https://msmarco.blob.core.windows.net/msmarco/train_v2.1.json.gz
  https://msmarco.blob.core.windows.net/msmarco/dev_v2.1.json.gz
class_docstr_tok: ["Data","Processor","for","MS","Marco","dialogues",".","(","https",":","\/\/github.com\/microsoft\/MSMARCO-Question-Answering",")","Please","agree","to","the","Terms","of","Use","before","downloading","data","at","https",":","\/\/msmarco.blob.core.windows.net\/msmarco\/train_v2.1.json.gz","https",":","\/\/msmarco.blob.core.windows.net\/msmarco\/dev_v2.1.json.gz"]
docstr: Process raw files into DialogueInputExample
  Args: 
      dataset_split: {train, dev, test}
  For the MS Marco dataset, there is no explicit dev set (instead uses the test set as the dev set)
  Therefore, this function creates a dev set and a new train set from the train set.
  Dev set contains self.cfg.dev_proportion % of samples with the rest going into the train set
docstr_tok: ["Process","raw","files","into","DialogueInputExample","Args",":","dataset_split",":","{","train",",","dev",",","test","}","For","the","MS","Marco","dataset",",","there","is","no","explicit","dev","set","(","instead","uses","the","test","set","as","the","dev","set",")","Therefore",",","this","function","creates","a","dev","set","and","a","new","train","set","from","the","train","set",".","Dev","set","contains","self.cfg.dev_proportion","%","of","samples","with","the","rest","going","into","the","train","set"]
code: def get_dialog_examples(self, dataset_split: str):
      """
      Process raw files into DialogueInputExample
      Args:
          dataset_split: {train, dev, test}
      For the MS Marco dataset, there is no explicit dev set (instead uses the test set as the dev set)
      Therefore, this function creates a dev set and a new train set from the train set.
      Dev set contains self.cfg.dev_proportion % of samples with the rest going into the train set
      """
  
      examples = []
  
      dataset_split_print = {
          "train": "train",
          "dev": "train",
          "test": "dev",
      }
  
      raw_examples = self.open_json(
          "{}_v2.1.json".format(dataset_split_print[dataset_split])
      )
  
      n_samples = len(raw_examples["answers"])
  
      idxs = DialogueDataProcessor.get_relevant_idxs(
          dataset_split, n_samples, self.cfg.dev_proportion
      )
  
      if self.cfg.debug_mode:
          idxs = idxs[:100]
  
      for i in idxs:
          utterance = raw_examples["query"][str(i)]
          # answer need not be extracted from passage
          # taking the first answer as the ground truth correct answer as only <1% has multiple answers
          answer = raw_examples["answers"][str(i)]
          answer = answer[0] if isinstance(answer, list) else answer
  
          well_formed_answer = raw_examples["wellFormedAnswers"][str(i)]
          well_formed_answer = (
              well_formed_answer
              if isinstance(well_formed_answer, list)
              else literal_eval(well_formed_answer)
          )
          well_formed_answer = (
              well_formed_answer[0] if well_formed_answer else None
          )
          query_type = raw_examples["query_type"][str(i)]
          candidate_passages = raw_examples["passages"][str(i)]
          passage = [
              candidate_passage["passage_text"]
              for candidate_passage in candidate_passages
              if int(candidate_passage["is_selected"])
          ]
          passage = passage[0] if passage else None
  
          possible_passages = [
              candidate_passage["passage_text"]
              for candidate_passage in candidate_passages
          ]
  
          input_example = {
              "utterance": utterance,
              "example_id": i,
              "labels": {
                  "service": query_type,
                  "response": answer,
                  "fluent_response": well_formed_answer,
                  "passage": passage,
              },
              "possible_labels": {
                  "service": "LOCATION,NUMERIC,PERSON,DESCRIPTION,ENTITY".split(
                      ","
                  ),
                  "passage": possible_passages,
              },
          }
          example = DialogueInputExample(input_example)
          examples.append(example)
      return examples
  
file_path: NeMo/nemo/collections/nlp/data/dialogue/data_processor/ms_marco_data_processor.py
imports: import json
  import os
  from ast import literal_eval
  from nemo.collections.nlp.data.dialogue.data_processor.data_processor import DialogueDataProcessor
  from nemo.collections.nlp.data.dialogue.input_example.input_example import DialogueInputExample
label_desc: Use image node_id 3 for calling the DialogueMSMarcoDataProcessor obj's underlying member method code with example usage: obj.get_dialog_examples(dataset_split) and returns: examples