class_docstr: Data Processor for Assistant dialogues.
class_docstr_tok: ["Data","Processor","for","Assistant","dialogues","."]
docstr: Process raw files into DialogueInputExample
  Args: 
      dataset_split: {train, dev, test}
  For the assistant dataset, there is no explicit dev set (instead uses the test set as the dev set)
  Therefore, this function creates a dev set and a new train set from the train set.
  This is done by taking every 10th example and putting it into the dev set,
  with all other examples going into the new train set.
docstr_tok: ["Process","raw","files","into","DialogueInputExample","Args",":","dataset_split",":","{","train",",","dev",",","test","}","For","the","assistant","dataset",",","there","is","no","explicit","dev","set","(","instead","uses","the","test","set","as","the","dev","set",")","Therefore",",","this","function","creates","a","dev","set","and","a","new","train","set","from","the","train","set",".","This","is","done","by","taking","every","10th","example","and","putting","it","into","the","dev","set",",","with","all","other","examples","going","into","the","new","train","set","."]
code: def get_dialog_examples(self, dataset_split: str):
      """
      Process raw files into DialogueInputExample
      Args:
          dataset_split: {train, dev, test}
      For the assistant dataset, there is no explicit dev set (instead uses the test set as the dev set)
      Therefore, this function creates a dev set and a new train set from the train set.
      This is done by taking every 10th example and putting it into the dev set,
      with all other examples going into the new train set.
      """
      examples = []
  
      dataset_split_print = {
          "train": "train",
          "dev": "train",
          "test": "test",
      }
  
      raw_examples_intent = self.open_file(
          "{}.tsv".format(dataset_split_print[dataset_split])
      )
      # removes header of tsv file
      raw_examples_intent = raw_examples_intent[1:]
      raw_examples_slots = self.open_file(
          "{}_slots.tsv".format(dataset_split_print[dataset_split])
      )
  
      if dataset_split in ["train", "dev"]:
          train_idx = []
          dev_idx = []
          for idx in range(len(raw_examples_intent)):
              if idx % 10 == 0:
                  dev_idx.append(idx)
              else:
                  train_idx.append(idx)
  
      if dataset_split == "train":
          raw_examples_intent = [
              raw_examples_intent[idx] for idx in train_idx
          ]
          raw_examples_slots = [
              raw_examples_slots[idx] for idx in train_idx
          ]
      elif dataset_split == "dev":
          raw_examples_intent = [
              raw_examples_intent[idx] for idx in dev_idx
          ]
          raw_examples_slots = [
              raw_examples_slots[idx] for idx in dev_idx
          ]
  
      for i in range(len(raw_examples_intent)):
          utterance, intent_id = raw_examples_intent[i].split("\t")
          slot_ids = raw_examples_slots[i].split()
          utterance_tokens = utterance.split()
          intent = self.intents[int(intent_id)]
          slot_id_to_start_and_exclusive_end = (
              DialogueAssistantDataProcessor.get_continuous_slots(
                  slot_ids,
                  self.empty_slot_id,
                  self.bio_slot_ids_to_unified_slot_ids,
              )
          )
  
          slot_to_start_and_exclusive_end = {
              self.slots[int(slot_id)]: position
              for slot_id, position in slot_id_to_start_and_exclusive_end.items()
          }
          slot_to_words = {
              slot: " ".join(
                  utterance_tokens[position[0] : position[1]]
              )
              for slot, position in slot_to_start_and_exclusive_end.items()
          }
          input_example = {
              "utterance": utterance,
              "labels": {
                  "service": intent.split("_")[0],
                  "intent": intent,
                  "slots": slot_to_words,
              },
              "label_positions": {
                  "slots": {
                      slot: {
                          "start": position[0],
                          "exclusive_end": position[1],
                          "slot": slot,
                      }
                      for slot, position in slot_to_start_and_exclusive_end.items()
                  }
              },
              "possible_labels": {
                  "service": self.services,
                  "intent": self.intents,
                  "slots": {
                      # this dataset does not support categorical slots (i.e. only extractive slots)
                      # therefore use empty list for all values
                      slot: []
                      for slot in self.slots
                  },
              },
          }
          example = DialogueInputExample(input_example)
          examples.append(example)
      return examples
  
file_path: NeMo/nemo/collections/nlp/data/dialogue/data_processor/assistant_data_processor.py
imports: import os
  from nemo.collections.nlp.data.dialogue.data_processor.data_processor import DialogueDataProcessor
  from nemo.collections.nlp.data.dialogue.input_example.input_example import DialogueInputExample
label_desc: Use image node_id 6 for calling the DialogueAssistantDataProcessor obj's underlying member method code with example usage: obj.get_dialog_examples(dataset_split) and returns: examples