class_docstr: Dataset Class 
      1. Performs Model-dependent (but Data-independent) operations (tokenization etc)
      2. This can allow the same model preprocessing for multiple datasources
      3. Users can configurate which labels to use for modelling 
          (e.g. intent classification, slot filling or both together etc)
class_docstr_tok: ["Dataset","Class","1",".","Performs","Model-dependent","(","but","Data-independent",")","operations","(","tokenization","etc",")","2",".","This","can","allow","the","same","model","preprocessing","for","multiple","datasources","3",".","Users","can","configurate","which","labels","to","use","for","modelling","(","e.g",".","intent","classification",",","slot","filling","or","both","together","etc",")"]
docstr: Creates an example for each frame in the user turn.
  Args:
      turn_id: turn number
      system_utterance: last system utterance
      user_utterance: lst user utterance
      system_frames: all system utterances and slot - slot value pairs
      user_frames: all user utterances and slot - slot value pairs
      prev_states: slot - slot value pairs from the previous turns
      schemas: schema for all services of all datasets
      subsample: whether to balance postive and negative samples in the dataset
  Returns:
      examples: a list of `InputExample`s.
      prev_states: updated dialogue state e.g. {'Restaurants_1': {'city': ['San Jose'], 'cuisine': ['American']}}
docstr_tok: ["Creates","an","example","for","each","frame","in","the","user","turn",".","Args",":","turn_id",":","turn","number","system_utterance",":","last","system","utterance","user_utterance",":","lst","user","utterance","system_frames",":","all","system","utterances","and","slot","-","slot","value","pairs","user_frames",":","all","user","utterances","and","slot","-","slot","value","pairs","prev_states",":","slot","-","slot","value","pairs","from","the","previous","turns","schemas",":","schema","for","all","services","of","all","datasets","subsample",":","whether","to","balance","postive","and","negative","samples","in","the","dataset","Returns",":","examples",":","a","list","of","`","InputExample","`","s",".","prev_states",":","updated","dialogue","state","e.g",".","{","'Restaurants_1","'",":","{","'city","'",":","[","'San","Jose","'","]",",","'cuisine","'",":","[","'American","'","]","}","}"]
code: def bert_process_one_sample(self, idx):
      """
      Creates an example for each frame in the user turn.
      Args:
          turn_id: turn number
          system_utterance: last system utterance
          user_utterance: lst user utterance
          system_frames: all system utterances and slot - slot value pairs
          user_frames: all user utterances and slot - slot value pairs
          prev_states: slot - slot value pairs from the previous turns
          schemas: schema for all services of all datasets
          subsample: whether to balance postive and negative samples in the dataset
      Returns:
          examples: a list of `InputExample`s.
          prev_states: updated dialogue state e.g. {'Restaurants_1': {'city': ['San Jose'], 'cuisine': ['American']}}
      """
  
      ex = self.raw_features[idx].data
      example_id_num = ex["example_id_num"]
      example_id = ex["example_id"]
      user_utterance = ex["utterance"]
      system_utterance = ex["system_utterance"]
      service = ex["labels"]["service"]
      schemas = self.schemas
      state_update = ex["labels"]["slots"]
      system_slots = ex["system_slots"]
  
      (
          user_tokens,
          user_alignments,
          user_inv_alignments,
      ) = self._tokenize(user_utterance)
      (
          system_tokens,
          system_alignments,
          system_inv_alignments,
      ) = self._tokenize(system_utterance)
      system_user_utterance = system_utterance + " " + user_utterance
      (
          system_user_tokens,
          system_user_alignments,
          system_user_inv_alignments,
      ) = self._tokenize(system_user_utterance)
      examples = []
  
      base_example = SGDInputExample(
          schema_config=self.schema_config, tokenizer=self.tokenizer
      )
      base_example.service_schema = self.schemas.get_service_schema(
          service
      )
      base_example.service_id = example_id_num[-1]
  
      base_example.example_id = example_id
      base_example.example_id_num = example_id_num
  
      for model_task in range(self.schema_config["NUM_TASKS"]):
          if model_task == 0:
              for intent_id, intent in enumerate(
                  schemas.get_service_schema(service).intents
              ):
                  task_example = base_example.make_copy()
                  task_example.task_mask[model_task] = 1
                  task_example.intent_id = intent_id
                  task_example.example_id += (
                      f"-{model_task}-{intent_id}-0"
                  )
                  task_example.example_id_num.extend(
                      [model_task, intent_id, 0]
                  )
                  intent_description = (
                      intent
                      + " "
                      + self.schemas.get_service_schema(
                          service
                      ).intent_descriptions[intent]
                  )
                  (
                      intent_tokens,
                      intent_alignments,
                      intent_inv_alignments,
                  ) = self._tokenize(intent_description)
                  task_example.add_utterance_features(
                      intent_tokens,
                      intent_inv_alignments,
                      system_user_tokens,
                      system_user_inv_alignments,
                      intent_description,
                      system_user_utterance,
                  )
  
                  task_example.add_intents(ex)
                  examples.append(task_example)
  
          if model_task == 1:
              for slot_id, slot in enumerate(
                  schemas.get_service_schema(service).slots
              ):
                  task_example = base_example.make_copy()
                  task_example.task_mask[model_task] = 1
                  task_example.requested_slot_id = slot_id
                  task_example.example_id += (
                      f"-{model_task}-{slot_id}-0"
                  )
                  task_example.example_id_num.extend(
                      [model_task, slot_id, 0]
                  )
                  slot_description = (
                      slot
                      + " "
                      + self.schemas.get_service_schema(
                          service
                      ).slot_descriptions[slot]
                  )
                  (
                      slot_tokens,
                      slot_alignments,
                      slot_inv_alignments,
                  ) = self._tokenize(slot_description)
                  task_example.add_utterance_features(
                      slot_tokens,
                      slot_inv_alignments,
                      user_tokens,
                      user_inv_alignments,
                      slot_description,
                      user_utterance,
                  )
  
                  task_example.add_requested_slots(ex)
                  examples.append(task_example)
  
          if model_task == 2:
              off_slots = []
              on_slots = []
              for slot_id, slot in enumerate(
                  schemas.get_service_schema(service).categorical_slots
              ):
                  task_example = base_example.make_copy()
                  task_example.task_mask[model_task] = 1
  
                  # assert task_example.task_mask == [0, 0, 1, 0, 0, 0]
                  task_example.categorical_slot_id = slot_id
                  task_example.example_id += (
                      f"-{model_task}-{slot_id}-0"
                  )
                  task_example.example_id_num.extend(
                      [model_task, slot_id, 0]
                  )
                  slot_description = (
                      slot
                      + " "
                      + schemas.get_service_schema(
                          service
                      ).slot_descriptions[slot]
                  )
                  (
                      slot_tokens,
                      slot_alignments,
                      slot_inv_alignments,
                  ) = self._tokenize(slot_description)
                  task_example.add_utterance_features(
                      slot_tokens,
                      slot_inv_alignments,
                      system_user_tokens,
                      system_user_inv_alignments,
                      slot_description,
                      system_user_utterance,
                  )
                  task_example.add_categorical_slots(state_update)
  
                  if task_example.categorical_slot_status == 0:
                      off_slots.append(task_example)
                  else:
                      on_slots.append(task_example)
                      examples.append(task_example)
                  old_example = task_example
  
                  for value_id, value in enumerate(
                      schemas.get_service_schema(
                          service
                      ).get_categorical_slot_values(slot)
                  ):
                      if (
                          self.dataset_split != "train"
                          or task_example.categorical_slot_status == 1
                      ):
                          task_example = (
                              old_example.make_copy_of_categorical_features()
                          )
                          task_example.task_mask[3] = 1
                          # assert task_example.task_mask == [0, 0, 0, 1, 0, 0]
                          task_example.categorical_slot_id = slot_id
                          task_example.categorical_slot_value_id = (
                              value_id
                          )
                          task_example.example_id = (
                              base_example.example_id
                              + f"-3-{slot_id}-{value_id}"
                          )
                          task_example.example_id_num = (
                              base_example.example_id_num
                              + [3, slot_id, value_id]
                          )
                          slot_description = (
                              slot + " " + value
                          )  # add slot description
                          (
                              slot_tokens,
                              slot_alignments,
                              slot_inv_alignments,
                          ) = self._tokenize(slot_description)
                          task_example.add_utterance_features(
                              slot_tokens,
                              slot_inv_alignments,
                              system_user_tokens,
                              system_user_inv_alignments,
                              slot_description,
                              system_user_utterance,
                          )
                          task_example.add_categorical_slots(
                              state_update
                          )
                          assert (
                              task_example.categorical_slot_status
                              == old_example.categorical_slot_status
                          )
                          examples.append(task_example)
  
              if self.dataset_split == "train" and self.subsample:
                  num_on_slots = len(on_slots)
                  examples.extend(
                      np.random.choice(
                          off_slots,
                          replace=False,
                          size=min(
                              max(num_on_slots, 1), len(off_slots)
                          ),
                      )
                  )
              else:
                  examples.extend(off_slots)
  
          if model_task == 4:  # noncat slot status
              off_slots = []
              on_slots = []
              for slot_id, slot in enumerate(
                  schemas.get_service_schema(
                      service
                  ).non_categorical_slots
              ):
                  task_example = base_example.make_copy()
                  task_example.task_mask[model_task] = 1
                  # assert task_example.task_mask == [0, 0, 0, 0, 1, 0]
                  task_example.noncategorical_slot_id = slot_id
                  task_example.example_id += (
                      f"-{model_task}-{slot_id}-0"
                  )
                  task_example.example_id_num.extend(
                      [model_task, slot_id, 0]
                  )
                  slot_description = (
                      slot
                      + " "
                      + schemas.get_service_schema(
                          service
                      ).slot_descriptions[slot]
                  )
                  (
                      slot_tokens,
                      slot_alignments,
                      slot_inv_alignments,
                  ) = self._tokenize(slot_description)
                  task_example.add_utterance_features(
                      slot_tokens,
                      slot_inv_alignments,
                      system_user_tokens,
                      system_user_inv_alignments,
                      slot_description,
                      system_user_utterance,
                  )
  
                  user_span_boundaries = self._find_subword_indices(
                      state_update,
                      user_utterance,
                      ex["label_positions"]["slots"],
                      user_alignments,
                      user_tokens,
                      2 + len(slot_tokens) + len(system_tokens),
                  )
  
                  if system_slots is not None:
                      system_span_boundaries = (
                          self._find_subword_indices(
                              state_update,
                              system_utterance,
                              system_slots,
                              system_alignments,
                              system_tokens,
                              2 + len(slot_tokens),
                          )
                      )
                  else:
                      system_span_boundaries = {}
  
                  task_example.add_noncategorical_slots(
                      state_update,
                      user_span_boundaries,
                      system_span_boundaries,
                  )
                  if task_example.noncategorical_slot_status == 0:
                      off_slots.append(task_example)
                  else:
                      on_slots.append(task_example)
                      examples.append(task_example)
  
                  if (
                      self.dataset_split != "train"
                      or task_example.noncategorical_slot_status == 1
                  ):
                      task_example = (
                          task_example.make_copy_of_non_categorical_features()
                      )
                      task_example.task_mask[5] = 1
                      # assert task_example.task_mask == [0, 0, 0, 0, 0, 1]
                      task_example.example_id = (
                          base_example.example_id + f"-5-{slot_id}-0"
                      )
                      task_example.example_id_num = (
                          base_example.example_id_num + [5, slot_id, 0]
                      )
                      examples.append(task_example)
  
              if self.dataset_split == "train" and self.subsample:
                  num_on_slots = len(on_slots)
                  examples.extend(
                      np.random.choice(
                          off_slots,
                          replace=False,
                          size=min(
                              max(num_on_slots, 1), len(off_slots)
                          ),
                      )
                  )
              else:
                  examples.extend(off_slots)
  
      for example in examples:
          self.features.append(example)
  
file_path: NeMo/nemo/collections/nlp/data/dialogue/dataset/dialogue_sgd_bert_dataset.py
imports: import os
  import re
  from typing import List
  import numpy
  from nemo.collections.nlp.data.dialogue.dataset.dialogue_dataset import DialogueDataset
  from nemo.collections.nlp.data.dialogue.input_example.sgd_input_example import SGDInputExample
label_desc: Use image node_id 9 for calling the DialogueSGDBERTDataset obj's underlying member method code with example usage: obj.bert_process_one_sample(idx) without return types