class_docstr: Dialogue State Tracking Model SGD-QA (https://arxiv.org/abs/2105.08049)
  
  The SGD-QA model is a fast multi-pass schema-guided state-tracking model, that is trained on the Google schema-guided state tracking dataset (https://arxiv.org/abs/1909.05855).
  The model takes dialogue as input and outputs the dialogue state, which includes slot-value pairs. 
  The model consists of two components: a neural natural language understanding model (NLU), and a rule-based state tracker.
  The NLU takes in a dialogue turn and different schema (entity) information options and outputs their match score. The state tracker takes the highest rated entities and composes
  the dialogue state across turns.
class_docstr_tok: ["Dialogue","State","Tracking","Model","SGD-QA","(","https",":","\/\/arxiv.org\/abs\/2105.08049",")","The","SGD-QA","model","is","a","fast","multi-pass","schema-guided","state-tracking","model",",","that","is","trained","on","the","Google","schema-guided","state","tracking","dataset","(","https",":","\/\/arxiv.org\/abs\/1909.05855",")",".","The","model","takes","dialogue","as","input","and","outputs","the","dialogue","state",",","which","includes","slot-value","pairs",".","The","model","consists","of","two","components",":","a","neural","natural","language","understanding","model","(","NLU",")",",","and","a","rule-based","state","tracker",".","The","NLU","takes","in","a","dialogue","turn","and","different","schema","(","entity",")","information","options","and","outputs","their","match","score",".","The","state","tracker","takes","the","highest","rated","entities","and","composes","the","dialogue","state","across","turns","."]
docstr: Helper called at the end of evaluation to post process outputs into human readable format
  Args:
      outputs: list of individual outputs of each step
      split: data split
      dataloader: dataloader
  Returns:
      metrics: metrics collection
docstr_tok: ["Helper","called","at","the","end","of","evaluation","to","post","process","outputs","into","human","readable","format","Args",":","outputs",":","list","of","individual","outputs","of","each","step","split",":","data","split","dataloader",":","dataloader","Returns",":","metrics",":","metrics","collection"]
code: def multi_eval_epoch_end_helper(
      self,
      outputs: List[dict],
      split: str,
      dataloader: torch.utils.data.DataLoader,
  ) -> dict:
      """
      Helper called at the end of evaluation to post process outputs into human readable format
      Args:
          outputs: list of individual outputs of each step
          split: data split
          dataloader: dataloader
      Returns:
          metrics: metrics collection
      """
  
      def get_str_example_id(
          split: str,
          ids_to_service_names_dict: dict,
          example_id_num: torch.Tensor,
      ) -> str:
          """
          Constructs string representation of example ID
          Args:
              split: evaluation data split
              ids_to_service_names_dict: id to service name mapping
              example_id_num: tensor example id
          """
  
          def format_turn_id(ex_id_num):
              (
                  dialog_id_1,
                  dialog_id_2,
                  turn_id,
                  service_id,
                  model_task_id,
                  slot_intent_id,
                  value_id,
              ) = ex_id_num
              return "{}-{}_{:05d}-{:02d}-{}-{}-{}-{}".format(
                  split,
                  dialog_id_1,
                  dialog_id_2,
                  turn_id,
                  ids_to_service_names_dict[service_id],
                  model_task_id,
                  slot_intent_id,
                  value_id,
              )
  
          return list(map(format_turn_id, tensor2list(example_id_num)))
  
      def combine_predictions_in_example(
          predictions: dict, batch_size: int
      ):
          """
          Combines predicted values to a single example.
          Args:
              predictions: predictions ordered by keys then batch
              batch_size: batch size
          Returns:
              examples_preds: predictions ordered by batch then key
          """
          examples_preds = [{} for _ in range(batch_size)]
          for k, v in predictions.items():
              if k != "example_id":
                  v = torch.chunk(v, batch_size)
  
              for i in range(batch_size):
                  if k == "example_id":
                      examples_preds[i][k] = v[i]
                  else:
                      examples_preds[i][k] = v[i].view(-1)
          return examples_preds
  
      example_id_num = torch.cat(
          [x[f"tensors"]["example_id_num"] for x in outputs]
      )
      service_id = torch.cat(
          [x[f"tensors"]["service_id"] for x in outputs]
      )
      intent_status = torch.cat(
          [x[f"tensors"]["intent_status"] for x in outputs]
      )
      req_slot_status = torch.cat(
          [x[f"tensors"]["req_slot_status"] for x in outputs]
      )
      cat_slot_status = torch.cat(
          [x[f"tensors"]["cat_slot_status"] for x in outputs]
      )
      cat_slot_status_p = torch.cat(
          [x[f"tensors"]["cat_slot_status_p"] for x in outputs]
      )
      cat_slot_value_status = torch.cat(
          [x[f"tensors"]["cat_slot_value_status"] for x in outputs]
      )
      noncat_slot_status = torch.cat(
          [x[f"tensors"]["noncat_slot_status"] for x in outputs]
      )
      noncat_slot_status_p = torch.cat(
          [x[f"tensors"]["noncat_slot_status_p"] for x in outputs]
      )
      noncat_slot_p = torch.cat(
          [x[f"tensors"]["noncat_slot_p"] for x in outputs]
      )
      noncat_slot_start = torch.cat(
          [x[f"tensors"]["noncat_slot_start"] for x in outputs]
      )
      noncat_slot_end = torch.cat(
          [x[f"tensors"]["noncat_slot_end"] for x in outputs]
      )
      noncat_alignment_start = torch.cat(
          [x[f"tensors"]["noncat_alignment_start"] for x in outputs]
      )
      noncat_alignment_end = torch.cat(
          [x[f"tensors"]["noncat_alignment_end"] for x in outputs]
      )
  
      ids_to_service_names_dict = (
          self.dialogues_processor.schemas._services_id_to_vocab
      )
      example_id = get_str_example_id(
          dataloader.dataset, ids_to_service_names_dict, example_id_num
      )
  
      metrics = {}
      try:
          prediction_dir = (
              self.trainer.log_dir
              if self.trainer.log_dir is not None
              else ""
          )
      except:
          prediction_dir = ""
  
      if self.trainer.global_rank == 0:
          prediction_dir = os.path.join(
              prediction_dir,
              "predictions",
              "pred_res_{}_{}".format(
                  split, self._cfg.dataset.task_name
              ),
          )
          os.makedirs(prediction_dir, exist_ok=True)
  
          input_json_files = (
              DialogueSGDDataProcessor.get_dialogue_files(
                  self._cfg.dataset.data_dir,
                  split,
                  self._cfg.dataset.task_name,
              )
          )
  
          predictions = {}
          predictions["example_id"] = example_id
          predictions["service_id"] = service_id
          predictions["intent_status"] = intent_status
          predictions["req_slot_status"] = req_slot_status
          predictions["cat_slot_status"] = cat_slot_status
          predictions["cat_slot_status_p"] = cat_slot_status_p
          predictions["cat_slot_value_status"] = cat_slot_value_status
          predictions["noncat_slot_status"] = noncat_slot_status
          predictions["noncat_slot_status_p"] = noncat_slot_status_p
          predictions["noncat_slot_p"] = noncat_slot_p
          predictions["noncat_slot_start"] = noncat_slot_start
          predictions["noncat_slot_end"] = noncat_slot_end
          predictions["noncat_alignment_start"] = noncat_alignment_start
          predictions["noncat_alignment_end"] = noncat_alignment_end
  
          in_domain_services = get_in_domain_services(
              os.path.join(
                  self._cfg.dataset.data_dir, split, "schema.json"
              ),
              self.dialogues_processor.get_seen_services("train"),
          )
          predictions = combine_predictions_in_example(
              predictions, service_id.shape[0]
          )
  
          # write predictions to file in Dstc8/SGD format
          write_predictions_to_file(
              predictions,
              input_json_files,
              output_dir=prediction_dir,
              schemas=self.dialogues_processor.schemas,
              state_tracker=self._cfg.dataset.state_tracker,
              eval_debug=False,
              in_domain_services=in_domain_services,
          )
          metrics = evaluate(
              prediction_dir,
              self._cfg.dataset.data_dir,
              split,
              in_domain_services,
              joint_acc_across_turn=self._cfg.dataset.joint_acc_across_turn,
              use_fuzzy_match=self._cfg.dataset.use_fuzzy_match,
          )
  
      return metrics
  
imports: import os
  from typing import List, Optional
  import torch
  from omegaconf import DictConfig
  from pytorch_lightning import Trainer
  from torch.utils.data import DataLoader
  from nemo.collections.nlp.data.dialogue import DialogueSGDBERTDataset, DialogueSGDDataProcessor
  from nemo.collections.nlp.data.dialogue.sgd.evaluate import evaluate, get_in_domain_services
  from nemo.collections.nlp.data.dialogue.sgd.prediction_utils import write_predictions_to_file
  from nemo.collections.nlp.losses import SGDDialogueStateLoss
  from nemo.collections.nlp.models.nlp_model import NLPModel
  from nemo.collections.nlp.modules import SGDDecoder, SGDEncoder
  from nemo.collections.nlp.parts.utils_funcs import tensor2list
  from nemo.core.classes.common import PretrainedModelInfo, typecheck
  from nemo.utils import logging
label_desc: Use image node_id 10 for calling the SGDQAModel obj's underlying member method code with example usage: obj.multi_eval_epoch_end_helper(outputs, split, dataloader) and returns: metrics, list, examples_preds, str