class_docstr: The implement of DialogReader.
code: def _convert_example_to_record(self, example, is_infer):
      # process src
      src_token_ids = []
      src_pos_ids = []
  
      # tokenize src
      s_token_ids_list = []
      for s in example.src.split("[SEP]"):
          s = tokenization.convert_to_unicode(s).strip()
  
          if self.data_format == "tokenized":
              s_tokens = s.split(" ")
          else:
              s_tokens = self.tokenizer.tokenize(s)
  
          s_token_ids = self.tokenizer.convert_tokens_to_ids(
              s_tokens
          ) + [self.eos_id]
          s_token_ids_list.append(s_token_ids)
  
      # trim src
      idx = len(s_token_ids_list) - 1
      total_token_num = 1
      while idx >= 0:
          total_token_num += len(s_token_ids_list[idx])
          if total_token_num > self.max_src_len:
              if self.truncate_first_turn and idx == 0:
                  truncated_ids = s_token_ids_list[idx][
                      : self.max_src_len - total_token_num
                  ]
                  if len(truncated_ids) > 1:
                      s_token_ids_list[idx] = truncated_ids[:-1] + [
                          self.eos_id
                      ]
                      idx -= 1
              break
          idx -= 1
  
      for i, s_token_ids in enumerate(
          s_token_ids_list[idx + 1 :], idx + 1
      ):
          src_token_ids += s_token_ids
          src_pos_ids += list(range(1, len(s_token_ids) + 1))
  
      src_token_ids = [self.bos_id] + src_token_ids
      src_type_ids = [0] * len(src_token_ids)
      src_pos_ids = [0] + src_pos_ids
      assert (
          len(src_token_ids) == len(src_type_ids) == len(src_pos_ids)
      ), "not len(src_token_ids) == len(src_type_ids) == len(src_pos_ids)"
  
      token_ids = src_token_ids
      type_ids = src_type_ids
      pos_ids = src_pos_ids
      tgt_start_idx = len(token_ids)
  
      if not is_infer:
          # process tgt
          # tokenize tgt
          tgt = tokenization.convert_to_unicode(example.tgt).strip()
          if self.data_format == "tokenized":
              tgt_tokens = tgt.split(" ")
          else:
              tgt_tokens = self.tokenizer.tokenize(tgt)
  
          tgt_token_ids = self.tokenizer.convert_tokens_to_ids(
              tgt_tokens
          )
          tgt_token_ids.append(self.eos_id)
  
          # trim tgt
          if len(tgt_token_ids) > self.max_tgt_len - 1:
              tgt_token_ids = tgt_token_ids[: self.max_tgt_len - 1]
  
          tgt_token_ids = [self.bos_id] + tgt_token_ids
          tgt_type_ids = [1] * len(tgt_token_ids)
          tgt_pos_ids = list(range(1, len(tgt_token_ids) + 1))
          assert (
              len(tgt_token_ids)
              == len(tgt_type_ids)
              == len(tgt_pos_ids)
          ), "not len(tgt_token_ids) == len(tgt_type_ids) == len(tgt_pos_ids)"
  
          token_ids += tgt_token_ids
          type_ids += tgt_type_ids
          pos_ids += tgt_pos_ids
  
      assert (
          len(token_ids) == len(type_ids) == len(pos_ids)
      ), "not len(token_ids) == len(type_ids) == len(pos_ids)"
  
      if self.continuous_position:
          src_pos_ids = list(range(len(src_token_ids)))
          if not is_infer:
              tgt_pos_ids = list(range(len(tgt_token_ids)))
          pos_ids = list(range(len(token_ids)))
  
      field_values = {
          "token_ids": src_token_ids,
          "type_ids": src_type_ids,
          "pos_ids": src_pos_ids,
      }
      field_values["tgt_start_idx"] = tgt_start_idx
      field_values["data_id"] = example.data_id
  
      record = self.Record(**field_values)
      return record
  
imports: import csv
  from collections import namedtuple
  from contextlib import contextmanager
  import gzip
  import numpy
  from ..utils import pad_batch_data
  from ..utils.args import str2bool
  from ..utils.masking import mask
  from ..utils import tokenization
label_desc: Use image node_id 4 for calling the DialogReader obj's underlying member method code with example usage: obj._convert_example_to_record(example, is_infer) and returns: record