class_docstr: Data preprocessor.
      
code: def __init__(self, jsonl_path, mode=None):
      self.mode = mode
      self.raw = []
      self.lst = []
      self.refs = []
      if mode == "test":
          lst = json.load(open(jsonl_path, "r"))
          for item in lst:
              context = item["context"]
              dialog = []
              for utts in context:
                  p = utts.find(":")
                  dialog.append(
                      (
                          (utts[p - 1] == "A") * 2 - 1,
                          utts[p + 2 : -1],
                          0,
                      )
                  )
  
              if dialog[0][1][-1] == ">":
                  dialog = dialog[1:]
  
              if len(dialog) == 0:
                  continue
  
              responses = []
              for resp in item["responses"]:
                  responses.append(resp)
  
              spk = (item["speaker"] == "A") * 2 - 1
              dialog.append((spk, responses[0], 0))
              responses = responses[1:]
              responses = [
                  " ".join(WordPunctTokenizer().tokenize(resp))
                  for resp in responses
              ]
  
              if len(responses) == 0:
                  continue
  
              self.raw.append(dialog)
              self.lst.append((len(self.raw) - 1, 0, len(dialog)))
              self.refs.append(responses)
  
          return
  
      from collections import Counter
  
      self.ct = Counter()
      self.topics = []
      with open(jsonl_path, "r") as f:
          for idx, item in enumerate(reader(f)):
              utts = item["utts"]
              self.topics.append(item["topic"])
              self.raw.append(
                  [
                      (int(speaker == "A") * 2 - 1, sentence, _)
                      for speaker, sentence, _ in utts
                  ]
              )
  
              lst = [
                  (idx, start, start + wnd_sz)
                  for start in range(0, len(utts) - wnd_sz)
              ] + [
                  (idx, 0, end)
                  for end in range(2, min(wnd_sz + 1, len(utts)))
              ]
  
              self.lst += lst
  
      self.refs = [["none"]] * len(self.lst)
  
file_path: texar/examples/hierarchical_dialog/sw_loader.py
imports: import os
  import json
  from json_lines import reader
  from nltk.tokenize import WordPunctTokenizer
  from sklearn.feature_extraction.text import TfidfVectorizer
  import texar.tf
  from config_data import data_root
label_desc: Use image node_id 1 to create a new Dataset object from inherited base classes: object with example: obj = Dataset(jsonl_path, mode)