class_docstr: Implements the chatbot problem with Cornell Movie Dialog dataset.
  
  https://www.cs.cornell.edu/~cristian/Cornell_Movie-Dialogs_Corpus.html
class_docstr_tok: ["Implements","the","chatbot","problem","with","Cornell","Movie","Dialog","dataset",".","https",":","\/\/www.cs.cornell.edu\/~cristian\/Cornell_Movie-Dialogs_Corpus.html"]
docstr: Create the source, target and vocab files.
  
  Args:
    train_mode: string, whether we are in train, dev or test mode
docstr_tok: ["Create","the","source",",","target","and","vocab","files",".","Args",":","train_mode",":","string",",","whether","we","are","in","train",",","dev","or","test","mode"]
code: def create_data(self, train_mode):
      """Create the source, target and vocab files.
  
      Args:
        train_mode: string, whether we are in train, dev or test mode
      """
  
      # Open the 6 files.
      (
          trainsource,
          traintarget,
          devsource,
          devtarget,
          testsource,
          testtarget,
      ) = self.open_6_files()
  
      # Open the raw data.
      movie_lines = open(
          os.path.join(self._raw_data, "movie_lines.txt"),
          errors="ignore",
      )
      dialog_list = self.extract_dialog_ids()
  
      vocabulary = collections.Counter()
      line_dict = {}
      number_of_lines = 0
      # Iterate through file.
      for line in movie_lines:
          if number_of_lines % 10000 == 0:
              print(
                  "problem_log: Parsed "
                  + str(number_of_lines)
                  + " lines."
              )
  
          line = line.split(" +++$+++ ")
          dialog_id = line[0]
          line = line[4].lower()
  
          # Do some cleaning.
          line = self.clean_line(line)
          line_dict[dialog_id] = line
  
          number_of_lines += 1
          # Check if we reached the desired dataset size.
          if (
              self.targeted_dataset_size != 0
              and self.targeted_dataset_size < number_of_lines
          ):
              break
  
      counter = 0
      dataset_split_counter = 0
      # Save the actual dialogs.
      for dialog in dialog_list:
          if counter % 10000 == 0:
              print(
                  "problem_log: Saved "
                  + str(counter)
                  + "/"
                  + str(len(dialog_list))
                  + " dialogs."
              )
  
          dataset_split_counter += 1
          i = 0
          # Save one utterance.
          for utterance in dialog:
              if (
                  utterance != dialog[-1]
                  and dialog[i + 1] != "L211194"
                  and dialog[i + 1] != "L1045"
              ):
                  source_line = line_dict[utterance] + "\n"
                  target_line = line_dict[dialog[i + 1]] + "\n"
  
                  # Save to the files according to dataset split.
                  if (
                      dataset_split_counter
                      <= self.dataset_split["train"]
                  ):
                      # Build vocabulary.
                      words = source_line.split()
                      for word in words:
                          vocabulary[word] = vocabulary.get(word, 0) + 1
  
                      trainsource.write(source_line)
                      traintarget.write(target_line)
  
                  elif dataset_split_counter <= (
                      self.dataset_split["train"]
                      + self.dataset_split["val"]
                  ):
                      devsource.write(source_line)
                      devtarget.write(target_line)
                  else:
                      testsource.write(source_line)
                      testtarget.write(target_line)
              i += 1
  
          # Reset the split counter if we reached 100%.
          if dataset_split_counter == 100:
              dataset_split_counter = 0
          counter += 1
  
      # Close the files.
      self.close_n_files(
          [
              trainsource,
              traintarget,
              devsource,
              devtarget,
              testsource,
              testtarget,
          ]
      )
      movie_lines.close()
  
      # Save the vocabulary.
      self.save_vocab(vocabulary)
  
file_path: tensor2tensor/tensor2tensor/data_generators/dialog_cornell.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import os
  import re
  from tensor2tensor.data_generators import dialog_abstract
  from tensor2tensor.data_generators import text_encoder
  from tensor2tensor.utils import registry
label_desc: Use image node_id 3 for calling the DialogCornell32k obj's underlying member method code with example usage: obj.create_data(train_mode) without return types