class_docstr: A class implementing the chatbot problem for the OpenSubtitles dataset.
  
  http://opus.nlpl.eu/OpenSubtitles-v2018.php
class_docstr_tok: ["A","class","implementing","the","chatbot","problem","for","the","OpenSubtitles","dataset",".","http",":","\/\/opus.nlpl.eu\/OpenSubtitles-v2018.php"]
docstr: Create the source, target and vocab files.
  
  Args:
    train_mode: string, whether we are in train, dev or test mode
docstr_tok: ["Create","the","source",",","target","and","vocab","files",".","Args",":","train_mode",":","string",",","whether","we","are","in","train",",","dev","or","test","mode"]
code: def create_data(self, train_mode):
      """Create the source, target and vocab files.
  
      Args:
        train_mode: string, whether we are in train, dev or test mode
      """
  
      # open the 6 files
      (
          trainsource,
          traintarget,
          devsource,
          devtarget,
          testsource,
          testtarget,
      ) = self.open_6_files()
  
      conv_id = 0
      number_of_lines = 0
      dataset_split_counter = 0
      vocabulary = collections.Counter()
      # Dind all the files.
      for root, _, files in os.walk(self._raw_data_dir):
          for f in files:
              if conv_id % 100 == 0:
                  print(
                      "problem_log: Parsed " + str(conv_id) + " files."
                  )
  
              source_lines = ""
              target_lines = ""
              conv_id += 1
              dataset_split_counter += 1
  
              # Open one .xml file and parse it.
              with open(
                  os.path.join(root, f), "r", errors="ignore"
              ) as txt_file:
                  words = ""
                  line_id = 1
  
                  # Parse one line.
                  for line in txt_file:
                      line = str(line)
  
                      # Check if it's a new sentence.
                      if line.find('<s id="') != -1:
                          if words:
                              # Do some cleaning.
                              words = self.clean_line(words)
  
                              # Build the vocabulary.
                              if (
                                  dataset_split_counter
                                  <= self.dataset_split["train"]
                              ):
                                  word_list = words.split()
                                  for word in word_list:
                                      vocabulary[word] = (
                                          vocabulary.get(word, 0) + 1
                                      )
  
                              # Add the previous line.
                              source_lines += words + "\n"
                              if line_id != 1:
                                  target_lines += words + "\n"
                              line_id += 1
                          words = ""
  
                      else:
                          index = line.find('<w id="')
                          if index >= 0:
                              line = line[index:]
                              word = line[
                                  line.find(">") + 1 : line.find("</w")
                              ]
                              words = (
                                  words + " " + word.replace("\t", " ")
                              )
  
                  # Delete the final source sentence, since it doesn't have a target.
                  source_lines = (
                      "\n".join(source_lines.split("\n")[:-2]) + "\n"
                  )
  
              # Save the dialog according to the dataset split.
              if dataset_split_counter <= self.dataset_split["train"]:
                  trainsource.write(source_lines)
                  traintarget.write(target_lines)
              elif dataset_split_counter <= (
                  self.dataset_split["train"]
                  + self.dataset_split["val"]
              ):
                  devsource.write(source_lines)
                  devtarget.write(target_lines)
              else:
                  testsource.write(source_lines)
                  testtarget.write(target_lines)
  
              # Reset the split counter if we reached 100%.
              if dataset_split_counter == 100:
                  dataset_split_counter = 0
  
              # Check if we reached the desired dataset size.
              number_of_lines += line_id
              if (
                  self.targeted_dataset_size != 0
                  and self.targeted_dataset_size < number_of_lines
              ):
                  break
          else:
              continue
          break
  
      # Close the files.
      self.close_n_files(
          [
              trainsource,
              traintarget,
              devsource,
              devtarget,
              testsource,
              testtarget,
          ]
      )
      # Save the vocabulary.
      self.save_vocab(vocabulary)
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import os
  import re
  import zipfile
  from tensor2tensor.data_generators import dialog_abstract
  from tensor2tensor.data_generators import text_encoder
  from tensor2tensor.utils import registry
label_desc: Use image node_id 5 for calling the DialogOpensubtitles64k2009 obj's underlying member method code with example usage: obj.create_data(train_mode) without return types