text: from __future__ import unicode_literals, print_function, division
  from io import open
  import unicodedata
  import string
  import re
  import random
  
  import torch
  import torch.nn as nn
  from torch import optim
  import torch.nn.functional as F
  import pandas as pd
  
  
  SOS_token = 0
  EOS_token = 1
  
  MAX_LENGTH = 100
  
  
  class Lang:
      def __init__(self, name):
          self.name = name
          self.char2index = {}
          self.char2count = {}
          self.index2char = {0: "SOS", 1: "EOS"}
          self.n_chars = 2  # Count SOS and EOS
  
      def addSentence(self, sentence):
          for word in sentence.split(' '):
              self.addWord(word)
  
      def addWord(self, word):
          for char in word:
              if char not in self.char2index:
                  self.char2index[char] = self.n_chars
                  self.char2count[char] = 1
                  self.index2char[self.n_chars] = char
                  self.n_chars += 1
              else:
                  self.char2count[char] += 1
  
  
  # Turn a Unicode string to plain ASCII, thanks to
  # https://stackoverflow.com/a/518232/2809427
  def unicodeToAscii(s):
      return ''.join(
          c for c in unicodedata.normalize('NFD', s)
          if unicodedata.category(c) != 'Mn'
      )
  
  # Lowercase, trim, and remove non-letter characters
  
  
  def normalizeString(s):
      # s = unicodeToAscii(s.lower().strip())
      s = re.sub(r"([.!?])", r" \1", s.lower().strip())
      # s = re.sub(r"[^a-zA-Z.!?]+", r" ", s)
      s = re.sub(r"[゛゜]+", r"", s)
      return s
  
  
  def readLangs(lang1, lang2, reverse=False):
      print("Reading lines...")
  
      # Read the file and split into lines
      df = pd.read_csv('data/bep-eng.csv', keep_default_na=False)
      # print(max(df[lang1].apply(len)), max(df[lang2].apply(len)))
  
      # Split every line into pairs and normalize
      pairs = [[normalizeString(l1), normalizeString(l2)] for l1, l2 in zip(df[lang1], df[lang2])]
  
      # Reverse pairs, make Lang instances
      if reverse:
          pairs = [list(reversed(p)) for p in pairs]
          input_lang = Lang(lang2)
          output_lang = Lang(lang1)
      else:
          input_lang = Lang(lang1)
          output_lang = Lang(lang2)
  
      return input_lang, output_lang, pairs
  
  
  def prepareData(lang1, lang2, reverse=False):
      input_lang, output_lang, pairs = readLangs(lang1, lang2, reverse)
      print("Read %s sentence pairs" % len(pairs))
      print("Trimmed to %s sentence pairs" % len(pairs))
      print("Counting words...")
      for pair in pairs:
          input_lang.addSentence(pair[0])
          output_lang.addSentence(pair[1])
      print("Counted words:")
      print(input_lang.name, input_lang.n_chars)
      print(output_lang.name, output_lang.n_chars)
      """
      for char in sorted(input_lang.char2index):
          print(char)
      for char in sorted(output_lang.char2index):
          print(char)
      """
      return input_lang, output_lang, pairs
  