text: import unicodedata
  import re
  import torch
  
  PAD_token = 0  # Used for padding short sentences
  SOS_token = 1  # Start-of-sentence token
  EOS_token = 2  # End-of-sentence token
  MAX_LENGTH = 10
  MIN_COUNT = 3
  
  
  class Voc:
      def __init__(self):
          self.trimmed = False
          self.word2index = {}
          self.word2count = {}
          self.index2word = {PAD_token: 'PAD', SOS_token: 'SOS', EOS_token: 'EOS'}
          self.num_words = 3
  
      def addSentence(self, sentence):
          for word in sentence.split(' '):
              self.addWord(word)
  
      def addWord(self, word):
          if word not in self.word2index:
              self.word2index[word] = self.num_words
              self.word2count[word] = 1
              self.index2word[self.num_words] = word
              self.num_words += 1
          else:
              self.word2count[word] += 1
  
      def trim(self, min_count):
          if self.trimmed:
              return
          self.trimmed = True
          keep_words = []
  
          for k, v in self.word2count.items():
              if v >= min_count:
                  keep_words.append(k)
  
          print('keep_words {} / {} = {:.4f}'.format(len(keep_words),
                len(self.word2index), len(keep_words) / len(self.word2index)))
  
          self.word2index = {}
          self.word2count = {}
          self.index2word = {PAD_token: "PAD", SOS_token: "SOS", EOS_token: "EOS"}
          self.num_words = 3  # Count default tokens
  
          for word in keep_words:
              self.addWord(word)
  
  
  def unicodeToAscii(s):
      return ''.join(c for c in unicodedata.normalize('NFD', s)
                     if unicodedata.category(c) != 'Mn')
  
  
  def normalizeString(s):
      s = unicodeToAscii(s.lower().strip())
      s = re.sub(r'([.!?])', r' \1', s)  # '1234!!qwer??' -> '1234 ! !qwer ? ?'
      s = re.sub(r'[^a-zA-Z.!?]+', r' ', s)  # -> ' !!qwer??'
      s = re.sub(r'\s+', r' ', s).strip()  # -> '!!qwer??'
      return s
  
  
  def readVocs(datafile):
      """
      :param datafile:
      :return: pairs: [[xxx, xxx], [], ...]
               voc:
  
      """
      print('Reading lines...')
      lines = open(datafile, encoding='utf-8').read().strip().split('\n')
      pairs = [[normalizeString(s) for s in l.split('\t')] for l in lines]
      voc = Voc()
      return voc, pairs
  
  
  def filterPair(p):
      return len(p[0].split(' ')) < MAX_LENGTH and len(p[1].split(' ')) < MAX_LENGTH
  
  
  def filterPairs(pairs):
      return [pair for pair in pairs if filterPair(pair)]
  
  
  def prepareData():
      print('Start preparing train data ...')
      voc, pairs = readVocs('data/formatted_movie_lines.txt')
      print('Read {!s} sentence pairs'.format(len(pairs)))
      pairs = filterPairs(pairs)
      print('Trimmed to {!s} sentence pairs'.format(len(pairs)))
      print('Count words...')
      for pair in pairs:
          voc.addSentence(pair[0])
          voc.addSentence(pair[1])
      print('Count words:', voc.num_words)
  
      torch.save(voc, 'save/voc.pkl')
      torch.save(pairs, 'save/pairs.pkl')
      return voc, pairs
  
  
  def trimRareWords(voc, pairs, MIN_COUNT):
      voc.trim(MIN_COUNT)
      keep_pairs = []
      for pair in pairs:
          input_sentence = pair[0]
          output_sentence = pair[1]
          keep_input = True
          keep_output = True
          for word in input_sentence.split(' '):
              if word not in voc.word2index:
                  keep_input = False
                  break
          for word in output_sentence.split(' '):
              if word not in voc.word2index:
                  keep_output = False
                  break
  
          if keep_input and keep_output:
              keep_pairs.append(pair)
      print("Trimmed from {} pairs to {}, {:.4f} of total"
            .format(len(pairs), len(keep_pairs), len(keep_pairs) / len(pairs)))
      return keep_pairs
  