text: import collections
  import os
  import re
  
  import numpy as np
  from six.moves import cPickle
  
  from .functional import *
  
  
  class BatchLoader:
      #TEST: initially path='../../'
      def __init__(self, path='../'):
  
          '''
              :properties
  
                  data_files - array containing paths to data sources
  
                  idx_files - array of paths to vocabulury files
  
                  tensor_files - matrix with shape of [2, target_num] containing paths to files
                      with data represented as tensors
                      where first index in shape corresponds to types of representation of data,
                      i.e. word representation and character-aware representation
  
                  blind_symbol - special symbol to fill spaces in every word in character-aware representation
                      to make all words be the same lenght
                  pad_token - the same special symbol as blind_symbol, but in case of lines of words
                  go_token - start of sequence symbol
                  end_token - end of sequence symbol
  
                  chars_vocab_size - number of unique characters
                  idx_to_char - array of shape [chars_vocab_size] containing ordered list of inique characters
                  char_to_idx - dictionary of shape [chars_vocab_size]
                      such that idx_to_char[char_to_idx[some_char]] = some_char
                      where some_char is such that idx_to_char contains it
  
                  words_vocab_size, idx_to_word, word_to_idx - same as for characters
  
                  max_word_len - maximum word length
                  max_seq_len - maximum sequence length
                  num_lines - num of lines in data with shape [target_num]
  
                  word_tensor -  tensor of shape [target_num, num_lines, line_lenght] c
                      ontains word's indexes instead of words itself
  
                  character_tensor - tensor of shape [target_num, num_lines, line_lenght, max_word_len].
                      Rows contain character indexes for every word in data
  
              :methods
  
                  build_character_vocab(self, data) -> chars_vocab_size, idx_to_char, char_to_idx
                      chars_vocab_size - size of unique characters in corpus
                      idx_to_char - array of shape [chars_vocab_size] containing ordered list of inique characters
                      char_to_idx - dictionary of shape [chars_vocab_size]
                          such that idx_to_char[char_to_idx[some_char]] = some_char
                          where some_char is such that idx_to_char contains it
  
                  build_word_vocab(self, sentences) -> words_vocab_size, idx_to_word, word_to_idx
                      same as for characters
  
                  preprocess(self, data_files, idx_files, tensor_files) -> Void
                      preprocessed and initialized properties and then save them
  
                  load_preprocessed(self, data_files, idx_files, tensor_files) -> Void
                      load and and initialized properties
  
                  next_batch(self, batch_size, target_str) -> encoder_word_input, encoder_character_input, input_seq_len,
                          decoder_input, decoder_output
                      randomly sampled batch_size num of sequences for target from target_str.
                      fills sequences with pad tokens to made them the same lenght.
                      encoder_word_input and encoder_character_input have reversed order of the words
                          in case of performance
          '''
  
          self.data_files = [path + 'data/train.txt',
                             path + 'data/test.txt',
                             path + 'data/ptb_test.txt']
  
          self.idx_files = [path + 'data/words_vocab.pkl',
                            path + 'data/characters_vocab.pkl']
  
          self.tensor_files = [[path + 'data/train_word_tensor.npy',
                                path + 'data/valid_word_tensor.npy',
                                path + 'data/test_word_tensor.npy'],
                               [path + 'data/train_character_tensor.npy',
                                path + 'data/valid_character_tensor.npy',
                                path + 'data/test_character_tensor.npy']]
  
          self.blind_symbol = ''
          self.pad_token = '_'
          self.go_token = '>'
          self.end_token = '|'
  
          idx_exists = fold(f_and,
                            [os.path.exists(file) for file in self.idx_files],
                            True)
  
          tensors_exists = fold(f_and,
                                [os.path.exists(file) for target in self.tensor_files
                                 for file in target],
                                True)
  
          if idx_exists and tensors_exists:
              self.load_preprocessed(self.data_files,
                                     self.idx_files,
                                     self.tensor_files)
              print('preprocessed data was found and loaded')
          else:
              self.preprocess(self.data_files,
                              self.idx_files,
                              self.tensor_files)
              print('data have preprocessed')
  
          self.word_embedding_index = 0
  
      def clean_whole_data(self, string):
          string = re.sub('^[\d\:]+ ', '', string, 0, re.M)
          string = re.sub('\n\s{11}', ' ', string, 0, re.M)
          string = re.sub('\n{2}', '\n', string, 0, re.M)
  
          return string.lower()
  
      def clean_str(self, string):
          '''
              Tokenization/string cleaning for all datasets except for SST.
              Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data
          '''
  
          string = re.sub(r"[^가-힣A-Za-z0-9(),!?:;.\'\`]", " ", string)
          string = re.sub(r"\'s", " \'s", string)
          string = re.sub(r"\'ve", " \'ve", string)
          string = re.sub(r"n\'t", " n\'t", string)
          string = re.sub(r"\'re", " \'re", string)
          string = re.sub(r"\'d", " \'d", string)
          string = re.sub(r"\'ll", " \'ll", string)
          string = re.sub(r"\.", " . ", string)
          string = re.sub(r",", " , ", string)
          string = re.sub(r":", " : ", string)
          string = re.sub(r";", " ; ", string)
          string = re.sub(r"!", " ! ", string)
          string = re.sub(r"\(", " ( ", string)
          string = re.sub(r"\)", " ) ", string)
          string = re.sub(r"\?", " ? ", string)
          string = re.sub(r"\s{2,}", " ", string)
          return string.strip()
  
      def build_character_vocab(self, data):
  
          # unique characters with blind symbol
          chars = list(set(data)) + [self.blind_symbol, self.pad_token, self.go_token, self.end_token]
          chars_vocab_size = len(chars)
  
          # mappings itself
          idx_to_char = chars
          char_to_idx = {x: i for i, x in enumerate(idx_to_char)}
  
          return chars_vocab_size, idx_to_char, char_to_idx
  
      def build_word_vocab(self, sentences):
  
          # Build vocabulary
          word_counts = collections.Counter(sentences)
  
          # Mapping from index to word
          idx_to_word = [x[0] for x in word_counts.most_common()]
          idx_to_word = list(sorted(idx_to_word)) + [self.pad_token, self.go_token, self.end_token]
  
          words_vocab_size = len(idx_to_word)
  
          # Mapping from word to index
          word_to_idx = {x: i for i, x in enumerate(idx_to_word)}
  
          return words_vocab_size, idx_to_word, word_to_idx
  
      def preprocess(self, data_files, idx_files, tensor_files):
  
          data = [open(file, "r").read() for file in data_files]
          merged_data = data[0] + '\n' + data[1]
  
          self.chars_vocab_size, self.idx_to_char, self.char_to_idx = self.build_character_vocab(merged_data)
  
          with open(idx_files[1], 'wb') as f:
              cPickle.dump(self.idx_to_char, f)
  
          data_words = [[line.split() for line in target.split('\n')] for target in data]
          merged_data_words = merged_data.split()
  
          self.words_vocab_size, self.idx_to_word, self.word_to_idx = self.build_word_vocab(merged_data_words)
          self.max_word_len = np.amax([len(word) for word in self.idx_to_word])
          self.max_seq_len = np.amax([len(line) for target in data_words for line in target])
          self.num_lines = [len(target) for target in data_words]
  
          with open(idx_files[0], 'wb') as f:
              cPickle.dump(self.idx_to_word, f)
  
          self.word_tensor = np.array(
              [[list(map(self.word_to_idx.get, line)) for line in target] for target in data_words])
          print(self.word_tensor.shape)
          for i, path in enumerate(tensor_files[0]):
              np.save(path, self.word_tensor[i])
  
          self.character_tensor = np.array(
              [[list(map(self.encode_characters, line)) for line in target] for target in data_words])
          for i, path in enumerate(tensor_files[1]):
              np.save(path, self.character_tensor[i])
  
          self.just_words = [word for line in self.word_tensor[0] for word in line]
  
      def load_preprocessed(self, data_files, idx_files, tensor_files):
  
          data = [open(file, "r").read() for file in data_files]
          data_words = [[line.split() for line in target.split('\n')] for target in data]
          self.max_seq_len = np.amax([len(line) for target in data_words for line in target])
          self.num_lines = [len(target) for target in data_words]
  
          [self.idx_to_word, self.idx_to_char] = [cPickle.load(open(file, "rb")) for file in idx_files]
  
          [self.words_vocab_size, self.chars_vocab_size] = [len(idx) for idx in [self.idx_to_word, self.idx_to_char]]
  
          [self.word_to_idx, self.char_to_idx] = [dict(zip(idx, range(len(idx)))) for idx in
                                                  [self.idx_to_word, self.idx_to_char]]
  
          self.max_word_len = np.amax([len(word) for word in self.idx_to_word])
  
          [self.word_tensor, self.character_tensor] = [np.array([np.load(target, allow_pickle=True) for target in input_type])
                                                       for input_type in tensor_files]
  
          self.just_words = [word for line in self.word_tensor[0] for word in line]
  
      def next_batch(self, batch_size, target_str):
          if target_str == 'train':
              target = 0
          elif target_str == 'valid':
              target = 1
          elif target_str == 'test':
              target = 2
  
          indexes = np.array(np.random.randint(self.num_lines[target], size=batch_size))
  
          encoder_word_input = [self.word_tensor[target][index] for index in indexes]
          encoder_character_input = [self.character_tensor[target][index] for index in indexes]
          input_seq_len = [len(line) for line in encoder_word_input]
          max_input_seq_len = np.amax(input_seq_len)
  
          encoded_words = [[idx for idx in line] for line in encoder_word_input]
          decoder_word_input = [[self.word_to_idx[self.go_token]] + line for line in encoder_word_input]
          decoder_character_input = [[self.encode_characters(self.go_token)] + line for line in encoder_character_input]
          decoder_output = [line + [self.word_to_idx[self.end_token]] for line in encoded_words]
  
          # sorry
          for i, line in enumerate(decoder_word_input):
              line_len = input_seq_len[i]
              to_add = max_input_seq_len - line_len
              decoder_word_input[i] = line + [self.word_to_idx[self.pad_token]] * to_add
  
          for i, line in enumerate(decoder_character_input):
              line_len = input_seq_len[i]
              to_add = max_input_seq_len - line_len
              decoder_character_input[i] = line + [self.encode_characters(self.pad_token)] * to_add
  
          for i, line in enumerate(decoder_output):
              line_len = input_seq_len[i]
              to_add = max_input_seq_len - line_len
              decoder_output[i] = line + [self.word_to_idx[self.pad_token]] * to_add
  
          for i, line in enumerate(encoder_word_input):
              line_len = input_seq_len[i]
              to_add = max_input_seq_len - line_len
              encoder_word_input[i] = [self.word_to_idx[self.pad_token]] * to_add + line[::-1]
  
          for i, line in enumerate(encoder_character_input):
              line_len = input_seq_len[i]
              to_add = max_input_seq_len - line_len
              encoder_character_input[i] = [self.encode_characters(self.pad_token)] * to_add + line[::-1]
  
          return np.array(encoder_word_input), np.array(encoder_character_input), \
                 np.array(decoder_word_input), np.array(decoder_character_input), np.array(decoder_output)
  
      def next_embedding_seq(self, seq_len):
          """
          :return:
              tuple of input and output for word embedding learning,
              where input = [b, b, c, c, d, d, e, e]
              and output  = [a, c, b, d, d, e, d, g]
              for line [a, b, c, d, e, g] at index i
          """
  
          words_len = len(self.just_words)
          seq = [self.just_words[i % words_len]
                 for i in np.arange(self.word_embedding_index, self.word_embedding_index + seq_len)]
  
          result = []
          for i in range(seq_len - 2):
              result.append([seq[i + 1], seq[i]])
              result.append([seq[i + 1], seq[i + 2]])
  
          self.word_embedding_index = (self.word_embedding_index + seq_len) % words_len - 2
  
          # input and target
          result = np.array(result)
  
          return result[:, 0], result[:, 1]
  
      def go_input(self, batch_size):
          go_word_input = [[self.word_to_idx[self.go_token]] for _ in range(batch_size)]
          go_character_input = [[self.encode_characters(self.go_token)] for _ in range(batch_size)]
  
          return np.array(go_word_input), np.array(go_character_input)
  
      def encode_word(self, idx):
          result = np.zeros(self.words_vocab_size)
          result[idx] = 1
          return result
  
      def decode_word(self, word_idx):
          word = self.idx_to_word[word_idx]
          return word
  
      def sample_word_from_distribution(self, distribution):
          ix = np.random.choice(range(self.words_vocab_size), p=distribution.ravel())
          x = np.zeros((self.words_vocab_size, 1))
          x[ix] = 1
          return self.idx_to_word[np.argmax(x)]
  
      def encode_characters(self, characters):
          word_len = len(characters)
          to_add = self.max_word_len - word_len
          characters_idx = [self.char_to_idx[i] for i in characters] + to_add * [self.char_to_idx['']]
          return characters_idx
  
      def decode_characters(self, characters_idx):
          characters = [self.idx_to_char[i] for i in characters_idx]
          return ''.join(characters)
  