text: # -*- coding: UTF-8 -*-
  """
  This module implements the core ML algorithm of gender classification.
  """
  from copy import copy
  import os
  import pickle
  from time import time
  
  import numpy as np
  import tensorflow as tf
  from sklearn.metrics import accuracy_score, classification_report
  from tensorflow.contrib.tensorboard.plugins import projector
  from tensorflow.python.layers.core import dropout
  from tensorflow.contrib.rnn import LSTMCell
  from tensorflow.python.client import timeline
  
  from ._batch import BatchGenerator
  from .constant import NEGATIVE_CLASS, NEUTRAL_CLASS, POSITIVE_CLASS, CLASS2DEFAULT_CUTOFF
  from ._encoder import CharEncoder
  from .util import get_logger, set_log_path as _set_log_path
  
  _TRAIN_PROFILE_FILE = 'profile_train.json'
  _VALID_PROFILE_FILE = 'profile_valid.json'
  _EMBEDDING_METADATA_FILE = 'metadata.tsv'
  
  _LOGGER = get_logger(__name__)
  
  __author__ = 'kensk8er'
  
  
  def set_log_path(log_path):
      """Set the log path of the logger in classifier module."""
      _set_log_path(_LOGGER, log_path)
  
  
  class CharLSTM(object):
      """Character-based language modeling using LSTM."""
      _padding_id = 0  # TODO: 0 is used for actual character as well, which is a bit confusing...
      _checkpoint_file_name = 'model.ckpt'
      _instance_file_name = 'instance.pkl'
      _tensorboard_dir = 'tensorboard.log'
  
      def __init__(self, embedding_size=32, char_rnn_size=128, word_rnn_size=128, learning_rate=0.001,
                   embedding_dropout=0., char_rnn_dropout=0., word_rnn_dropout=0.):
          # hyper-parameters
          self._embedding_size = embedding_size
          self._char_rnn_size = char_rnn_size
          self._word_rnn_size = word_rnn_size
          self._learning_rate = learning_rate
          self._embedding_dropout = embedding_dropout
          self._char_rnn_dropout = char_rnn_dropout
          self._word_rnn_dropout = word_rnn_dropout
  
          # other instance variables
          self._nodes = None
          self._graph = None
          self._vocab_size = None
          self._encoder = CharEncoder()
          self._num_params = None
          self._session = None
  
      def train(self, names_train, y_train, names_valid, y_valid, model_path, batch_size=128,
                patience=1024000, stat_interval=1000, valid_interval=1000, summary_interval=1000,
                valid_batch_size=2048, profile=False):
          """Train a gender classifier on the name/gender pairs."""
          start_time = time()
  
          def add_metric_summaries(mode, iteration, name2metric):
              """Add summary for metric."""
              metric_summary = tf.Summary()
              for name, metric in name2metric.items():
                  metric_summary.value.add(tag='{}_{}'.format(mode, name), simple_value=metric)
              summary_writer.add_summary(metric_summary, global_step=iteration)
  
          def show_train_stats(epoch, iteration, losses, y_cat, y_cat_pred):
              # compute mean statistics
              loss = np.mean(losses)
              accuracy = accuracy_score(y_cat, y_cat_pred)
              score = accuracy - loss
  
              _LOGGER.info('Epoch={}, Iter={:,}, Mean Training Loss={:.4f}, Accuracy={:.4f}, '
                           'Accuracy - Loss={:.4f}'.format(epoch, iteration, loss, accuracy, score))
              add_metric_summaries('train', iteration, {'cross_entropy': loss, 'accuracy': accuracy,
                                                        'accuracy - loss': score})
              _LOGGER.info('\n{}'.format(classification_report(y_cat, y_cat_pred, digits=3)))
              return list(), list(), list()
  
          def validate(epoch, iteration, X, y, best_score, patience):
              """Validate the model on validation set."""
              batch_generator = BatchGenerator(X, y, batch_size=valid_batch_size, valid=True)
              losses, y_cat, y_cat_pred = list(), list(), list()
              for X_batch, y_batch in batch_generator:
                  X_batch, word_lens, char_lens = self._add_padding(X_batch)
                  loss, y_pred = session.run(
                      [nodes['loss'], nodes['y_pred']],
                      feed_dict={nodes['X']: X_batch, nodes['y']: y_batch,
                                 nodes['word_lens']: word_lens, nodes['char_lens']: char_lens,
                                 nodes['is_train']: False},
                      options=run_options, run_metadata=run_metadata)
                  losses.append(loss)
                  y_cat.extend(self._categorize_y(y_batch))
                  y_cat_pred.extend(self._categorize_y(y_pred))
  
              # compute mean statistics
              loss = np.mean(losses)
              accuracy = accuracy_score(y_cat, y_cat_pred)
              score = accuracy - loss
  
              _LOGGER.info('Epoch={}, Iter={:,}, Validation Loss={:.4f}, Accuracy={:.4f}, '
                           'Accuracy - Loss={:.4f}'.format(epoch, iteration, loss, accuracy, score))
              add_metric_summaries('valid', iteration, {'cross_entropy': loss, 'accuracy': accuracy,
                                                        'accuracy - loss': score})
              _LOGGER.info('\n{}'.format(classification_report(y_cat, y_cat_pred, digits=3)))
  
              if score > best_score:
                  _LOGGER.info('Best score (Accuracy - Loss) so far, save the model.')
                  self._save(model_path, session)
                  best_score = score
  
                  if iteration * 2 > patience:
                      patience = iteration * 2
                      _LOGGER.info('Increased patience to {:,}'.format(patience))
  
              if run_metadata:
                  with open(_VALID_PROFILE_FILE, 'w') as file_:
                      file_.write(
                          timeline.Timeline(run_metadata.step_stats).generate_chrome_trace_format())
  
              return best_score, patience
  
          _LOGGER.info('Prepare inputs and other variables for the model...')
          self._fit_encoder(names_train + names_valid)
          X_train = self._encode_chars(names_train)
          X_valid = self._encode_chars(names_valid)
          train_size = len(X_train)
          train_batch_generator = BatchGenerator(X_train, y_train, batch_size)
          best_valid_score = np.float64('-inf')
          losses = list()
          y_cat = list()
          y_cat_pred = list()
          iteration = 0
  
          # profiler
          run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) if profile else None
          run_metadata = tf.RunMetadata() if profile else None
  
          _LOGGER.info('Building the tensorflow graph...')
          self._build_graph()
          nodes = self._nodes
          session = tf.Session(graph=self._graph)
          summary_writer = tf.summary.FileWriter(
              os.path.join(model_path, self._tensorboard_dir), session.graph)
          self._visualize_embedding(model_path, summary_writer)
          session.run(nodes['init'])
          _LOGGER.info('Start fitting a model...')
  
          # iterate over batches
          for batch_id, (X_batch, y_batch) in enumerate(train_batch_generator):
              epoch = 1 + iteration // train_size
  
              if batch_id % summary_interval == 0:
                  summaries = session.run(nodes['summaries'])
                  summary_writer.add_summary(summaries, global_step=iteration)
  
              X_batch, word_lens, char_lens = self._add_padding(X_batch)
  
              # Predict labels and update the parameters
              _, loss, y_pred = session.run(
                  [nodes['optimizer'], nodes['loss'], nodes['y_pred']],
                  feed_dict={nodes['X']: X_batch, nodes['y']: y_batch, nodes['word_lens']: word_lens,
                             nodes['char_lens']: char_lens, nodes['is_train']: True},
                  options=run_options, run_metadata=run_metadata)
  
              losses.append(loss)
              y_cat.extend(self._categorize_y(y_batch))
              y_cat_pred.extend(self._categorize_y(y_pred))
              iteration += batch_size
  
              if run_metadata:
                  with open(_TRAIN_PROFILE_FILE, 'w') as file_:
                      file_.write(
                          timeline.Timeline(run_metadata.step_stats).generate_chrome_trace_format())
  
              if batch_id % stat_interval == 0:
                  losses, y_cat, y_cat_pred = show_train_stats(
                      epoch, iteration, losses, y_cat, y_cat_pred)
  
              if batch_id % valid_interval == 0:
                  best_valid_score, patience = validate(
                      epoch, iteration, X_valid, y_valid, best_valid_score, patience)
  
              if iteration > patience:
                  _LOGGER.info('Iteration is more than patience, finish training.')
                  break
  
          _LOGGER.info('Finished fitting the model.')
          _LOGGER.info('Best Validation Score (Accuracy - Cross-entropy Loss): {:.4f}'
                       .format(best_valid_score))
  
          # close the session
          session.close()
  
          end_time = time()
          _LOGGER.info('Took {:,} seconds to train the model.'.format(int(end_time - start_time)))
          return best_valid_score
  
      @classmethod
      def load(cls, model_path):
          """
          Load the model from the saved model directory.
  
          :param model_path: path to the model directory you want to load the model from.
          :return: instance of the model
          """
          _LOGGER.debug('Started loading the model...')
          # load the instance, set _model_path appropriately
          with open(os.path.join(model_path, cls._instance_file_name), 'rb') as model_file:
              instance = pickle.load(model_file)
  
          # build the graph and restore the session
          instance._build_graph()
          instance._session = tf.Session(graph=instance._graph)
          instance._session.run(instance._nodes['init'])
          instance._nodes['saver'].restore(
              instance._session, os.path.join(model_path, instance._checkpoint_file_name))
  
          _LOGGER.debug('Finished loading the model.')
          return instance
  
      def predict(self, names: list, return_proba=True, return_attention=False,
                  low_cutoff=CLASS2DEFAULT_CUTOFF[NEGATIVE_CLASS],
                  high_cutoff=CLASS2DEFAULT_CUTOFF[POSITIVE_CLASS]):
          """
          Predict the genders of given names.
  
          :param names: list of names
          :param return_proba: output probability if set as True
          :param return_attention: if True, return attentions (weights for each time step)
          """
          nodes = self._nodes
          X = self._encode_chars(names)
          X, word_lens, char_lens = self._add_padding(X)
          y_pred, attentions = self._session.run(
              [nodes['y_pred'], nodes['attentions']],
              feed_dict={nodes['X']: X, nodes['word_lens']: word_lens, nodes['char_lens']: char_lens,
                         nodes['is_train']: False})
  
          # np.ndarray isn't returned when len(X) == 1
          if not isinstance(y_pred, np.ndarray):
              y_pred = [y_pred]
  
          if return_proba:
              # return_value = [{POSITIVE_CLASS: float(proba), NEGATIVE_CLASS: float(1 - proba)}
              #                 for proba in y_pred]
              return_value = [float(proba) for proba in y_pred]
          else:
              return_value = self._categorize_y(y_pred, low_cutoff, high_cutoff)
  
          if return_attention:
              return return_value, attentions.tolist()
          else:
              return return_value
  
      def _save(self, model_path, session):
          """Save the tensorflow session and the instance object of this Python class."""
          if not os.path.exists(model_path):
              os.makedirs(model_path)
  
          # save the session
          self._nodes['saver'].save(session, os.path.join(model_path, self._checkpoint_file_name))
  
          # save the instance
          instance = copy(self)
          instance._graph = None  # _graph is not picklable
          instance._nodes = None  # _nodes is not pciklable
          instance._session = None  # _session is not pciklable
          with open(os.path.join(model_path, self._instance_file_name), 'wb') as pickle_file:
              pickle.dump(instance, pickle_file)
  
      def _build_graph(self):
          """Build computational graph."""
  
          def get_num_params():
              """Count the number of trainable parameters."""
              num_params = 0
              for variable in tf.trainable_variables():
                  shape = variable.get_shape()
                  var_num_params = 1
                  for dimension in shape:
                      var_num_params *= dimension.value
                  num_params += var_num_params
              return num_params
  
          _LOGGER.debug('Building a computational graph...')
          graph = tf.Graph()
          nodes = dict()
  
          with graph.as_default():
              with tf.name_scope('inputs'):
                  # inputs
                  nodes['X'] = tf.placeholder(tf.int32, [None, None, None], name='X')
                  nodes['y'] = tf.placeholder(tf.float32, [None], name='y')
                  nodes['word_lens'] = tf.placeholder(tf.int32, [None], name='word_lens')
                  nodes['char_lens'] = tf.placeholder(tf.int32, [None], name='char_lens')
                  nodes['is_train'] = tf.placeholder(tf.bool, shape=[], name='is_train')
  
                  # get the shape of the input
                  X_shape = tf.shape(nodes['X'])
                  batch_size = X_shape[0]
                  max_word_len = X_shape[1]
                  max_char_len = X_shape[2]
  
              with tf.name_scope('embedding_layer'):
                  nodes['embeddings'] = tf.Variable(
                      tf.random_uniform([self._vocab_size, self._embedding_size], -1.0, 1.0),
                      trainable=True, name='embeddings')
                  embedded = tf.nn.embedding_lookup(nodes['embeddings'], nodes['X'])
                  embedded = dropout(
                      embedded, rate=self._embedding_dropout, training=nodes['is_train'])
  
              with tf.name_scope('char_rnn_layer') as scope:
                  # reshape the embedded matrix in order to pass it to dynamic_rnn
                  embedded = tf.reshape(
                      embedded, [batch_size * max_word_len, max_char_len, self._embedding_size])
  
                  char_rnn_fw_cell = LSTMCell(num_units=self._char_rnn_size)
                  char_rnn_bw_cell = LSTMCell(num_units=self._char_rnn_size)
                  (char_output_fw, char_output_bw), states = tf.nn.bidirectional_dynamic_rnn(
                      char_rnn_fw_cell, char_rnn_bw_cell, embedded, dtype=tf.float32,
                      sequence_length=nodes['char_lens'], scope='{}bidirectional_rnn'.format(scope))
  
                  char_rnn_outputs = tf.concat([char_output_fw, char_output_bw], axis=2)
  
                  with tf.name_scope('char_pooling_layer'):
                      char_rnn_outputs = self._mean_pool(
                          char_rnn_outputs, batch_size, max_char_len, max_word_len,
                          nodes['char_lens'])
  
                  char_rnn_outputs = dropout(
                      char_rnn_outputs, rate=self._char_rnn_dropout, training=nodes['is_train'])
  
              with tf.name_scope('word_rnn_layer') as scope:
                  word_rnn_fw_cell = LSTMCell(num_units=self._word_rnn_size)
                  word_rnn_bw_cell = LSTMCell(num_units=self._word_rnn_size)
                  (char_output_fw, char_output_bw), states = tf.nn.bidirectional_dynamic_rnn(
                      word_rnn_fw_cell, word_rnn_bw_cell, char_rnn_outputs, dtype=tf.float32,
                      sequence_length=nodes['word_lens'], scope='{}bidirectional_rnn'.format(scope))
                  word_rnn_outputs = tf.concat([char_output_fw, char_output_bw], axis=2)
  
                  with tf.name_scope('word_pooling_layer'):
                      word_rnn_outputs, nodes['attentions'] = self._attention_pool(word_rnn_outputs)
  
                  word_rnn_outputs = dropout(
                      word_rnn_outputs, rate=self._word_rnn_dropout, training=nodes['is_train'])
  
              with tf.variable_scope('softmax_layer'):
                  nodes['W_s'] = tf.Variable(
                      tf.random_normal([self._word_rnn_size * 2, 1]), name='weight')
                  nodes['b_s'] = tf.Variable(tf.random_normal([1]), name='bias')
                  logits = tf.squeeze(tf.matmul(word_rnn_outputs, nodes['W_s']) + nodes['b_s'])
                  nodes['y_pred'] = tf.nn.sigmoid(logits)
  
              with tf.variable_scope('optimizer'):
                  nodes['loss'] = tf.reduce_mean(
                      tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=nodes['y']))
                  nodes['optimizer'] = tf.train.AdamOptimizer(self._learning_rate).minimize(
                      nodes['loss'])
  
              # initialize the variables
              nodes['init'] = tf.global_variables_initializer()
  
              # count the number of parameters
              self._num_params = get_num_params()
              _LOGGER.debug('Total number of parameters = {:,}'.format(self._num_params))
  
              # generate summaries
              for variable in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES):
                  # having ":" in the name is illegal, so replace to "/"
                  tf.summary.histogram(variable.name.replace(':', '/'), variable)
              nodes['summaries'] = tf.summary.merge_all()
  
              # save the model to checkpoint
              nodes['saver'] = tf.train.Saver()
  
          self._graph = graph
          self._nodes = nodes
  
      def _add_padding(self, X):
          """
          Add padding to X in order to align the sequence lengths.
  
          :param X: list (each name) of list (each word) of character IDs
          :return: padded list of list of character IDs & list of word length before padding & list of
              character length before padding
          """
  
          def get_max(X):
              """Compute the maximum word length and maximum character length."""
              max_word_len, max_char_len = 0, 0
              for name in X:
                  if max_word_len < len(name):
                      max_word_len = len(name)
                  for word in name:
                      if max_char_len < len(word):
                          max_char_len = len(word)
              return max_word_len, max_char_len
  
          max_word_len, max_char_len = get_max(X)
          word_lens = list()
          char_lens = list()
  
          for name in X:
              word_lens.append(len(name))
              word_pad_len = max_word_len - len(name)
              name.extend([[] for _ in range(word_pad_len)])
  
              for word in name:
                  char_lens.append(len(word))
                  char_pad_len = max_char_len - len(word)
                  word.extend([self._padding_id for _ in range(char_pad_len)])
  
          return X, word_lens, char_lens
  
      def _encode_chars(self, names, fit=False):
          """Encode list of names into list (each name) of list (each word) of character IDs."""
          if fit:
              name_id2word_id2char_ids = self._encoder.fit_encode(names)
              self._vocab_size = self._encoder.vocab_size
          else:
              name_id2word_id2char_ids = self._encoder.encode(names)
          return name_id2word_id2char_ids
  
      def _fit_encoder(self, names):
          """Fit the encoder to the given list of names."""
          self._encoder.fit(names)
          self._vocab_size = self._encoder.vocab_size
  
      def _decode_chars(self, name_id2word_id2char_ids):
          """Decode list (each name) of list (each word) of encoded character IDs into characters."""
          return self._encoder.decode(name_id2word_id2char_ids)
  
      @staticmethod
      def _categorize_y(y, low_cutoff=CLASS2DEFAULT_CUTOFF[NEGATIVE_CLASS],
                        high_cutoff=CLASS2DEFAULT_CUTOFF[POSITIVE_CLASS]):
          """Categorize a list of continuous values y into a list of male/neutral/female labels."""
  
          def categorize_label(label):
              if label < low_cutoff:
                  return NEGATIVE_CLASS
              elif label <= high_cutoff:
                  return NEUTRAL_CLASS
              else:
                  return POSITIVE_CLASS
  
          return [categorize_label(label) for label in y]
  
      def _mean_pool(self, rnn_outputs, batch_size, max_char_len, max_word_len, char_lens):
          """
          Perform mean-pooling after the character-RNN layer.
  
          :param rnn_outputs: hidden states of all the time steps after the character-RNN layer
          :return: mean of the hidden states over every time step
          """
          # perform mean pooling over characters
          rnn_outputs = tf.reduce_mean(rnn_outputs, reduction_indices=1)
  
          # In order to avoid 0 padding affect the mean, multiply by `n / m` where `n` is
          # `max_char_len` and `m` is `char_lens`
          rnn_outputs = tf.multiply(rnn_outputs, tf.cast(max_char_len, tf.float32))  # multiply by `n`
  
          # swap the dimensions in order to divide by an appropriate value for each time step
          rnn_outputs = tf.transpose(rnn_outputs)
  
          rnn_outputs = tf.divide(rnn_outputs, tf.cast(char_lens, tf.float32))  # divide by `m`
          rnn_outputs = tf.transpose(rnn_outputs)  # shape back to the original shape
  
          # batch and word-len dimensions were merged before running character-RNN so shape it back
          rnn_outputs = tf.reshape(rnn_outputs, [batch_size, max_word_len, self._char_rnn_size * 2])
  
          # there are NaN due to padded words (with char_len=0) so convert those NaN to 0
          rnn_outputs = tf.where(tf.is_nan(rnn_outputs), tf.zeros_like(rnn_outputs), rnn_outputs)
  
          return rnn_outputs
  
      def _attention_pool(self, rnn_outputs):
          """
          Perform attention-pooling. Train an attention layer to soft search on hidden states to use
          and return weighted sum of the hidden states.
  
          :param rnn_outputs: hidden states of all the time steps after the word-RNN layer
          :return: weighted sum of the hidden states and attention weights for each time step
          """
          W = tf.Variable(tf.random_normal([2 * self._word_rnn_size]), name='weight_attention')
          b = tf.Variable(tf.random_normal([1]), name='bias_attention')
  
          # shape: batch_size * word_len
          attentions = tf.reduce_sum(tf.multiply(W, rnn_outputs), reduction_indices=2) + b
          attentions = tf.nn.softmax(attentions)  # convert to probability
  
          # swap the dimensions in order to multiply by attentions to each word (the 2nd dimension)
          rnn_outputs = tf.transpose(rnn_outputs, perm=[0, 2, 1])
  
          # expand the dimension in order to multiply outputs by attentions
          attentions = tf.expand_dims(attentions, axis=1)
  
          rnn_outputs = tf.multiply(attentions, rnn_outputs)
          rnn_outputs = tf.transpose(rnn_outputs, perm=[0, 2, 1])  # shape back to the original shape
  
          # pool hidden states of multiple words (after applying attention) into one hidden states
          rnn_outputs = tf.reduce_sum(rnn_outputs, reduction_indices=1)
  
          return rnn_outputs, tf.squeeze(attentions, axis=1)
  
      def _visualize_embedding(self, model_path, summary_writer):
          """Create metadata file (and its config file) for tensorboard's embedding visualization."""
          metadata_path = os.path.join(model_path, self._tensorboard_dir, _EMBEDDING_METADATA_FILE)
  
          # create the metadata config file
          config = projector.ProjectorConfig()
          embedding = config.embeddings.add()
          embedding.tensor_name = self._nodes['embeddings'].name
          embedding.metadata_path = metadata_path
          projector.visualize_embeddings(summary_writer, config)
  
          # create metadata file
          with open(metadata_path, 'w', encoding='utf8') as metadata_file:
              metadata_file.write('Character\tID\n')
              for id_, char in enumerate(self._encoder.chars):
                  metadata_file.write('{}\t{}\n'.format(char, id_))
  