text: """
  Module to do callbacks for Keras models.
  """
  from __future__ import division
  
  from keras.callbacks import Callback  # ModelCheckpoint , EarlyStopping
  
  import matplotlib.pyplot as plt
  import h5py
  import itertools
  import logging
  import numpy as np
  import os
  import subprocess
  import shutil
  import codecs
  import sys
  from time import gmtime, strftime
  import math
  import time
  from copy import deepcopy
  
  # Set up logger
  logging.basicConfig(level=logging.INFO, stream=sys.stdout)
  #logging.basicConfig(level=logging.DEBUG)
  logger = logging.getLogger(__name__)
  
  # Dimensionality of image feature vector
  IMG_FEATS = 4096
  HSN_SIZE = 409
  MULTEVAL_DIR = '../multeval-0.5.1' if "util" in os.getcwd() else "multeval-0.5.1"
  
  
  class cd:
      """Context manager for changing the current working directory"""
      """http://stackoverflow.com/questions/431684/how-do-i-cd-in-python"""
      def __init__(self, newPath):
          self.newPath = newPath
  
      def __enter__(self):
          self.savedPath = os.getcwd()
          os.chdir(self.newPath)
  
      def __exit__(self, etype, value, traceback):
          os.chdir(self.savedPath)
  
  class CompilationOfCallbacks(Callback):
      """ Collection of compiled callbacks."""
  
      def __init__(self, word2index, index2word, argsDict, dataset,
                   data_generator, use_sourcelang=False, use_image=True):
          super(Callback, self).__init__()
  
          self.verbose = True
          self.filename = "weights.hdf5"
          self.save_best_only = True
  
          self.val_loss = []
          self.best_val_loss = np.inf
  
          self.val_metric = []
          self.best_val_metric = np.NINF
  
          self.word2index = word2index
          self.index2word = index2word
          self.args = argsDict
  
          # used to control early stopping on the validation data
          self.wait = 0
          self.patience = self.args.patience
  
          # needed by model.predict in generate_sentences
          self.use_sourcelang = use_sourcelang
          self.use_image = use_image
  
          # controversial assignment but it makes it much easier to
          # do early stopping based on metrics
          self.data_generator = data_generator
  
          # this results in two file handlers for dataset (here and
          # data_generator)
          if not dataset:
              logger.warn("No dataset given, using flickr8k")
              self.dataset = h5py.File("flickr8k/dataset.h5", "r")
          else:
              self.dataset = h5py.File("%s/dataset.h5" % dataset, "r")
          if self.args.source_vectors is not None:
              self.source_dataset = h5py.File("%s/dataset.h5" % self.args.source_vectors, "r")
  
      def on_epoch_end(self, epoch, logs={}):
          '''
          At the end of each epoch we
            1. create a directory to checkpoint data
            2. save the arguments used to initialise the run
            3. generate N sentences in the val data by sampling from the model
            4. calculate metric score of the generated sentences
            5. determine whether to stop training and sys.exit(0)
            6. save the model parameters using BLEU
          '''
          savetime = strftime("%d%m%Y-%H%M%S", gmtime())
          path = self.create_checkpoint_directory(savetime)
          self.save_run_arguments(path)
  
          # Generate training and val sentences to check for overfitting
          self.generate_sentences(path)
          meteor, bleu, ter = self.multeval_scores(path)
          val_loss = logs.get('val_loss')
  
          self.early_stop_decision(len(self.val_metric)+1, meteor, val_loss)
          self.checkpoint_parameters(epoch, logs, path, meteor, val_loss)
          self.log_performance()
  
      def early_stop_decision(self, epoch, val_metric, val_loss):
          '''
  	Stop training if validation loss has stopped decreasing and
  	validation BLEU score has not increased for --patience epochs.
  
          WARNING: quits with sys.exit(0).
  
  	TODO: this doesn't yet support early stopping based on TER
          '''
  
  	if val_loss < self.best_val_loss:
  	    self.wait = 0
          elif val_metric > self.best_val_metric or self.args.no_early_stopping:
              self.wait = 0
          else:
              self.wait += 1
              if self.wait >= self.patience:
                  # we have exceeded patience
                  if val_loss > self.best_val_loss:
                      # and loss is no longer decreasing
                      logger.info("Epoch %d: early stopping", epoch)
                      handle = open("checkpoints/%s/summary"
                                    % self.args.run_string, "a")
                      handle.write("Early stopping because patience exceeded\n")
                      best_bleu = np.nanargmax(self.val_metric)
                      best_loss = np.nanargmin(self.val_loss)
                      logger.info("Best Metric: %d | val loss %.5f score %.2f",
                                  best_bleu+1, self.val_loss[best_bleu],
                                  self.val_metric[best_bleu])
                      logger.info("Best loss: %d | val loss %.5f score %.2f",
                                  best_loss+1, self.val_loss[best_loss],
                                  self.val_metric[best_loss])
                      handle.close()
                      sys.exit(0)
  
      def log_performance(self):
          '''
          Record model performance so far, based on validation loss.
          '''
          handle = open("checkpoints/%s/summary" % self.args.run_string, "w")
  
          for epoch in range(len(self.val_loss)):
              handle.write("Checkpoint %d | val loss: %.5f bleu %.2f\n"
                           % (epoch+1, self.val_loss[epoch],
                              self.val_metric[epoch]))
  
          logger.info("---")  # break up the presentation for clarity
  
          # BLEU is the quickest indicator of performance for our task
          # but loss is our objective function
          best_bleu = np.nanargmax(self.val_metric)
          best_loss = np.nanargmin(self.val_loss)
          logger.info("Best Metric: %d | val loss %.5f score %.2f",
                      best_bleu+1, self.val_loss[best_bleu],
                      self.val_metric[best_bleu])
          handle.write("Best Metric: %d | val loss %.5f score %.2f\n"
                       % (best_bleu+1, self.val_loss[best_bleu],
                          self.val_metric[best_bleu]))
          logger.info("Best loss: %d | val loss %.5f score %.2f",
                      best_loss+1, self.val_loss[best_loss],
                      self.val_metric[best_loss])
          handle.write("Best loss: %d | val loss %.5f score %.2f\n"
                       % (best_loss+1, self.val_loss[best_loss],
                          self.val_metric[best_loss]))
          logger.info("Early stopping marker: wait/patience: %d/%d\n",
                      self.wait, self.patience)
          handle.write("Early stopping marker: wait/patience: %d/%d\n" %
                       (self.wait, self.patience))
          handle.close()
  
      def extract_references(self, directory, split):
          """
          Get reference descriptions for val or test data.
          """
          references = self.data_generator.get_refs_by_split_as_list(split)
  
          for refid in xrange(len(references[0])):
              codecs.open('%s/%s_reference.ref%d' % (directory, split, refid),
                          'w', 'utf-8').write('\n'.join([x[refid] for x in references]))
                          #'w', 'utf-8').write('\n'.join(['\n'.join(x) for x in references]))
          return references
  
      def __bleu_score__(self, directory, val=True):
          '''
          Loss is only weakly correlated with improvements in BLEU,
          and thus improvements in human judgements. Let's also track
          BLEU score of a subset of generated sentences in the val split
          to decide on early stopping, etc.
          '''
  
          prefix = "val" if val else "test"
  
          self.extract_references(directory, split=prefix)
  
          subprocess.check_call(
              ['perl multi-bleu.perl %s/%s_reference.ref < %s/%sGenerated > %s/%sBLEU'
               % (directory, prefix, directory, prefix, directory, prefix)],
              shell=True)
          bleudata = open("%s/%sBLEU" % (directory, prefix)).readline()
          data = bleudata.split(",")[0]
          bleuscore = data.split("=")[1]
          bleu = float(bleuscore.lstrip())
          return bleu
  
      def multeval_scores(self, directory, val=True):
          '''
          Maybe you want to do early stopping using Meteor, TER, or BLEU?
          '''
          prefix = "val" if val else "test"
          self.extract_references(directory, prefix)
  
          # First you want re-compound the split German words
          if self.args.meteor_lang == 'de':
              subprocess.check_call(
                  ["cp %s/%sGenerated %s/%sGenerated.orig" % (directory, prefix,
                      directory, prefix)], shell=True)
              subprocess.check_call(
                  ["sed -i -r 's/ @(.*?)@ //g' %s/%sGenerated" % (directory, prefix)], shell=True)
              subprocess.check_call(
                  ["sed -i -r 's/ @(.*?)@ //g' %s/%s_reference.*" % (directory, prefix)], shell=True)
  
          with cd(MULTEVAL_DIR):
              subprocess.check_call(
                  ['./multeval.sh eval --refs ../%s/%s_reference.* \
                   --hyps-baseline ../%s/%sGenerated \
                   --meteor.language %s \
                   --threads 1 \
                   2> %s-multevaloutput 1> %s-multevaloutput'
                  % (directory, prefix, directory, prefix,
                      self.args.meteor_lang, self.args.run_string,
                      self.args.run_string)], shell=True)
              handle = open("%s-multevaloutput" % self.args.run_string)
              multdata = handle.readlines()
              handle.close()
              for line in multdata:
                if line.startswith("RESULT: baseline: BLEU: AVG:"):
                  mbleu = line.split(":")[4]
                  mbleu = mbleu.replace("\n","")
                  mbleu = mbleu.strip()
                  lr = mbleu.split(".")
                  mbleu = float(lr[0]+"."+lr[1][0:2])
                if line.startswith("RESULT: baseline: METEOR: AVG:"):
                  mmeteor = line.split(":")[4]
                  mmeteor = mmeteor.replace("\n","")
                  mmeteor = mmeteor.strip()
                  lr = mmeteor.split(".")
                  mmeteor = float(lr[0]+"."+lr[1][0:2])
                if line.startswith("RESULT: baseline: TER: AVG:"):
                  mter = line.split(":")[4]
                  mter = mter.replace("\n","")
                  mter = mter.strip()
                  lr = mter.split(".")
                  mter = float(lr[0]+"."+lr[1][0:2])
  
              logger.info("Meteor = %.2f | BLEU = %.2f | TER = %.2f", 
  			mmeteor, mbleu, mter)
  
              return mmeteor, mbleu, mter
  
      def create_checkpoint_directory(self, savetime):
          '''
          We will create one directory to store all of the epochs data inside.
          The name is based on the run_string (if provided) or the current time.
          '''
  
          prefix = self.args.run_string if self.args.run_string != "" else ""
          number = "%03d" % (len(self.val_metric) + 1)
          filepath = "checkpoints/%s/%s-%s" % ((prefix, number, savetime))
          try:
              os.mkdir("checkpoints/%s/" % (prefix))
              shutil.copyfile("train.py", "checkpoints/%s/train.py" % prefix)
              shutil.copyfile("models.py", "checkpoints/%s/models.py" % prefix)
          except OSError:
              pass  # directory already exists
          try:
              os.mkdir(filepath)
          except OSError:
              pass  # directory already exists
          logger.info("\nIn %s ...",filepath)
          return filepath
  
      def save_run_arguments(self, filepath):
          '''
          Save the command-line arguments, along with the method defaults,
          used to parameterise this run.
          '''
          handle = open("%s/argparse.args" % filepath, "w")
          for arg, value in self.args.__dict__.iteritems():
              handle.write("%s: %s\n" % (arg, str(value)))
          handle.close()
  
      def checkpoint_parameters(self, epoch, logs, filepath, cur_val_metric,
                                cur_val_loss=0.):
          '''
          We checkpoint the model parameters based on either PPLX reduction or
          metric score increase in the validation data. This is driven by the
          user-specified argument self.args.stopping_loss.
  
  	TODO: this doesn't yet support early stopping based on TER
          '''
  
          weights_path = "%s/weights.hdf5" % filepath
  
          self.val_loss.append(cur_val_loss)
          if cur_val_loss < self.best_val_loss:
              self.best_val_loss = cur_val_loss
  
          # save the weights anyway for debug purposes
          self.model.save_weights(weights_path, overwrite=True)
  
          # update the best values, if applicable
          self.val_metric.append(cur_val_metric)
          if cur_val_metric > self.best_val_metric:
              self.best_val_metric = cur_val_metric
  
          optimiser_params = open("%s/optimiser_params" % filepath, "w")
          for key, value in self.model.optimizer.get_config().items():
              optimiser_params.write("%s: %s\n" % (key, value))
          optimiser_params.close()
  
      def reset_text_arrays(self, text_arrays, fixed_words=1):
          """ Reset the values in the text data structure to zero so we cannot
          accidentally pass them into the model """
          reset_arrays = deepcopy(text_arrays)
          reset_arrays[:,fixed_words:, :] = 0
          return reset_arrays
  
      def generate_sentences(self, filepath, val=True):
          """
          Generates descriptions of images for --generation_timesteps
          iterations through the LSTM. Each input description is clipped to
          the first <BOS> token, or, if --generate_from_N_words is set, to the
          first N following words (N + 1 BOS token).
          This process can be additionally conditioned
          on source language hidden representations, if provided by the
          --source_vectors parameter.
          The output is clipped to the first EOS generated, if it exists.
  
          TODO: duplicated method with generate.py
          """
          prefix = "val" if val else "test"
          logger.info("Generating %s descriptions", prefix)
          start_gen = self.args.generate_from_N_words + 1  # include BOS
          handle = codecs.open("%s/%sGenerated" % (filepath, prefix), 
                               "w", 'utf-8')
  
          val_generator = self.data_generator.generation_generator(prefix,
                                                                   in_callbacks=True)
          seen = 0
          for data in val_generator:
              inputs = data[0]
              text = deepcopy(inputs['text'])
              # Append the first start_gen words to the complete_sentences list
              # for each instance in the batch.
              complete_sentences = [[] for _ in range(text.shape[0])]
              for t in range(start_gen):  # minimum 1
                  for i in range(text.shape[0]):
                      w = np.argmax(text[i, t])
                      complete_sentences[i].append(self.index2word[w])
              del inputs['text']
              text = self.reset_text_arrays(text, start_gen)
              Y_target = data[1]
              inputs['text'] = text
  
              for t in range(start_gen, self.args.generation_timesteps):
                  logger.debug("Input token: %s" % self.index2word[np.argmax(inputs['text'][0,t-1])])
                  preds = self.model.predict(inputs, verbose=0)
  
                  # Look at the last indices for the words.
                  #next_word_indices = np.argmax(preds['output'][:, t-1], axis=1)
                  next_word_indices = np.argmax(preds[:, t-1], axis=1)
                  logger.debug("Predicted token: %s" % self.index2word[next_word_indices[0]])
                  # update array[0]/sentence-so-far with generated words.
                  for i in range(len(next_word_indices)):
                      inputs['text'][i, t, next_word_indices[i]] = 1.
                  next_words = [self.index2word[x] for x in next_word_indices]
                  for i in range(len(next_words)):
                      complete_sentences[i].append(next_words[i])
  
              sys.stdout.flush()
              # print/extract each sentence until it hits the first end-of-string token
              for s in complete_sentences:
                  decoded_str = ' '.join([x for x
                                          in itertools.takewhile(
                                              lambda n: n != "<E>", s[1:])])
                  handle.write(decoded_str + "\n")
  
              seen += text.shape[0]
              if seen >= self.data_generator.split_sizes['val']:
                  # Hacky way to break out of the generator
                  break
          handle.close()
  