import tensorflow as tf

            for i in range(hparams.tacotron_num_gpus):
                tf.summary.histogram("mel_outputs %d" % i, model.tower_linear_outputs[i])
                tf.summary.histogram("mel_targets %d" % i, model.tower_linear_targets[i])

        tf.summary.scalar("regularization_loss", model.regularization_loss)
        tf.summary.scalar("stop_token_loss", model.stop_token_loss)
        tf.summary.scalar("loss", model.loss)
        tf.summary.scalar("learning_rate", model.learning_rate)  # Control learning rate decay speed
        if hparams.tacotron_teacher_forcing_mode == "scheduled":
            tf.summary.scalar("teacher_forcing_ratio", model.ratio)  # Control teacher forcing
        # ratio decay when mode = "scheduled"
        gradient_norms = [tf.norm(grad) for grad in model.gradients]
        tf.summary.histogram("gradient_norm", gradient_norms)
        tf.summary.scalar("max_gradient_norm", tf.reduce_max(gradient_norms))  # visualize
        # gradients (in case of explosion)
        return tf.summary.merge_all()


def add_eval_stats(summary_writer, step, linear_loss, before_loss, after_loss, stop_token_loss,
                   loss):
    values = [
        tf.Summary.Value(tag="Tacotron_eval_model/eval_stats/eval_before_loss",
                         simple_value=before_loss),
        tf.Summary.Value(tag="Tacotron_eval_model/eval_stats/eval_after_loss",
