'Builds the reinforcement learning tensorflow graph.'

def build_graph(self):

    tf.logging.info('Adding    reward    computation    portion    of    the    graph')
    with tf.name_scope('reward_computation'):
        self.reward_scores = tf.identity(self.reward_rnn(), name='reward_scores')
    tf.logging.info('Adding    taking    action    portion    of    graph')
    with tf.name_scope('taking_action'):
        self.action_scores = tf.identity(self.q_network(), name='action_scores')
        tf.summary.histogram('action_scores', self.action_scores)
        if (self.algorithm == 'g'):
            self.g_action_scores = (self.action_scores + self.reward_scores)
            self.action_softmax = tf.nn.softmax(self.g_action_scores, name='action_softmax')
            self.predicted_actions = tf.one_hot(tf.argmax(self.g_action_scores, dimension=1, name='predicted_actions'), self.num_actions)
        else:
            self.action_softmax = tf.nn.softmax(self.action_scores, name='action_softmax')
            self.predicted_actions = tf.one_hot(tf.argmax(self.action_scores, dimension=1, name='predicted_actions'), self.num_actions)
    tf.logging.info('Add    estimating    future    rewards    portion    of    graph')
    with tf.name_scope('estimating_future_rewards'):
        self.next_action_scores = tf.stop_gradient(self.target_q_network())
        tf.summary.histogram('target_action_scores', self.next_action_scores)
        self.rewards = tf.placeholder(tf.float32, (None,), name='rewards')
        if (self.algorithm == 'psi'):
            self.target_vals = tf.reduce_logsumexp(self.next_action_scores, reduction_indices=[1])
        elif (self.algorithm == 'g'):
            self.g_normalizer = tf.reduce_logsumexp(self.reward_scores, reduction_indices=[1])
            self.g_normalizer = tf.reshape(self.g_normalizer, [(-1), 1])
            self.g_normalizer = tf.tile(self.g_normalizer, [1, self.num_actions])
            self.g_action_scores = tf.subtract((self.next_action_scores + self.reward_scores), self.g_normalizer)
            self.target_vals = tf.reduce_logsumexp(self.g_action_scores, reduction_indices=[1])
        else:
            self.target_vals = tf.reduce_max(self.next_action_scores, reduction_indices=[1])
        self.future_rewards = (self.rewards + (self.discount_rate * self.target_vals))
    tf.logging.info('Adding    q    value    prediction    portion    of    graph')
    with tf.name_scope('q_value_prediction'):
        self.action_mask = tf.placeholder(tf.float32, (None, self.num_actions), name='action_mask')
        self.masked_action_scores = tf.reduce_sum((self.action_scores * self.action_mask), reduction_indices=[1])
        temp_diff = (self.masked_action_scores - self.future_rewards)
        self.prediction_error = tf.reduce_mean(tf.square(temp_diff))
        self.params = tf.trainable_variables()
        self.gradients = self.optimizer.compute_gradients(self.prediction_error)
        for (i, (grad, var)) in enumerate(self.gradients):
            if (grad is not None):
                self.gradients[i] = (tf.clip_by_norm(grad, 5), var)
        for (grad, var) in self.gradients:
            tf.summary.histogram(var.name, var)
            if (grad is not None):
                tf.summary.histogram((var.name + '/gradients'), grad)
        self.train_op = self.optimizer.apply_gradients(self.gradients)
    tf.logging.info('Adding    target    network    update    portion    of    graph')
    with tf.name_scope('target_network_update'):
        self.target_network_update = []
        for (v_source, v_target) in zip(self.q_network.variables(), self.target_q_network.variables()):
            update_op = v_target.assign_sub((self.target_network_update_rate * (v_target - v_source)))
            self.target_network_update.append(update_op)
        self.target_network_update = tf.group(*self.target_network_update)
    tf.summary.scalar('prediction_error', self.prediction_error)
    self.summarize = tf.summary.merge_all()
    self.no_op1 = tf.no_op()
