import tensorflow as tf

            self.pi = tf.nn.softmax(logits)
            self.log_pi = tf.nn.log_softmax(logits)
            self.sample = policy_utils.categorical_sample(
                tf.reshape(logits, [-1, num_acts]), num_acts)[0, :]

    def build_value(self, _input):
        with tf.variable_scope('VF'):
            hidden = tf.layers.dense(inputs=_input,
                                     units=self.vf_hidden_size,
                                     activation=tf.nn.elu)

            w = tf.get_variable("weights", (self.vf_hidden_size, 1))
            return tf.matmul(hidden, w)

    def build_loss(self):
        cutoff_vf_manager = tf.reshape(tf.stop_gradient(self.manager_vf), [-1])
        dot = tf.reduce_sum(tf.multiply(self.s_diff, self.g), axis=1)
        gcut = tf.stop_gradient(self.g)
        mag = tf.norm(self.s_diff, axis=1) * tf.norm(gcut, axis=1) + .0001
        dcos = dot / mag
        manager_loss = -tf.reduce_sum((self.r - cutoff_vf_manager) * dcos)

