'Calculate log-probs and other calculations on batch of episodes.'

def multi_step(self, all_obs, initial_state, all_actions):

    batch_size = tf.shape(initial_state)[0]
    time_length = tf.shape(all_obs[0])[0]
    initial_actions = [act[0] for act in all_actions]
    all_actions = [tf.concat([act[1:], act[0:1]], 0) for act in all_actions]
    (internal_states, _, logits, log_probs, entropies, self_kls) = tf.scan(self.single_step, (all_obs, all_actions), initializer=self.get_initializer(batch_size, initial_state, initial_actions))
    log_probs = [log_prob[:(-1)] for log_prob in log_probs]
    entropies = [entropy[:(-1)] for entropy in entropies]
    self_kls = [self_kl[:(-1)] for self_kl in self_kls]
    return (internal_states, logits, log_probs, entropies, self_kls)
