def calc_policy_loss(self, model, policy_params, policy_entropy, rollout):
        """
        Policy gradient loss - calculate from probability distribution

        Calculate surrogate loss - advantage * policy_probability / fixed_initial_policy_probability

        Because we operate with logarithm of -probability (neglogp) we do
        - advantage * exp(fixed_neglogps - model_neglogps)
        """
        actions = rollout.batch_tensor('actions')
        advantages = rollout.batch_tensor('advantages')
        fixed_logprobs = rollout.batch_tensor('action:logprobs')

        model_logprobs = model.logprob(actions, policy_params)

        # Normalize advantages
        advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)

        # We put - in front because we want to maximize the surrogate objective
        policy_loss = -advantages * torch.exp(model_logprobs - fixed_logprobs)

        return policy_loss.mean() - policy_entropy * self.entropy_coef
Policy gradient loss - calculate from probability distribution

        Calculate surrogate loss - advantage * policy_probability / fixed_initial_policy_probability

        Because we operate with logarithm of -probability (neglogp) we do
        - advantage * exp(fixed_neglogps - model_neglogps)