docstr: Calculates the expected PPO loss (components) given Policy,
  Model, distribution, some batch, logits & vf outputs, using numpy.
docstr_tok: ["Calculates","the","expected","PPO","loss","(","components",")","given","Policy",",","Model",",","distribution",",","some","batch",",","logits","&","vf","outputs",",","using","numpy","."]
code: def _ppo_loss_helper(
      self,
      policy,
      model,
      dist_class,
      train_batch,
      logits,
      vf_outs,
      sess=None,
  ):
      """
      Calculates the expected PPO loss (components) given Policy,
      Model, distribution, some batch, logits & vf outputs, using numpy.
      """
      # Calculate expected PPO loss results.
      dist = dist_class(logits, policy.model)
      dist_prev = dist_class(
          train_batch[SampleBatch.ACTION_DIST_INPUTS], policy.model
      )
      expected_logp = dist.logp(train_batch[SampleBatch.ACTIONS])
      if isinstance(model, TorchModelV2):
          train_batch.set_get_interceptor(None)
          expected_rho = np.exp(
              expected_logp.detach().cpu().numpy()
              - train_batch[SampleBatch.ACTION_LOGP]
          )
          # KL(prev vs current action dist)-loss component.
          kl = np.mean(dist_prev.kl(dist).detach().cpu().numpy())
          # Entropy-loss component.
          entropy = np.mean(dist.entropy().detach().cpu().numpy())
      else:
          if sess:
              expected_logp = sess.run(expected_logp)
          expected_rho = np.exp(
              expected_logp - train_batch[SampleBatch.ACTION_LOGP]
          )
          # KL(prev vs current action dist)-loss component.
          kl = dist_prev.kl(dist)
          if sess:
              kl = sess.run(kl)
          kl = np.mean(kl)
          # Entropy-loss component.
          entropy = dist.entropy()
          if sess:
              entropy = sess.run(entropy)
          entropy = np.mean(entropy)
  
      # Policy loss component.
      pg_loss = np.minimum(
          train_batch[Postprocessing.ADVANTAGES] * expected_rho,
          train_batch[Postprocessing.ADVANTAGES]
          * np.clip(
              expected_rho,
              1 - policy.config["clip_param"],
              1 + policy.config["clip_param"],
          ),
      )
  
      # Value function loss component.
      vf_loss1 = np.power(
          vf_outs - train_batch[Postprocessing.VALUE_TARGETS], 2.0
      )
      vf_clipped = train_batch[SampleBatch.VF_PREDS] + np.clip(
          vf_outs - train_batch[SampleBatch.VF_PREDS],
          -policy.config["vf_clip_param"],
          policy.config["vf_clip_param"],
      )
      vf_loss2 = np.power(
          vf_clipped - train_batch[Postprocessing.VALUE_TARGETS], 2.0
      )
      vf_loss = np.maximum(vf_loss1, vf_loss2)
  
      # Overall loss.
      if sess:
          policy_sess = policy.get_session()
          kl_coeff, entropy_coeff = policy_sess.run(
              [policy.kl_coeff, policy.entropy_coeff]
          )
      else:
          kl_coeff, entropy_coeff = (
              policy.kl_coeff,
              policy.entropy_coeff,
          )
      overall_loss = np.mean(
          -pg_loss
          + kl_coeff * kl
          + policy.config["vf_loss_coeff"] * vf_loss
          - entropy_coeff * entropy
      )
      return kl, entropy, pg_loss, vf_loss, overall_loss
  
imports: import unittest
  import numpy
  import ray
  from ray.rllib.algorithms.callbacks import DefaultCallbacks
  import ray.rllib.algorithms.ppo
  from ray.rllib.algorithms.ppo.ppo_tf_policy import PPOTF2Policy
  from ray.rllib.algorithms.ppo.ppo_torch_policy import PPOTorchPolicy
  from ray.rllib.evaluation.postprocessing import compute_gae_for_sample_batch, Postprocessing
  from ray.rllib.models.tf.tf_action_dist import Categorical
  from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
  from ray.rllib.models.torch.torch_action_dist import TorchCategorical
  from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID, SampleBatch
  from ray.rllib.utils.metrics.learner_info import LEARNER_INFO, LEARNER_STATS_KEY
  from ray.rllib.utils.numpy import fc
  from ray.rllib.utils.test_utils import check, check_compute_single_action, check_off_policyness, check_train_results, framework_iterator, check_inference_w_connectors
label_desc: Use image node_id 8 for calling the TestPPO obj's underlying member method code with example usage: obj._ppo_loss_helper(policy, model, dist_class, train_batch, logits, vf_outs, sess) and returns: kl,  entropy,  pg_loss,  vf_loss,  overall_loss