class_docstr: Proximal Policy Optimization (PPO). See https://arxiv.org/pdf/1707.06347.pdf
  This is the "clip" version of PPO. It does not include:
  - KL divergence
  - Bootstrapping with a critic model (our approach only works if full trajectories up to terminal state are fed in)
  Optionally, a value network can be trained and used as a baseline for rewards.
docstr: Get a dict of losses for the trajectory. Dict always includes PPO loss.
  If a value baseline is trained, a loss for the value network is also included.
docstr_tok: ["Get","a","dict","of","losses","for","the","trajectory",".","Dict","always","includes","PPO","loss",".","If","a","value","baseline","is","trained",",","a","loss","for","the","value","network","is","also","included","."]
code: def _trajectory_to_losses(
      self, trajectory: rlt.PolicyGradientInput
  ) -> Dict[str, torch.Tensor]:
      """
      Get a dict of losses for the trajectory. Dict always includes PPO loss.
      If a value baseline is trained, a loss for the value network is also included.
      """
      losses = {}
      actions = trajectory.action
      rewards = trajectory.reward.detach()
      scorer_inputs = []
      if inspect.getattr_static(trajectory, "graph", None) is not None:
          # TODO: can this line be hit currently in ReAgent?
          # GNN
          scorer_inputs.append(trajectory.graph)
      else:
          scorer_inputs.append(trajectory.state)
      if trajectory.possible_actions_mask is not None:
          scorer_inputs.append(trajectory.possible_actions_mask)
      scores = self.scorer(*scorer_inputs)
      offset_reinforcement = discounted_returns(
          torch.clamp(rewards, max=self.reward_clip).clone(), self.gamma
      )
      if self.normalize:
          offset_reinforcement = whiten(
              offset_reinforcement, subtract_mean=self.subtract_mean
          )
      if self.offset_clamp_min:
          offset_reinforcement = offset_reinforcement.clamp(min=0)
      if self.value_net is not None:
          # subtract learned value function baselines from rewards
          baselines = self.value_net(trajectory.state).squeeze()
          # use reward-to-go as label for training the value function
          losses["value_net_loss"] = self.value_loss_fn(
              baselines, offset_reinforcement
          )
          # detach bcs we want PPO to tweak policy, not baseline
          offset_reinforcement = (
              offset_reinforcement - baselines.detach()
          )
  
      target_propensity = self.sampler.log_prob(scores, actions).float()
      characteristic_eligibility = torch.exp(
          target_propensity - trajectory.log_prob.detach()
      ).float()
  
      losses["ppo_loss"] = -torch.min(
          offset_reinforcement.float() @ characteristic_eligibility,
          offset_reinforcement.float()
          @ torch.clamp(
              characteristic_eligibility,
              1 - self.ppo_epsilon,
              1 + self.ppo_epsilon,
          ),
      )
      if self.entropy_weight != 0:
          entropy = self.sampler.entropy(scores)
          # "-" bcs minimizing, not maximizing
          losses["ppo_loss"] = (
              losses["ppo_loss"] - self.entropy_weight * entropy
          )
      return losses
  
imports: import inspect
  import logging
  from dataclasses import field
  from typing import Dict, List, Optional, Union
  import reagent.core.types
  import torch
  import torch.optim
  from reagent.core.configuration import resolve_defaults
  from reagent.gym.policies.policy import Policy
  from reagent.models.base import ModelBase
  from reagent.optimizer.union import Optimizer__Union
  from reagent.training.reagent_lightning_module import ReAgentLightningModule
  from reagent.training.utils import discounted_returns, whiten
label_desc: Use image node_id 2 for calling the PPOTrainer obj's underlying member method code with example usage: obj._trajectory_to_losses(trajectory) and returns: losses