class_docstr: Proximal Policy Optimization (PPO). See https://arxiv.org/pdf/1707.06347.pdf
  This is the "clip" version of PPO. It does not include:
  - KL divergence
  - Bootstrapping with a critic model (our approach only works if full trajectories up to terminal state are fed in)
  Optionally, a value network can be trained and used as a baseline for rewards.
code: def _update_model(
      self, training_batch_list: List[rlt.PolicyGradientInput]
  ):
      losses = {
          "ppo_loss": [],
          "value_net_loss": [],
      }
      value_net_opt, ppo_opt = self.get_optimizers()
  
      for traj in training_batch_list:
          loss = self._trajectory_to_losses(traj)
          for k, v in loss.items():
              losses[k].append(v)
  
      if self.value_net is not None:
          # TD loss for the baseline value network
          value_net_loss = torch.stack(losses["value_net_loss"]).sum()
          value_net_opt.zero_grad()
          self.manual_backward(value_net_loss)
          value_net_opt.step()
  
      # PPO "loss" for the policy network
      ppo_loss = torch.stack(losses["ppo_loss"]).sum()
      ppo_opt.zero_grad()
      self.manual_backward(ppo_loss)
      ppo_opt.step()
  
imports: import inspect
  import logging
  from dataclasses import field
  from typing import Dict, List, Optional, Union
  import reagent.core.types
  import torch
  import torch.optim
  from reagent.core.configuration import resolve_defaults
  from reagent.gym.policies.policy import Policy
  from reagent.models.base import ModelBase
  from reagent.optimizer.union import Optimizer__Union
  from reagent.training.reagent_lightning_module import ReAgentLightningModule
  from reagent.training.utils import discounted_returns, whiten
label_desc: Use image node_id 7 for calling the PPOTrainer obj's underlying member method code with example usage: obj._update_model(training_batch_list) without return types