class_docstr: Proximal Policy Optimization (PPO). See https://arxiv.org/pdf/1707.06347.pdf
  This is the "clip" version of PPO. It does not include:
  - KL divergence
  - Bootstrapping with a critic model (our approach only works if full trajectories up to terminal state are fed in)
  Optionally, a value network can be trained and used as a baseline for rewards.
code: def __init__(
      self,
      policy: Policy,
      gamma: float = 0.9,
      optimizer: Optimizer__Union = field(  # noqa: B008
          default_factory=Optimizer__Union.default
      ),
      optimizer_value_net: Optimizer__Union = field(  # noqa: B008
          default_factory=Optimizer__Union.default
      ),
      actions: List[str] = field(default_factory=list),  # noqa: B008
      reward_clip: float = 1e6,
      normalize: bool = True,
      subtract_mean: bool = True,
      offset_clamp_min: bool = False,
      update_freq: int = 1,  # how many env steps between updates
      update_epochs: int = 1,  # how many epochs to run when updating (for PPO)
      ppo_batch_size: int = 1,  # batch size (number of trajectories) used for PPO updates
      ppo_epsilon: float = 0.2,  # clamp importance weights between 1-epsilon and 1+epsilon
      entropy_weight: float = 0.0,  # weight of the entropy term in the PPO loss
      value_net: Optional[ModelBase] = None,
  ):
      # PPO relies on customized update schemas, achieved by manual_backward()
      super().__init__(automatic_optimization=False)
      self.scorer = policy.scorer
      self.sampler = policy.sampler
      self.gamma = gamma
      self.optimizer_value_net = optimizer_value_net
      self.actions = actions
      self.reward_clip = reward_clip
      self.normalize = normalize
      self.subtract_mean = subtract_mean
      self.offset_clamp_min = offset_clamp_min
      self.update_freq = update_freq
      self.update_epochs = update_epochs
      self.ppo_batch_size = ppo_batch_size
      self.ppo_epsilon = ppo_epsilon
      self.entropy_weight = entropy_weight
  
      self.optimizer = optimizer
      self.value_net = value_net
      if value_net is not None:
          self.value_loss_fn = torch.nn.MSELoss(reduction="mean")
          assert (
              not self.normalize
          ), "Can't apply a value baseline and normalize rewards simultaneously"
      assert (ppo_epsilon >= 0) and (
          ppo_epsilon <= 1
      ), "ppo_epslion has to be in [0;1]"
  
      self.traj_buffer = []
  
imports: import inspect
  import logging
  from dataclasses import field
  from typing import Dict, List, Optional, Union
  import reagent.core.types
  import torch
  import torch.optim
  from reagent.core.configuration import resolve_defaults
  from reagent.gym.policies.policy import Policy
  from reagent.models.base import ModelBase
  from reagent.optimizer.union import Optimizer__Union
  from reagent.training.reagent_lightning_module import ReAgentLightningModule
  from reagent.training.utils import discounted_returns, whiten
label_desc: Use image node_id 1 to create a new PPOTrainer object from inherited base classes: ReAgentLightningModule with example: obj = PPOTrainer(policy, gamma, optimizer, optimizer_value_net, actions, reward_clip, normalize, subtract_mean, offset_clamp_min, update_freq, update_epochs, ppo_batch_size, ppo_epsilon, entropy_weight, value_net)