class_docstr: Policy Gradient Theorem with an approximate policy and a Q-function.
  
  This agent is almost the same with DDPG except that it uses the likelihood
  ratio gradient estimation instead of value gradients.
  
  Args:
      model (chainer.Chain): Chain that contains both a policy and a
          Q-function
      actor_optimizer (Optimizer): Optimizer setup with the policy
      critic_optimizer (Optimizer): Optimizer setup with the Q-function
      replay_buffer (ReplayBuffer): Replay buffer
      gamma (float): Discount factor
      explorer (Explorer): Explorer that specifies an exploration strategy.
      gpu (int): GPU device id. -1 for CPU.
      replay_start_size (int): if the replay buffer's size is less than
          replay_start_size, skip update
      minibatch_size (int): Minibatch size
      update_interval (int): Model update interval in step
      target_update_interval (int): Target model update interval in step
      phi (callable): Feature extractor applied to observations
      target_update_method (str): 'hard' or 'soft'.
      soft_update_tau (float): Tau of soft target update.
      n_times_update (int): Number of repetition of update
      average_q_decay (float): Decay rate of average Q, only used for
          recording statistics
      average_loss_decay (float): Decay rate of average loss, only used for
          recording statistics
      batch_accumulator (str): 'mean' or 'sum'
      logger (Logger): Logger used
      beta (float): Coefficient for entropy regularization
      act_deterministically (bool): Act deterministically by selecting most
          probable actions in test time
      batch_states (callable): method which makes a batch of observations.
          default is `chainerrl.misc.batch_states.batch_states`
docstr: Update the model from experiences.
docstr_tok: ["Update","the","model","from","experiences","."]
code: def update(self, experiences, errors_out=None):
      """Update the model from experiences."""
  
      batch_size = len(experiences)
  
      batch_exp = batch_experiences(
          experiences,
          xp=self.xp,
          phi=self.phi,
          gamma=self.gamma,
          batch_states=self.batch_states,
      )
  
      batch_state = batch_exp["state"]
      batch_actions = batch_exp["action"]
      batch_next_state = batch_exp["next_state"]
      batch_rewards = batch_exp["reward"]
      batch_terminal = batch_exp["is_state_terminal"]
      batch_discount = batch_exp["discount"]
  
      # Update Q-function
      def compute_critic_loss():
          with chainer.no_backprop_mode():
              pout = self.target_policy(batch_next_state)
              next_actions = pout.sample()
              next_q = self.target_q_function(
                  batch_next_state, next_actions
              )
              assert next_q.shape == (batch_size, 1)
  
              target_q = batch_rewards[..., None] + (
                  batch_discount[..., None]
                  * (1.0 - batch_terminal[..., None])
                  * next_q
              )
              assert target_q.shape == (batch_size, 1)
  
          predict_q = self.q_function(batch_state, batch_actions)
          assert predict_q.shape == (batch_size, 1)
  
          loss = F.mean_squared_error(target_q, predict_q)
  
          # Update stats
          self.average_critic_loss *= self.average_loss_decay
          self.average_critic_loss += (
              1 - self.average_loss_decay
          ) * float(loss.array)
  
          return loss
  
      def compute_actor_loss():
          pout = self.policy(batch_state)
          sampled_actions = pout.sample().array
          log_probs = pout.log_prob(sampled_actions)
          with chainer.using_config("train", False):
              q = self.q_function(batch_state, sampled_actions)
              v = self.q_function(batch_state, pout.most_probable)
          advantage = F.reshape(q - v, (batch_size,))
          advantage = chainer.Variable(advantage.array)
          loss = (
              -F.sum(advantage * log_probs + self.beta * pout.entropy)
              / batch_size
          )
  
          # Update stats
          self.average_actor_loss *= self.average_loss_decay
          self.average_actor_loss += (
              1 - self.average_loss_decay
          ) * float(loss.array)
  
          return loss
  
      self.critic_optimizer.update(compute_critic_loss)
      self.actor_optimizer.update(compute_actor_loss)
  
imports: import copy
  from logging import getLogger
  import chainer
  from chainer import cuda
  import chainer.functions
  from chainerrl.agent import Agent
  from chainerrl.agent import AttributeSavingMixin
  from chainerrl.agents.ddpg import disable_train
  from chainerrl.misc.batch_states import batch_states
  from chainerrl.misc.copy_param import synchronize_parameters
  from chainerrl.recurrent import Recurrent
  from chainerrl.replay_buffer import batch_experiences
  from chainerrl.replay_buffer import ReplayUpdater
label_desc: Use image node_id 3 for calling the PGT obj's underlying member method code with example usage: obj.update(experiences, errors_out) and returns: loss, loss