class_docstr: Deep Deterministic Policy Gradients.
  
  This can be used as SVG(0) by specifying a Gaussian policy instead of a
  deterministic policy.
  
  Args:
      model (DDPGModel): DDPG model that contains both a policy and a
          Q-function
      actor_optimizer (Optimizer): Optimizer setup with the policy
      critic_optimizer (Optimizer): Optimizer setup with the Q-function
      replay_buffer (ReplayBuffer): Replay buffer
      gamma (float): Discount factor
      explorer (Explorer): Explorer that specifies an exploration strategy.
      gpu (int): GPU device id if not None nor negative.
      replay_start_size (int): if the replay buffer's size is less than
          replay_start_size, skip update
      minibatch_size (int): Minibatch size
      update_interval (int): Model update interval in step
      target_update_interval (int): Target model update interval in step
      phi (callable): Feature extractor applied to observations
      target_update_method (str): 'hard' or 'soft'.
      soft_update_tau (float): Tau of soft target update.
      n_times_update (int): Number of repetition of update
      average_q_decay (float): Decay rate of average Q, only used for
          recording statistics
      average_loss_decay (float): Decay rate of average loss, only used for
          recording statistics
      batch_accumulator (str): 'mean' or 'sum'
      episodic_update (bool): Use full episodes for update if set True
      episodic_update_len (int or None): Subsequences of this length are used
          for update if set int and episodic_update=True
      logger (Logger): Logger used
      batch_states (callable): method which makes a batch of observations.
          default is `chainerrl.misc.batch_states.batch_states`
      burnin_action_func (callable or None): If not None, this callable
          object is used to select actions before the model is updated
          one or more times during training.
docstr: Compute loss for critic.
  
  Preconditions:
    target_q_function must have seen up to s_t and a_t.
    target_policy must have seen up to s_t.
    q_function must have seen up to s_{t-1} and a_{t-1}.
  Postconditions:
    target_q_function must have seen up to s_{t+1} and a_{t+1}.
    target_policy must have seen up to s_{t+1}.
    q_function must have seen up to s_t and a_t.
docstr_tok: ["Compute","loss","for","critic",".","Preconditions",":","target_q_function","must","have","seen","up","to","s_t","and","a_t",".","target_policy","must","have","seen","up","to","s_t",".","q_function","must","have","seen","up","to","s_","{","t-1","}","and","a_","{","t-1","}",".","Postconditions",":","target_q_function","must","have","seen","up","to","s_","{","t+1","}","and","a_","{","t+1","}",".","target_policy","must","have","seen","up","to","s_","{","t+1","}",".","q_function","must","have","seen","up","to","s_t","and","a_t","."]
code: def compute_critic_loss(self, batch):
      """Compute loss for critic.
  
      Preconditions:
        target_q_function must have seen up to s_t and a_t.
        target_policy must have seen up to s_t.
        q_function must have seen up to s_{t-1} and a_{t-1}.
      Postconditions:
        target_q_function must have seen up to s_{t+1} and a_{t+1}.
        target_policy must have seen up to s_{t+1}.
        q_function must have seen up to s_t and a_t.
      """
  
      batch_next_state = batch["next_state"]
      batch_rewards = batch["reward"]
      batch_terminal = batch["is_state_terminal"]
      batch_state = batch["state"]
      batch_actions = batch["action"]
      batchsize = len(batch_rewards)
  
      with chainer.no_backprop_mode():
          # Target policy observes s_{t+1}
          next_actions = self.target_policy(batch_next_state).sample()
  
          # Q(s_{t+1}, mu(a_{t+1})) is evaluated.
          # This should not affect the internal state of Q.
          with state_kept(self.target_q_function):
              next_q = self.target_q_function(
                  batch_next_state, next_actions
              )
  
          # Target Q-function observes s_{t+1} and a_{t+1}
          if isinstance(self.target_q_function, Recurrent):
              batch_next_actions = batch["next_action"]
              self.target_q_function.update_state(
                  batch_next_state, batch_next_actions
              )
  
          target_q = batch_rewards + self.gamma * (
              1.0 - batch_terminal
          ) * F.reshape(next_q, (batchsize,))
  
      # Estimated Q-function observes s_t and a_t
      predict_q = F.reshape(
          self.q_function(batch_state, batch_actions), (batchsize,)
      )
  
      loss = F.mean_squared_error(target_q, predict_q)
  
      # Update stats
      self.average_critic_loss *= self.average_loss_decay
      self.average_critic_loss += (1 - self.average_loss_decay) * float(
          loss.array
      )
  
      return loss
  
imports: import copy
  from logging import getLogger
  import chainer
  from chainer import cuda
  import chainer.functions
  from chainerrl.agent import AttributeSavingMixin
  from chainerrl.agent import BatchAgent
  from chainerrl.misc.batch_states import batch_states
  from chainerrl.misc.copy_param import synchronize_parameters
  from chainerrl.recurrent import Recurrent
  from chainerrl.recurrent import RecurrentChainMixin
  from chainerrl.recurrent import state_kept
  from chainerrl.replay_buffer import batch_experiences
  from chainerrl.replay_buffer import ReplayUpdater
label_desc: Use image node_id 3 for calling the DDPG obj's underlying member method code with example usage: obj.compute_critic_loss(batch) and returns: loss