class_docstr: ACER (Actor-Critic with Experience Replay).
  
  See http://arxiv.org/abs/1611.01224
  
  Args:
      model (ACERModel): Model to train. It must be a callable that accepts
          observations as input and return three values: action distributions
          (Distribution), Q values (ActionValue) and state values
          (torch.Tensor).
      optimizer (torch.optim.Optimizer): optimizer used to train the model
      t_max (int): The model is updated after every t_max local steps
      gamma (float): Discount factor [0,1]
      replay_buffer (EpisodicReplayBuffer): Replay buffer to use. If set
          None, this agent won't use experience replay.
      beta (float): Weight coefficient for the entropy regularizaiton term.
      phi (callable): Feature extractor function
      pi_loss_coef (float): Weight coefficient for the loss of the policy
      Q_loss_coef (float): Weight coefficient for the loss of the value
          function
      use_trust_region (bool): If set true, use efficient TRPO.
      trust_region_alpha (float): Decay rate of the average model used for
          efficient TRPO.
      trust_region_delta (float): Threshold used for efficient TRPO.
      truncation_threshold (float or None): Threshold used to truncate larger
          importance weights. If set None, importance weights are not
          truncated.
      disable_online_update (bool): If set true, disable online on-policy
          update and rely only on experience replay.
      n_times_replay (int): Number of times experience replay is repeated per
          one time of online update.
      replay_start_size (int): Experience replay is disabled if the number of
          transitions in the replay buffer is lower than this value.
      normalize_loss_by_steps (bool): If set true, losses are normalized by
          the number of steps taken to accumulate the losses
      act_deterministically (bool): If set true, choose most probable actions
          in act method.
      max_grad_norm (float or None): Maximum L2 norm of the gradient used for
          gradient clipping. If set to None, the gradient is not clipped.
      recurrent (bool): If set to True, `model` is assumed to implement
          `pfrl.nn.StatelessRecurrent`.
      use_Q_opc (bool): If set true, use Q_opc, a Q-value estimate without
          importance sampling, is used to compute advantage values for policy
          gradients. The original paper recommend to use in case of
          continuous action.
      average_entropy_decay (float): Decay rate of average entropy. Used only
          to record statistics.
      average_value_decay (float): Decay rate of average value. Used only
          to record statistics.
      average_kl_decay (float): Decay rate of kl value. Used only to record
          statistics.
class_docstr_tok: ["ACER","(","Actor-Critic","with","Experience","Replay",")",".","See","http",":","\/\/arxiv.org\/abs\/1611.01224","Args",":","model","(","ACERModel",")",":","Model","to","train",".","It","must","be","a","callable","that","accepts","observations","as","input","and","return","three","values",":","action","distributions","(","Distribution",")",",","Q","values","(","ActionValue",")","and","state","values","(","torch.Tensor",")",".","optimizer","(","torch.optim.Optimizer",")",":","optimizer","used","to","train","the","model","t_max","(","int",")",":","The","model","is","updated","after","every","t_max","local","steps","gamma","(","float",")",":","Discount","factor","[","0,1","]","replay_buffer","(","EpisodicReplayBuffer",")",":","Replay","buffer","to","use",".","If","set","None",",","this","agent","wo","n't","use","experience","replay",".","beta","(","float",")",":","Weight","coefficient","for","the","entropy","regularizaiton","term",".","phi","(","callable",")",":","Feature","extractor","function","pi_loss_coef","(","float",")",":","Weight","coefficient","for","the","loss","of","the","policy","Q_loss_coef","(","float",")",":","Weight","coefficient","for","the","loss","of","the","value","function","use_trust_region","(","bool",")",":","If","set","true",",","use","efficient","TRPO",".","trust_region_alpha","(","float",")",":","Decay","rate","of","the","average","model","used","for","efficient","TRPO",".","trust_region_delta","(","float",")",":","Threshold","used","for","efficient","TRPO",".","truncation_threshold","(","float","or","None",")",":","Threshold","used","to","truncate","larger","importance","weights",".","If","set","None",",","importance","weights","are","not","truncated",".","disable_online_update","(","bool",")",":","If","set","true",",","disable","online","on-policy","update","and","rely","only","on","experience","replay",".","n_times_replay","(","int",")",":","Number","of","times","experience","replay","is","repeated","per","one","time","of","online","update",".","replay_start_size","(","int",")",":","Experience","replay","is","disabled","if","the","number","of","transitions","in","the","replay","buffer","is","lower","than","this","value",".","normalize_loss_by_steps","(","bool",")",":","If","set","true",",","losses","are","normalized","by","the","number","of","steps","taken","to","accumulate","the","losses","act_deterministically","(","bool",")",":","If","set","true",",","choose","most","probable","actions","in","act","method",".","max_grad_norm","(","float","or","None",")",":","Maximum","L2","norm","of","the","gradient","used","for","gradient","clipping",".","If","set","to","None",",","the","gradient","is","not","clipped",".","recurrent","(","bool",")",":","If","set","to","True",",","`","model","`","is","assumed","to","implement","`","pfrl.nn.StatelessRecurrent","`",".","use_Q_opc","(","bool",")",":","If","set","true",",","use","Q_opc",",","a","Q-value","estimate","without","importance","sampling",",","is","used","to","compute","advantage","values","for","policy","gradients",".","The","original","paper","recommend","to","use","in","case","of","continuous","action",".","average_entropy_decay","(","float",")",":","Decay","rate","of","average","entropy",".","Used","only","to","record","statistics",".","average_value_decay","(","float",")",":","Decay","rate","of","average","value",".","Used","only","to","record","statistics",".","average_kl_decay","(","float",")",":","Decay","rate","of","kl","value",".","Used","only","to","record","statistics","."]
code: def compute_loss(
      self,
      t_start,
      t_stop,
      R,
      actions,
      rewards,
      values,
      action_values,
      action_distribs,
      action_distribs_mu,
      avg_action_distribs,
  ):
      assert np.isscalar(R)
      pi_loss = 0
      Q_loss = 0
      Q_ret = R
      Q_opc = R
      discrete = isinstance(
          action_distribs[t_start], torch.distributions.Categorical
      )
      del R
      for i in reversed(range(t_start, t_stop)):
          r = rewards[i]
          v = values[i]
          action_distrib = action_distribs[i]
          action_distrib_mu = (
              action_distribs_mu[i] if action_distribs_mu else None
          )
          avg_action_distrib = avg_action_distribs[i]
          action_value = action_values[i]
          ba = torch.as_tensor(actions[i]).unsqueeze(0)
          if action_distrib_mu is not None:
              # Off-policy
              rho = compute_importance(
                  action_distrib, action_distrib_mu, ba
              )
          else:
              # On-policy
              rho = 1
  
          Q_ret = r + self.gamma * Q_ret
          Q_opc = r + self.gamma * Q_opc
  
          assert np.isscalar(Q_ret)
          assert np.isscalar(Q_opc)
          if self.use_Q_opc:
              advantage = Q_opc - float(v)
          else:
              advantage = Q_ret - float(v)
          pi_loss += self.compute_one_step_pi_loss(
              action=ba,
              advantage=advantage,
              action_distrib=action_distrib,
              action_distrib_mu=action_distrib_mu,
              action_value=action_value,
              v=float(v),
              avg_action_distrib=avg_action_distrib,
          )
  
          # Accumulate gradients of value function
          Q = action_value.evaluate_actions(ba)
          assert Q.requires_grad, "Q must be backprop-able"
          Q_loss += nn.functional.mse_loss(torch.tensor(Q_ret), Q) / 2
  
          if not discrete:
              assert v.requires_grad, "v must be backprop-able"
              v_target = min(1, rho) * (Q_ret - float(Q)) + float(v)
              Q_loss += (
                  nn.functional.mse_loss(torch.tensor(v_target), v) / 2
              )
  
          if self.process_idx == 0:
              self.logger.debug(
                  "t:%s v:%s Q:%s Q_ret:%s Q_opc:%s",
                  i,
                  float(v),
                  float(Q),
                  Q_ret,
                  Q_opc,
              )
  
          if discrete:
              c = min(1, rho)
          else:
              c = min(1, rho ** (1 / ba.numel()))
          Q_ret = c * (Q_ret - float(Q)) + float(v)
          Q_opc = Q_opc - float(Q) + float(v)
  
      pi_loss *= self.pi_loss_coef
      Q_loss *= self.Q_loss_coef
  
      if self.normalize_loss_by_steps:
          pi_loss /= t_stop - t_start
          Q_loss /= t_stop - t_start
  
      if self.process_idx == 0:
          self.logger.debug(
              "pi_loss:%s Q_loss:%s", float(pi_loss), float(Q_loss)
          )
  
      return pi_loss + Q_loss.reshape(*pi_loss.shape)
  
imports: import copy
  from logging import getLogger
  import numpy
  import torch
  from torch import nn
  from pfrl import agent
  from pfrl.action_value import SingleActionValue
  from pfrl.utils import clip_l2_grad_norm_, copy_param
  from pfrl.utils.batch_states import batch_states
  from pfrl.utils.mode_of_distribution import mode_of_distribution
  from pfrl.utils.recurrent import detach_recurrent_state, one_step_forward
label_desc: Use image node_id 7 for calling the ACER obj's underlying member method code with example usage: obj.compute_loss(t_start, t_stop, R, actions, rewards, values, action_values, action_distribs, action_distribs_mu, avg_action_distribs) and returns: unknown