class_docstr: ACER (Actor-Critic with Experience Replay).
  
  See http://arxiv.org/abs/1611.01224
  
  Args:
      model (ACERModel): Model to train. It must be a callable that accepts
          observations as input and return three values: action distributions
          (Distribution), Q values (ActionValue) and state values
          (torch.Tensor).
      optimizer (torch.optim.Optimizer): optimizer used to train the model
      t_max (int): The model is updated after every t_max local steps
      gamma (float): Discount factor [0,1]
      replay_buffer (EpisodicReplayBuffer): Replay buffer to use. If set
          None, this agent won't use experience replay.
      beta (float): Weight coefficient for the entropy regularizaiton term.
      phi (callable): Feature extractor function
      pi_loss_coef (float): Weight coefficient for the loss of the policy
      Q_loss_coef (float): Weight coefficient for the loss of the value
          function
      use_trust_region (bool): If set true, use efficient TRPO.
      trust_region_alpha (float): Decay rate of the average model used for
          efficient TRPO.
      trust_region_delta (float): Threshold used for efficient TRPO.
      truncation_threshold (float or None): Threshold used to truncate larger
          importance weights. If set None, importance weights are not
          truncated.
      disable_online_update (bool): If set true, disable online on-policy
          update and rely only on experience replay.
      n_times_replay (int): Number of times experience replay is repeated per
          one time of online update.
      replay_start_size (int): Experience replay is disabled if the number of
          transitions in the replay buffer is lower than this value.
      normalize_loss_by_steps (bool): If set true, losses are normalized by
          the number of steps taken to accumulate the losses
      act_deterministically (bool): If set true, choose most probable actions
          in act method.
      max_grad_norm (float or None): Maximum L2 norm of the gradient used for
          gradient clipping. If set to None, the gradient is not clipped.
      recurrent (bool): If set to True, `model` is assumed to implement
          `pfrl.nn.StatelessRecurrent`.
      use_Q_opc (bool): If set true, use Q_opc, a Q-value estimate without
          importance sampling, is used to compute advantage values for policy
          gradients. The original paper recommend to use in case of
          continuous action.
      average_entropy_decay (float): Decay rate of average entropy. Used only
          to record statistics.
      average_value_decay (float): Decay rate of average value. Used only
          to record statistics.
      average_kl_decay (float): Decay rate of kl value. Used only to record
          statistics.
class_docstr_tok: ["ACER","(","Actor-Critic","with","Experience","Replay",")",".","See","http",":","\/\/arxiv.org\/abs\/1611.01224","Args",":","model","(","ACERModel",")",":","Model","to","train",".","It","must","be","a","callable","that","accepts","observations","as","input","and","return","three","values",":","action","distributions","(","Distribution",")",",","Q","values","(","ActionValue",")","and","state","values","(","torch.Tensor",")",".","optimizer","(","torch.optim.Optimizer",")",":","optimizer","used","to","train","the","model","t_max","(","int",")",":","The","model","is","updated","after","every","t_max","local","steps","gamma","(","float",")",":","Discount","factor","[","0,1","]","replay_buffer","(","EpisodicReplayBuffer",")",":","Replay","buffer","to","use",".","If","set","None",",","this","agent","wo","n't","use","experience","replay",".","beta","(","float",")",":","Weight","coefficient","for","the","entropy","regularizaiton","term",".","phi","(","callable",")",":","Feature","extractor","function","pi_loss_coef","(","float",")",":","Weight","coefficient","for","the","loss","of","the","policy","Q_loss_coef","(","float",")",":","Weight","coefficient","for","the","loss","of","the","value","function","use_trust_region","(","bool",")",":","If","set","true",",","use","efficient","TRPO",".","trust_region_alpha","(","float",")",":","Decay","rate","of","the","average","model","used","for","efficient","TRPO",".","trust_region_delta","(","float",")",":","Threshold","used","for","efficient","TRPO",".","truncation_threshold","(","float","or","None",")",":","Threshold","used","to","truncate","larger","importance","weights",".","If","set","None",",","importance","weights","are","not","truncated",".","disable_online_update","(","bool",")",":","If","set","true",",","disable","online","on-policy","update","and","rely","only","on","experience","replay",".","n_times_replay","(","int",")",":","Number","of","times","experience","replay","is","repeated","per","one","time","of","online","update",".","replay_start_size","(","int",")",":","Experience","replay","is","disabled","if","the","number","of","transitions","in","the","replay","buffer","is","lower","than","this","value",".","normalize_loss_by_steps","(","bool",")",":","If","set","true",",","losses","are","normalized","by","the","number","of","steps","taken","to","accumulate","the","losses","act_deterministically","(","bool",")",":","If","set","true",",","choose","most","probable","actions","in","act","method",".","max_grad_norm","(","float","or","None",")",":","Maximum","L2","norm","of","the","gradient","used","for","gradient","clipping",".","If","set","to","None",",","the","gradient","is","not","clipped",".","recurrent","(","bool",")",":","If","set","to","True",",","`","model","`","is","assumed","to","implement","`","pfrl.nn.StatelessRecurrent","`",".","use_Q_opc","(","bool",")",":","If","set","true",",","use","Q_opc",",","a","Q-value","estimate","without","importance","sampling",",","is","used","to","compute","advantage","values","for","policy","gradients",".","The","original","paper","recommend","to","use","in","case","of","continuous","action",".","average_entropy_decay","(","float",")",":","Decay","rate","of","average","entropy",".","Used","only","to","record","statistics",".","average_value_decay","(","float",")",":","Decay","rate","of","average","value",".","Used","only","to","record","statistics",".","average_kl_decay","(","float",")",":","Decay","rate","of","kl","value",".","Used","only","to","record","statistics","."]
code: def init_history_data_for_online_update(self):
      self.past_actions = {}
      self.past_rewards = {}
      self.past_values = {}
      self.past_action_distrib = {}
      self.past_action_values = {}
      self.past_avg_action_distrib = {}
      self.t_start = self.t
  
imports: import copy
  from logging import getLogger
  import numpy
  import torch
  from torch import nn
  from pfrl import agent
  from pfrl.action_value import SingleActionValue
  from pfrl.utils import clip_l2_grad_norm_, copy_param
  from pfrl.utils.batch_states import batch_states
  from pfrl.utils.mode_of_distribution import mode_of_distribution
  from pfrl.utils.recurrent import detach_recurrent_state, one_step_forward
label_desc: Use image node_id 2 for calling the ACER obj's underlying member method code with example usage: obj.init_history_data_for_online_update() without return types