text: import torch
  import torch.nn as nn
  import torch.optim as optim
  
  from a2c_ppo_acktr.algo.kfac import KFACOptimizer
  
  
  class A2C_ACKTR():
      def __init__(self,
                   actor_critic,
                   value_loss_coef,
                   entropy_coef,
                   lr=None,
                   filter_mem=None,
                   eps=None,
                   alpha=None,
                   max_grad_norm=None,
                   acktr=False):
  
          self.actor_critic = actor_critic
          self.acktr = acktr
  
          self.value_loss_coef = value_loss_coef
          self.entropy_coef = entropy_coef
  
          self.max_grad_norm = max_grad_norm
  
          if acktr:
              self.optimizer = KFACOptimizer(actor_critic)
  
          self.filter_list = []
          self.param_list = []
          for name, param in actor_critic.named_parameters():
              if "base.filter_net" in name :
                  self.filter_list.append(param)
              else:
                  self.param_list.append(param)
  
          else:
              self.optimizer = optim.RMSprop([{'params': self.param_list},
                   {'params': self.filter_list}], lr, eps=eps, alpha=alpha)
  
      def update(self, rollouts, value_prev_eval, filter_mem_latent_eval, filter_type, filter_mem):
          obs_shape = rollouts.obs.size()[2:]
          action_shape = rollouts.actions.size()[-1]
          num_steps, num_processes, _ = rollouts.rewards.size()
  
          values, action_log_probs, dist_entropy, _ , value_prev_eval, filter_mem_latent_eval, att_list, grad_term = self.actor_critic.evaluate_actions(
              rollouts.obs[:-1],
              rollouts.recurrent_hidden_states[0],
              rollouts.masks[:-1],
              rollouts.actions,
              rollouts.att_target,
              value_prev_eval=value_prev_eval,
              filter_mem_latent_eval = filter_mem_latent_eval,
              filter_type=filter_type,
              filter_mem=filter_mem)
  
          values = values.view(num_steps, num_processes, 1)
          action_log_probs = action_log_probs.view(num_steps, num_processes, 1)
  
          #advantages = rollouts.returns[:-1] - grad_term.unsqueeze(2)
          #value_loss = advantages.pow(2).mean()
          advantages = rollouts.returns[:-1] - values
          value_loss = (-2*advantages*grad_term.unsqueeze(2)).mean()  # credit-TD error, assignment-FIR
          action_loss = -(advantages.detach() * action_log_probs).mean()
  
          if self.acktr and self.optimizer.steps % self.optimizer.Ts == 0:
              # Sampled fisher, see Martens 2014
              self.actor_critic.zero_grad()
              pg_fisher_loss = -action_log_probs.mean()
  
              value_noise = torch.randn(values.size())
              if values.is_cuda:
                  value_noise = value_noise.cuda()
  
              sample_values = values + value_noise
              vf_fisher_loss = -(values - sample_values.detach()).pow(2).mean()
  
              fisher_loss = pg_fisher_loss + vf_fisher_loss
              self.optimizer.acc_stats = True
              fisher_loss.backward(retain_graph=True)
              self.optimizer.acc_stats = False
  
          self.optimizer.zero_grad()
          (value_loss * self.value_loss_coef + action_loss -
           dist_entropy * self.entropy_coef).backward(retain_graph=True)
  
          if self.acktr == False:
              nn.utils.clip_grad_norm_(self.actor_critic.parameters(),
                                       self.max_grad_norm)
  
          self.optimizer.step()
  
          return value_loss.item(), action_loss.item(), dist_entropy.item(), value_prev_eval, filter_mem_latent_eval, att_list
  