class_docstr: PCL (Path Consistency Learning).
  
  Not only the batch PCL algorithm proposed in the paper but also its
  asynchronous variant is implemented.
  
  See https://arxiv.org/abs/1702.08892
  
  Args:
      model (chainer.Link): Model to train. It must be a callable that
          accepts a batch of observations as input and return two values:
  
          - action distributions (Distribution)
          - state values (chainer.Variable)
      optimizer (chainer.Optimizer): optimizer used to train the model
      t_max (int or None): The model is updated after every t_max local
          steps. If set None, the model is updated after every episode.
      gamma (float): Discount factor [0,1]
      tau (float): Weight coefficient for the entropy regularizaiton term.
      phi (callable): Feature extractor function
      pi_loss_coef (float): Weight coefficient for the loss of the policy
      v_loss_coef (float): Weight coefficient for the loss of the value
          function
      rollout_len (int): Number of rollout steps
      batchsize (int): Number of episodes or sub-trajectories used for an
          update. The total number of transitions used will be
          (batchsize x t_max).
      disable_online_update (bool): If set true, disable online on-policy
          update and rely only on experience replay.
      n_times_replay (int): Number of times experience replay is repeated per
          one time of online update.
      replay_start_size (int): Experience replay is disabled if the number of
          transitions in the replay buffer is lower than this value.
      normalize_loss_by_steps (bool): If set true, losses are normalized by
          the number of steps taken to accumulate the losses
      act_deterministically (bool): If set true, choose most probable actions
          in act method.
      average_loss_decay (float): Decay rate of average loss. Used only
          to record statistics.
      average_entropy_decay (float): Decay rate of average entropy. Used only
          to record statistics.
      average_value_decay (float): Decay rate of average value. Used only
          to record statistics.
      explorer (Explorer or None): If not None, this explorer is used for
          selecting actions.
      logger (None or Logger): Logger to be used
      batch_states (callable): Method which makes a batch of observations.
          default is `chainerrl.misc.batch_states.batch_states`
      backprop_future_values (bool): If set True, value gradients are
          computed not only wrt V(s_t) but also V(s_{t+d}).
      train_async (bool): If set True, use a process-local model to compute
          gradients and update the globally shared model.
code: def update_from_replay(self):
      if self.replay_buffer is None:
          return
  
      if len(self.replay_buffer) < self.replay_start_size:
          return
  
      if self.replay_buffer.n_episodes < self.batchsize:
          return
  
      if self.process_idx == 0:
          self.logger.debug("update_from_replay")
  
      episodes = self.replay_buffer.sample_episodes(
          self.batchsize, max_len=self.t_max
      )
      if isinstance(episodes, tuple):
          # Prioritized replay
          episodes, weights = episodes
      else:
          weights = [1] * len(episodes)
      sorted_episodes = list(reversed(sorted(episodes, key=len)))
      max_epi_len = len(sorted_episodes[0])
  
      with state_reset(self.model):
          # Batch computation of multiple episodes
          rewards = {}
          values = {}
          next_values = {}
          log_probs = {}
          next_action_distrib = None
          next_v = None
          for t in range(max_epi_len):
              transitions = []
              for ep in sorted_episodes:
                  if len(ep) <= t:
                      break
                  transitions.append([ep[t]])
              batch = batch_experiences(
                  transitions,
                  xp=self.xp,
                  phi=self.phi,
                  gamma=self.gamma,
                  batch_states=self.batch_states,
              )
              batchsize = batch["action"].shape[0]
              if next_action_distrib is not None:
                  action_distrib = next_action_distrib[0:batchsize]
                  v = next_v[0:batchsize]
              else:
                  action_distrib, v = self.model(batch["state"])
              next_action_distrib, next_v = self.model(
                  batch["next_state"]
              )
              values[t] = v
              next_values[t] = next_v * (
                  1 - batch["is_state_terminal"].reshape(next_v.shape)
              )
              rewards[t] = chainer.cuda.to_cpu(batch["reward"])
              log_probs[t] = action_distrib.log_prob(batch["action"])
          # Loss is computed one by one episode
          losses = []
          for i, ep in enumerate(sorted_episodes):
              e_values = {}
              e_next_values = {}
              e_rewards = {}
              e_log_probs = {}
              for t in range(len(ep)):
                  assert values[t].shape[0] > i
                  assert next_values[t].shape[0] > i
                  assert rewards[t].shape[0] > i
                  assert log_probs[t].shape[0] > i
                  e_values[t] = values[t][i : i + 1]
                  e_next_values[t] = next_values[t][i : i + 1]
                  e_rewards[t] = float(rewards[t][i : i + 1])
                  e_log_probs[t] = log_probs[t][i : i + 1]
              losses.append(
                  self.compute_loss(
                      t_start=0,
                      t_stop=len(ep),
                      rewards=e_rewards,
                      values=e_values,
                      next_values=e_next_values,
                      log_probs=e_log_probs,
                  )
              )
          loss = (
              chainerrl.functions.weighted_sum_arrays(losses, weights)
              / self.batchsize
          )
          self.update(loss)
  
imports: import copy
  from logging import getLogger
  import chainer
  from chainer import functions
  import chainerrl
  from chainerrl import agent
  from chainerrl.agents import a3c
  from chainerrl.misc import async_
  from chainerrl.misc.batch_states import batch_states
  from chainerrl.misc import copy_param
  from chainerrl.recurrent import Recurrent
  from chainerrl.recurrent import state_kept
  from chainerrl.recurrent import state_reset
  from chainerrl.replay_buffer import batch_experiences
label_desc: Use image node_id 6 for calling the PCL obj's underlying member method code with example usage: obj.update_from_replay() without return types