class_docstr: Deep Q-Network algorithm.
  
  Args:
      q_function (StateQFunction): Q-function
      optimizer (Optimizer): Optimizer that is already setup
      replay_buffer (ReplayBuffer): Replay buffer
      gamma (float): Discount factor
      explorer (Explorer): Explorer that specifies an exploration strategy.
      gpu (int): GPU device id if not None nor negative.
      replay_start_size (int): if the replay buffer's size is less than
          replay_start_size, skip update
      minibatch_size (int): Minibatch size
      update_interval (int): Model update interval in step
      target_update_interval (int): Target model update interval in step
      clip_delta (bool): Clip delta if set True
      phi (callable): Feature extractor applied to observations
      target_update_method (str): 'hard' or 'soft'.
      soft_update_tau (float): Tau of soft target update.
      n_times_update (int): Number of repetition of update
      batch_accumulator (str): 'mean' or 'sum'
      episodic_update_len (int or None): Subsequences of this length are used
          for update if set int and episodic_update=True
      logger (Logger): Logger used
      batch_states (callable): method which makes a batch of observations.
          default is `pfrl.utils.batch_states.batch_states`
      recurrent (bool): If set to True, `model` is assumed to implement
          `pfrl.nn.Recurrent` and is updated in a recurrent
          manner.
      max_grad_norm (float or None): Maximum L2 norm of the gradient used for
          gradient clipping. If set to None, the gradient is not clipped.
class_docstr_tok: ["Deep","Q-Network","algorithm",".","Args",":","q_function","(","StateQFunction",")",":","Q-function","optimizer","(","Optimizer",")",":","Optimizer","that","is","already","setup","replay_buffer","(","ReplayBuffer",")",":","Replay","buffer","gamma","(","float",")",":","Discount","factor","explorer","(","Explorer",")",":","Explorer","that","specifies","an","exploration","strategy",".","gpu","(","int",")",":","GPU","device","id","if","not","None","nor","negative",".","replay_start_size","(","int",")",":","if","the","replay","buffer","'s","size","is","less","than","replay_start_size",",","skip","update","minibatch_size","(","int",")",":","Minibatch","size","update_interval","(","int",")",":","Model","update","interval","in","step","target_update_interval","(","int",")",":","Target","model","update","interval","in","step","clip_delta","(","bool",")",":","Clip","delta","if","set","True","phi","(","callable",")",":","Feature","extractor","applied","to","observations","target_update_method","(","str",")",":","'hard","'","or","'soft","'",".","soft_update_tau","(","float",")",":","Tau","of","soft","target","update",".","n_times_update","(","int",")",":","Number","of","repetition","of","update","batch_accumulator","(","str",")",":","'mean","'","or","'sum'","episodic_update_len","(","int","or","None",")",":","Subsequences","of","this","length","are","used","for","update","if","set","int","and","episodic_update=True","logger","(","Logger",")",":","Logger","used","batch_states","(","callable",")",":","method","which","makes","a","batch","of","observations",".","default","is","`","pfrl.utils.batch_states.batch_states","`","recurrent","(","bool",")",":","If","set","to","True",",","`","model","`","is","assumed","to","implement","`","pfrl.nn.Recurrent","`","and","is","updated","in","a","recurrent","manner",".","max_grad_norm","(","float","or","None",")",":","Maximum","L2","norm","of","the","gradient","used","for","gradient","clipping",".","If","set","to","None",",","the","gradient","is","not","clipped","."]
code: def __init__(
      self,
      q_function: torch.nn.Module,
      optimizer: torch.optim.Optimizer,  # type: ignore  # somehow mypy complains
      replay_buffer: pfrl.replay_buffer.AbstractReplayBuffer,
      gamma: float,
      explorer: Explorer,
      gpu: Optional[int] = None,
      replay_start_size: int = 50000,
      minibatch_size: int = 32,
      update_interval: int = 1,
      target_update_interval: int = 10000,
      clip_delta: bool = True,
      phi: Callable[[Any], Any] = lambda x: x,
      target_update_method: str = "hard",
      soft_update_tau: float = 1e-2,
      n_times_update: int = 1,
      batch_accumulator: str = "mean",
      episodic_update_len: Optional[int] = None,
      logger: Logger = getLogger(__name__),
      batch_states: Callable[
          [Sequence[Any], torch.device, Callable[[Any], Any]], Any
      ] = batch_states,
      recurrent: bool = False,
      max_grad_norm: Optional[float] = None,
  ):
      self.model = q_function
  
      if gpu is not None and gpu >= 0:
          assert torch.cuda.is_available()
          self.device = torch.device("cuda:{}".format(gpu))
          self.model.to(self.device)
      else:
          self.device = torch.device("cpu")
  
      self.replay_buffer = replay_buffer
      self.optimizer = optimizer
      self.gamma = gamma
      self.explorer = explorer
      self.gpu = gpu
      self.target_update_interval = target_update_interval
      self.clip_delta = clip_delta
      self.phi = phi
      self.target_update_method = target_update_method
      self.soft_update_tau = soft_update_tau
      self.batch_accumulator = batch_accumulator
      assert batch_accumulator in ("mean", "sum")
      self.logger = logger
      self.batch_states = batch_states
      self.recurrent = recurrent
      update_func: Callable[..., None]
      if self.recurrent:
          assert isinstance(
              self.replay_buffer, AbstractEpisodicReplayBuffer
          )
          update_func = self.update_from_episodes
      else:
          update_func = self.update
      self.replay_updater = ReplayUpdater(
          replay_buffer=replay_buffer,
          update_func=update_func,
          batchsize=minibatch_size,
          episodic_update=recurrent,
          episodic_update_len=episodic_update_len,
          n_times_update=n_times_update,
          replay_start_size=replay_start_size,
          update_interval=update_interval,
      )
      self.minibatch_size = minibatch_size
      self.episodic_update_len = episodic_update_len
      self.replay_start_size = replay_start_size
      self.update_interval = update_interval
      self.max_grad_norm = max_grad_norm
  
      assert (
          target_update_interval % update_interval == 0
      ), "target_update_interval should be a multiple of update_interval"
  
      self.t = 0
      self.optim_t = 0  # Compensate pytorch optim not having `t`
      self._cumulative_steps = 0
      self.target_model = make_target_model_as_copy(self.model)
  
      # Statistics
      self.q_record: collections.deque = collections.deque(maxlen=1000)
      self.loss_record: collections.deque = collections.deque(
          maxlen=100
      )
  
      # Recurrent states of the model
      self.train_recurrent_states: Any = None
      self.train_prev_recurrent_states: Any = None
      self.test_recurrent_states: Any = None
  
      # Error checking
      if (
          self.replay_buffer.capacity is not None
          and self.replay_buffer.capacity
          < self.replay_updater.replay_start_size
      ):
          raise ValueError(
              "Replay start size cannot exceed replay buffer capacity."
          )
  
imports: import collections
  import copy
  import ctypes
  import multiprocessing
  import multiprocessing.synchronize
  import os
  import time
  import typing
  from logging import Logger, getLogger
  from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
  import numpy
  import torch
  import torch.nn.functional
  import pfrl
  from pfrl import agent
  from pfrl.action_value import ActionValue
  from pfrl.explorer import Explorer
  from pfrl.replay_buffer import AbstractEpisodicReplayBuffer, ReplayUpdater, batch_experiences, batch_recurrent_experiences
  from pfrl.replay_buffers import PrioritizedReplayBuffer
  from pfrl.utils.batch_states import batch_states
  from pfrl.utils.contexts import evaluating
  from pfrl.utils.copy_param import synchronize_parameters
  from pfrl.utils.recurrent import get_recurrent_state_at, mask_recurrent_state_at, one_step_forward, pack_and_forward, recurrent_state_as_numpy
label_desc: Use image node_id 1 to create a new DQN object from inherited base classes: agent, agent with example: obj = DQN(q_function, optimizer, replay_buffer, gamma, explorer, gpu, replay_start_size, minibatch_size, update_interval, target_update_interval, clip_delta, phi, target_update_method, soft_update_tau, n_times_update, batch_accumulator, episodic_update_len, logger, batch_states, recurrent, max_grad_norm)