class_docstr: # Arguments
      model__: A Keras model.
      policy__: A Keras-rl policy that are defined in [policy](https://github.com/keras-rl/keras-rl/blob/master/rl/policy.py).
      test_policy__: A Keras-rl policy.
      enable_double_dqn__: A boolean which enable target network as a second network proposed by van Hasselt et al. to decrease overfitting.
      enable_dueling_dqn__: A boolean which enable dueling architecture proposed by Mnih et al.
      dueling_type__: If `enable_dueling_dqn` is set to `True`, a type of dueling architecture must be chosen which calculate Q(s,a) from V(s) and A(s,a) differently. Note that `avg` is recommanded in the [paper](https://arxiv.org/abs/1511.06581).
          `avg`: Q(s,a;theta) = V(s;theta) + (A(s,a;theta)-Avg_a(A(s,a;theta)))
          `max`: Q(s,a;theta) = V(s;theta) + (A(s,a;theta)-max_a(A(s,a;theta)))
          `naive`: Q(s,a;theta) = V(s;theta) + A(s,a;theta)
code: def backward(self, reward, terminal):
      # Store most recent experience in memory.
      if self.step % self.memory_interval == 0:
          self.memory.append(
              self.recent_observation,
              self.recent_action,
              reward,
              terminal,
              training=self.training,
          )
  
      metrics = [np.nan for _ in self.metrics_names]
      if not self.training:
          # We're done here. No need to update the experience memory since we only use the working
          # memory to obtain the state over the most recent observations.
          return metrics
  
      # Train the network on a single stochastic batch.
      if (
          self.step > self.nb_steps_warmup
          and self.step % self.train_interval == 0
      ):
          experiences = self.memory.sample(self.batch_size)
          assert len(experiences) == self.batch_size
  
          # Start by extracting the necessary parameters (we use a vectorized implementation).
          state0_batch = []
          reward_batch = []
          action_batch = []
          terminal1_batch = []
          state1_batch = []
          for e in experiences:
              state0_batch.append(e.state0)
              state1_batch.append(e.state1)
              reward_batch.append(e.reward)
              action_batch.append(e.action)
              terminal1_batch.append(0.0 if e.terminal1 else 1.0)
  
          # Prepare and validate parameters.
          state0_batch = self.process_state_batch(state0_batch)
          state1_batch = self.process_state_batch(state1_batch)
          terminal1_batch = np.array(terminal1_batch)
          reward_batch = np.array(reward_batch)
          assert reward_batch.shape == (self.batch_size,)
          assert terminal1_batch.shape == reward_batch.shape
          assert len(action_batch) == len(reward_batch)
  
          # Compute Q values for mini-batch update.
          if self.enable_double_dqn:
              # According to the paper "Deep Reinforcement Learning with Double Q-learning"
              # (van Hasselt et al., 2015), in Double DQN, the online network predicts the actions
              # while the target network is used to estimate the Q value.
              q_values = self.model.predict_on_batch(state1_batch)
              assert q_values.shape == (
                  self.batch_size,
                  self.nb_actions,
              )
              actions = np.argmax(q_values, axis=1)
              assert actions.shape == (self.batch_size,)
  
              # Now, estimate Q values using the target network but select the values with the
              # highest Q value wrt to the online model (as computed above).
              target_q_values = self.target_model.predict_on_batch(
                  state1_batch
              )
              assert target_q_values.shape == (
                  self.batch_size,
                  self.nb_actions,
              )
              q_batch = target_q_values[range(self.batch_size), actions]
          else:
              # Compute the q_values given state1, and extract the maximum for each sample in the batch.
              # We perform this prediction on the target_model instead of the model for reasons
              # outlined in Mnih (2015). In short: it makes the algorithm more stable.
              target_q_values = self.target_model.predict_on_batch(
                  state1_batch
              )
              assert target_q_values.shape == (
                  self.batch_size,
                  self.nb_actions,
              )
              q_batch = np.max(target_q_values, axis=1).flatten()
          assert q_batch.shape == (self.batch_size,)
  
          targets = np.zeros((self.batch_size, self.nb_actions))
          dummy_targets = np.zeros((self.batch_size,))
          masks = np.zeros((self.batch_size, self.nb_actions))
  
          # Compute r_t + gamma * max_a Q(s_t+1, a) and update the target targets accordingly,
          # but only for the affected output units (as given by action_batch).
          discounted_reward_batch = self.gamma * q_batch
          # Set discounted reward to zero for all states that were terminal.
          discounted_reward_batch *= terminal1_batch
          assert discounted_reward_batch.shape == reward_batch.shape
          Rs = reward_batch + discounted_reward_batch
          for idx, (target, mask, R, action) in enumerate(
              zip(targets, masks, Rs, action_batch)
          ):
              target[
                  action
              ] = R  # update action with estimated accumulated reward
              dummy_targets[idx] = R
              mask[action] = 1.0  # enable loss for this specific action
          targets = np.array(targets).astype("float32")
          masks = np.array(masks).astype("float32")
  
          # Finally, perform a single update on the entire batch. We use a dummy target since
          # the actual loss is computed in a Lambda layer that needs more complex input. However,
          # it is still useful to know the actual target to compute metrics properly.
          ins = (
              [state0_batch]
              if type(self.model.input) is not list
              else state0_batch
          )
          metrics = self.trainable_model.train_on_batch(
              ins + [targets, masks], [dummy_targets, targets]
          )
          metrics = [
              metric
              for idx, metric in enumerate(metrics)
              if idx not in (1, 2)
          ]  # throw away individual losses
          metrics += self.policy.metrics
          if self.processor is not None:
              metrics += self.processor.metrics
  
      if (
          self.target_model_update >= 1
          and self.step % self.target_model_update == 0
      ):
          self.update_target_model_hard()
  
      return metrics
  
imports: from __future__ import division
  import warnings
  import keras.backend
  from keras.models import Model
  from keras.layers import Lambda, Input, Layer, Dense
  from rl.core import Agent
  from rl.policy import EpsGreedyQPolicy, GreedyQPolicy
  from rl.util import *
label_desc: Use image node_id 9 for calling the DQNAgent obj's underlying member method code with example usage: obj.backward(reward, terminal) and returns: metrics, metrics