code: def double_qlearning(
      q_tm1,
      a_tm1,
      r_t,
      pcont_t,
      q_t_value,
      q_t_selector,
      name="DoubleQLearning",
  ):
      """Implements the double Q-learning loss as a TensorFlow op.
  
      The loss is `0.5` times the squared difference between `q_tm1[a_tm1]` and
      the target `r_t + pcont_t * q_t_value[argmax q_t_selector]`.
  
      See "Double Q-learning" by van Hasselt.
      (https://papers.nips.cc/paper/3964-double-q-learning.pdf).
  
      Args:
        q_tm1: Tensor holding Q-values for first timestep in a batch of
          transitions, shape `[B x num_actions]`.
        a_tm1: Tensor holding action indices, shape `[B]`.
        r_t: Tensor holding rewards, shape `[B]`.
        pcont_t: Tensor holding pcontinue values, shape `[B]`.
        q_t_value: Tensor of Q-values for second timestep in a batch of transitions,
          used to estimate the value of the best action, shape `[B x num_actions]`.
        q_t_selector: Tensor of Q-values for second timestep in a batch of
          transitions used to estimate the best action, shape `[B x num_actions]`.
        name: name to prefix ops created within this op.
  
      Returns:
        A namedtuple with fields:
  
        * `loss`: a tensor containing the batch of losses, shape `[B]`.
        * `extra`: a namedtuple with fields:
            * `target`: batch of target values for `q_tm1[a_tm1]`, shape `[B]`
            * `td_error`: batch of temporal difference errors, shape `[B]`
            * `best_action`: batch of greedy actions wrt `q_t_selector`, shape `[B]`
      """
      # Rank and compatibility checks.
      base_ops.wrap_rank_shape_assert(
          [[q_tm1, q_t_value, q_t_selector], [a_tm1, r_t, pcont_t]],
          [2, 1],
          name,
      )
  
      # double Q-learning op.
      with tf.name_scope(
          name,
          values=[q_tm1, a_tm1, r_t, pcont_t, q_t_value, q_t_selector],
      ):
          # Build target and select head to update.
          best_action = tf.argmax(q_t_selector, 1, output_type=tf.int32)
          double_q_bootstrapped = indexing_ops.batched_index(
              q_t_value, best_action
          )
          target = tf.stop_gradient(
              r_t + pcont_t * double_q_bootstrapped
          )
          qa_tm1 = indexing_ops.batched_index(q_tm1, a_tm1)
  
          # Temporal difference error and loss.
          # Loss is MSE scaled by 0.5, so the gradient is equal to the TD error.
          td_error = target - qa_tm1
          loss = 0.5 * tf.square(td_error)
          return base_ops.LossOutput(
              loss, DoubleQExtra(target, td_error, best_action)
          )
  
file_path: trfl/trfl/action_value_ops.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import tensorflow.compat.v1
  from trfl import base_ops
  from trfl import indexing_ops
  from trfl import sequence_ops
label_desc: Use image node_id 2 for calling a global function with example usage: double_qlearning(q_tm1, a_tm1, r_t, pcont_t, q_t_value, q_t_selector, name) and returns: base_ops