code: def retrace_core(
      lambda_,
      q_tm1,
      a_tm1,
      r_t,
      pcont_t,
      target_policy_t,
      behaviour_policy_t,
      targnet_q_t,
      a_t,
      stop_targnet_gradients=True,
      name=None,
  ):
      """Retrace algorithm core loss calculation op.
  
      Given a minibatch of temporally-contiguous sequences of Q values, policy
      probabilities, and various other typical RL algorithm inputs, this
      Op creates a subgraph that computes a loss according to the
      Retrace multi-step off-policy value learning algorithm. This Op supports the
      use of target networks, but does not require them.
  
      This function is the "core" Retrace op only because its arguments are less
      user-friendly and more implementation-convenient. For a more user-friendly
      operator, consider using `retrace`. For more details of Retrace, refer to
      [the arXiv paper](http://arxiv.org/abs/1606.02647).
  
      Construct the "core" retrace loss subgraph for a batch of sequences.
  
      Note that two pairs of arguments (one holding target network values; the
      other, actions) are temporally-offset versions of each other and will share
      many values in common (nb: a good setting for using `IndexedSlices`). *This
      op does not include any checks that these pairs of arguments are
      consistent*---that is, it does not ensure that temporally-offset
      arguments really do share the values they are supposed to share.
  
      In argument descriptions, `T` counts the number of transitions over which
      the Retrace loss is computed, and `B` is the minibatch size. All tensor
      arguments are indexed first by transition, with specific details of this
      indexing in the argument descriptions (pay close attention to "subscripts"
      in variable names).
  
      Args:
        lambda_: Positive scalar value or 0-D `Tensor` controlling the degree to
          which future timesteps contribute to the loss computed at each
          transition.
        q_tm1: 3-D tensor holding per-action Q-values for the states encountered
          just before taking the transitions that correspond to each major index.
          Since these values are the predicted values we wish to update (in other
          words, the values we intend to change as we learn), in a target network
          setting, these nearly always come from the "non-target" network, which
          we usually call the "learning network".
          Shape is `[T, B, num_actions]`.
        a_tm1: 2-D tensor holding the indices of actions executed during the
          transition that corresponds to each major index.
          Shape is `[T, B]`.
        r_t: 2-D tensor holding rewards received during the transition
          that corresponds to each major index.
          Shape is `[T, B]`.
        pcont_t: 2-D tensor holding pcontinue values received during the
          transition that corresponds to each major index.
          Shape is `[T, B]`.
        target_policy_t: 3-D tensor holding per-action policy probabilities for
          the states encountered just AFTER the transitions that correspond to
          each major index, according to the target policy (i.e. the policy we
          wish to learn). These usually derive from the learning net.
          Shape is `[T, B, num_actions]`.
        behaviour_policy_t: 2-D tensor holding the *behaviour* policy's
          probabilities of having taken action `a_t` at the states encountered
          just AFTER the transitions that correspond to each major index. Derived
          from whatever policy you used to generate the data. All values MUST be
          greater that 0. Shape is `[T, B]`.
        targnet_q_t: 3-D tensor holding per-action Q-values for the states
          encountered just AFTER taking the transitions that correspond to each
          major index. Since these values are used to calculate target values for
          the network, in a target in a target network setting, these should
          probably come from the target network.
          Shape is `[T, B, num_actions]`.
        a_t: 2-D tensor holding the indices of actions executed during the
          transition AFTER the transition that corresponds to each major index.
          Shape is `[T, B]`.
        stop_targnet_gradients: `bool` that enables a sensible default way of
          handling gradients through the Retrace op (essentially, gradients
          are not permitted to involve the `targnet_q_t` input).
          Can be disabled if you require a different arragement, but
          you'll probably want to block some gradients somewhere.
        name: name to prefix ops created by this function.
  
      Returns:
        A namedtuple with fields:
  
        * `loss`: Tensor containing the batch of losses, shape `[B]`.
        * `extra`: A namedtuple with fields:
            * `retrace_weights`: Tensor containing batch of retrace weights,
            shape `[T, B]`.
            * `target`: Tensor containing target action values, shape `[T, B]`.
      """
      all_args = [
          lambda_,
          q_tm1,
          a_tm1,
          r_t,
          pcont_t,
          target_policy_t,
          behaviour_policy_t,
          targnet_q_t,
          a_t,
      ]
  
      with tf.name_scope(name, "RetraceCore", all_args):
          (
              lambda_,
              q_tm1,
              a_tm1,
              r_t,
              pcont_t,
              target_policy_t,
              behaviour_policy_t,
              targnet_q_t,
              a_t,
          ) = (tf.convert_to_tensor(arg) for arg in all_args)
  
          # Evaluate importance weights.
          c_t = (
              _retrace_weights(
                  indexing_ops.batched_index(target_policy_t, a_t),
                  behaviour_policy_t,
              )
              * lambda_
          )
          # Targets are evaluated by using only Q values from the target network.
          # This provides fixed regression targets until the next target network
          # update.
          target = _general_off_policy_corrected_multistep_target(
              r_t,
              pcont_t,
              target_policy_t,
              c_t,
              targnet_q_t,
              a_t,
              not stop_targnet_gradients,
          )
  
          if stop_targnet_gradients:
              target = tf.stop_gradient(target)
          # Regress Q values of the learning network towards the targets evaluated
          # by using the target network.
          qa_tm1 = indexing_ops.batched_index(q_tm1, a_tm1)
          delta = target - qa_tm1
          loss = 0.5 * tf.square(delta)
  
          return base_ops.LossOutput(
              loss, RetraceCoreExtra(retrace_weights=c_t, target=target)
          )
  
file_path: trfl/trfl/retrace_ops.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  from six.moves import zip
  import tensorflow.compat.v1
  from trfl import base_ops
  from trfl import indexing_ops
  from trfl import sequence_ops
label_desc: Use image node_id 4 for calling a global function with example usage: retrace_core(lambda_, q_tm1, a_tm1, r_t, pcont_t, target_policy_t, behaviour_policy_t, targnet_q_t, a_t, stop_targnet_gradients, name) and returns: base_ops