class_docstr: Tests for `Retrace` ops.
class_docstr_tok: ["Tests","for","`","Retrace","`","ops","."]
docstr: Defines example data for, and an expected result of, Retrace operations.
  
  The example data comprises a minibatch of two sequences of four
  consecutive timesteps, allowing the data to be interpreted by Retrace
  as three successive transitions.
docstr_tok: ["Defines","example","data","for",",","and","an","expected","result","of",",","Retrace","operations",".","The","example","data","comprises","a","minibatch","of","two","sequences","of","four","consecutive","timesteps",",","allowing","the","data","to","be","interpreted","by","Retrace","as","three","successive","transitions","."]
code: def setUp(self):
      """Defines example data for, and an expected result of, Retrace operations.
  
      The example data comprises a minibatch of two sequences of four
      consecutive timesteps, allowing the data to be interpreted by Retrace
      as three successive transitions.
      """
      super(RetraceOpsTest, self).setUp()
  
      ### Example input data:
  
      self.lambda_ = 0.9
      self.qs = [
          [[2.2, 3.2, 4.2], [5.2, 6.2, 7.2]],
          [[7.2, 6.2, 5.2], [4.2, 3.2, 2.2]],
          [[3.2, 5.2, 7.2], [4.2, 6.2, 9.2]],
          [[2.2, 8.2, 4.2], [9.2, 1.2, 8.2]],
      ]
      self.targnet_qs = [
          [[2.0, 3.0, 4.0], [5.0, 6.0, 7.0]],
          [[7.0, 6.0, 5.0], [4.0, 3.0, 2.0]],
          [[3.0, 5.0, 7.0], [4.0, 6.0, 9.0]],
          [[2.0, 8.0, 4.0], [9.0, 1.0, 8.0]],
      ]
      self.actions = [[2, 0], [1, 2], [0, 1], [2, 0]]
      self.rewards = [
          [1.9, 2.9],
          [3.9, 4.9],
          [5.9, 6.9],
          [np.nan, np.nan],  # nan marks entries we should never use.
      ]
      self.pcontinues = [
          [0.8, 0.9],
          [0.7, 0.8],
          [0.6, 0.5],
          [np.nan, np.nan],
      ]
      self.target_policy_probs = [
          [[np.nan] * 3, [np.nan] * 3],
          [[0.41, 0.28, 0.31], [0.19, 0.77, 0.04]],
          [[0.22, 0.44, 0.34], [0.14, 0.25, 0.61]],
          [[0.16, 0.72, 0.12], [0.33, 0.30, 0.37]],
      ]
      self.behaviour_policy_probs = [
          [np.nan, np.nan],
          [0.85, 0.86],
          [0.87, 0.88],
          [0.89, 0.84],
      ]
  
      ### Expected results of Retrace as applied to the above:
  
      # NOTE: To keep the test code compact, we don't use the example data when
      # manually computing our expected results, but instead duplicate their
      # values explictly in those calculations. Some patterns in the values can
      # help you track who's who: for example, note that target network Q values
      # are integers, whilst learning network Q values all end in 0.2.
  
      # In a purely theoretical setting, we would compute the quantity we call
      # the "trace" using this recurrence relation:
      #
      #     ΔQ_tm1 = δ_tm1  +  λγπ(a_t | s_t)/μ(a_t | s_t) ⋅ ΔQ_t
      #      δ_tm1 = r_t + γ𝔼_π[Q(s_t, .)] - Q(s_tm1, a_tm1)
      #
      # In a target network setting, you might rewrite ΔQ_t as ΔQ'_t, indicating
      # that this value is the next-timestep trace as computed when all
      # Q(s_tm1, a_tm1) terms (in δ_t, δ_t+1, ...) come from the target network,
      # not the learning network.
      #
      # To generate our collection of expected outputs, we'll first compute
      # "ΔQ'_tm1" (the "target network trace") at all timesteps.
      #
      # We start at the end of the sequence and work backward, like the
      # implementation does.
      targ_trace = np.zeros((3, 2))
      targ_trace[2, 0] = (
          5.9 + 0.6 * (0.16 * 2 + 0.72 * 8 + 0.12 * 4) - 3
      )  # δ_tm1[2,0]
      targ_trace[2, 1] = (
          6.9 + 0.5 * (0.33 * 9 + 0.30 * 1 + 0.37 * 8) - 6
      )  # δ_tm1[2,1]
  
      targ_trace[1, 0] = (
          3.9
          + 0.7 * (0.22 * 3 + 0.44 * 5 + 0.34 * 7)
          - 6
          + 0.9 * 0.7 * 0.22 / 0.87 * targ_trace[2, 0]  # δ_tm1[1,0]
      )
      targ_trace[1, 1] = (
          4.9
          + 0.8 * (0.14 * 4 + 0.25 * 6 + 0.61 * 9)
          - 2
          + 0.9 * 0.8 * 0.25 / 0.88 * targ_trace[2, 1]  # δ_tm1[1,1]
      )
  
      targ_trace[0, 0] = (
          1.9
          + 0.8 * (0.41 * 7 + 0.28 * 6 + 0.31 * 5)
          - 4
          + 0.9 * 0.8 * 0.28 / 0.85 * targ_trace[1, 0]  # δ_tm1[0,0]
      )
      targ_trace[0, 1] = (
          2.9
          + 0.9 * (0.19 * 4 + 0.77 * 3 + 0.04 * 2)
          - 5
          + 0.9 * 0.9 * 0.04 / 0.86 * targ_trace[1, 1]  # δ_tm1[0,1]
      )
  
      # We can evaluate target Q values by adding targ_trace to single step
      # returns.
      target_q = np.zeros((3, 2))
      target_q[2, 0] = 5.9 + 0.6 * (0.16 * 2 + 0.72 * 8 + 0.12 * 4)
      target_q[2, 1] = 6.9 + 0.5 * (0.33 * 9 + 0.30 * 1 + 0.37 * 8)
  
      target_q[1, 0] = (
          3.9
          + 0.7 * (0.22 * 3 + 0.44 * 5 + 0.34 * 7)
          + 0.9 * 0.7 * 0.22 / 0.87 * targ_trace[2, 0]
      )
      target_q[1, 1] = (
          4.9
          + 0.8 * (0.14 * 4 + 0.25 * 6 + 0.61 * 9)
          + 0.9 * 0.8 * 0.25 / 0.88 * targ_trace[2, 1]
      )
  
      target_q[0, 0] = (
          1.9
          + 0.8 * (0.41 * 7 + 0.28 * 6 + 0.31 * 5)
          + 0.9 * 0.8 * 0.28 / 0.85 * targ_trace[1, 0]
      )
      target_q[0, 1] = (
          2.9
          + 0.9 * (0.19 * 4 + 0.77 * 3 + 0.04 * 2)
          + 0.9 * 0.9 * 0.04 / 0.86 * targ_trace[1, 1]
      )
  
      # Now we can compute the "official" trace (ΔQ_tm1), which involves the
      # learning network. The only difference from the "target network trace"
      # calculations is the Q(s_tm1, a_tm1) terms we use:
      trace = np.zeros((3, 2))  #    ↓ Q(s_tm1, a_tm1)
      trace[2, 0] = target_q[2, 0] - 3.2  # δ_tm1[2,0]
      trace[2, 1] = target_q[2, 1] - 6.2  # δ_tm1[2,1]
  
      trace[1, 0] = target_q[1, 0] - 6.2  # δ_tm1[1,0]
      trace[1, 1] = target_q[1, 1] - 2.2  # δ_tm1[1,1]
  
      trace[0, 0] = target_q[0, 0] - 4.2  # δ_tm1[0,0]
      trace[0, 1] = target_q[0, 1] - 5.2  # δ_tm1[0,0]
  
      self.expected_result = 0.5 * np.square(trace)
      self.target_q = target_q
  
file_path: trfl/trfl/retrace_ops_test.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import numpy
  from six.moves import xrange
  import tensorflow.compat.v1
  from trfl import retrace_ops
label_desc: Use image node_id 1 for calling the RetraceOpsTest obj's underlying member method code with example usage: obj.setUp() without return types