code: def testLossSequence(self, is_multi_actions, normalise_entropy):
      # A sequence of length 2, batch size 1, 3 possible actions.
      num_actions = 3
      policy_logits = [[[0.0, 0.0, 1.0]], [[0.0, 1.0, 0.0]]]
      actions = [[0], [1]]
      baseline_values = [[0.2], [0.3]]
      rewards = [[0.4], [0.5]]
      pcontinues = [[0.9], [0.8]]
      bootstrap_value = [0.1]
      baseline_cost = 0.15
      entropy_cost = 0.25
  
      if is_multi_actions:
          num_action_components = 3
          policy_logits_nest = [
              tf.constant(policy_logits, dtype=tf.float32)
              for _ in xrange(num_action_components)
          ]
          actions_nest = [
              tf.constant(actions, dtype=tf.int32)
              for _ in xrange(num_action_components)
          ]
      else:
          num_action_components = 1
          policy_logits_nest = tf.constant(
              policy_logits, dtype=tf.float32
          )
          actions_nest = tf.constant(actions, dtype=tf.int32)
  
      loss, extra = pg_ops.sequence_advantage_actor_critic_loss(
          policy_logits_nest,
          tf.constant(baseline_values, dtype=tf.float32),
          actions_nest,
          tf.constant(rewards, dtype=tf.float32),
          tf.constant(pcontinues, dtype=tf.float32),
          tf.constant(bootstrap_value, dtype=tf.float32),
          baseline_cost=baseline_cost,
          entropy_cost=entropy_cost,
          normalise_entropy=normalise_entropy,
      )
  
      # Manually calculate the discounted returns.
      return1 = 0.5 + 0.8 * 0.1
      return0 = 0.4 + 0.9 * return1
  
      with self.test_session() as sess:
          # Discounted returns
          self.assertAllClose(
              sess.run(extra.discounted_returns), [[return0], [return1]]
          )
  
          # Advantages
          advantages = [
              return0 - baseline_values[0][0],
              return1 - baseline_values[1][0],
          ]
          self.assertAllClose(
              sess.run(extra.advantages), [[adv] for adv in advantages]
          )
  
          # Baseline
          expected_baseline_loss = baseline_cost * sum(
              [0.5 * adv**2 for adv in advantages]
          )
          self.assertAllClose(
              sess.run(extra.baseline_loss), [expected_baseline_loss]
          )
  
          # Policy Gradient loss
          #   loss = sum_t(action_value*(-logits[action] +
          #                              log(sum_a(exp(logits[a])))))
          #
          # The below takes advantage of there only being one minibatch dim.
          normalise = lambda logits: np.log(np.exp(logits).sum())
          batch = 0
          expected_policy_gradient_loss = num_action_components * sum(
              [
                  advantages[0]
                  * (
                      -(policy_logits[0][batch][actions[0][batch]])
                      + normalise(policy_logits[0])
                  ),
                  advantages[1]
                  * (
                      -(policy_logits[1][batch][actions[1][batch]])
                      + normalise(policy_logits[1])
                  ),
              ]
          )
          self.assertAllClose(
              sess.run(extra.policy_gradient_loss),
              [expected_policy_gradient_loss],
          )
  
          # Entropy, calculated as per discrete_policy_entropy tests.
          expected_entropy = num_action_components * 0.97533 * 2
          expected_entropy_loss = -entropy_cost * expected_entropy
          if normalise_entropy:
              expected_entropy_loss /= num_action_components * np.log(
                  num_actions
              )
          self.assertAllClose(
              sess.run(extra.entropy), [expected_entropy], atol=1e-4
          )
          self.assertAllClose(
              sess.run(extra.entropy_loss),
              [expected_entropy_loss],
              atol=1e-4,
          )
  
          # Total loss
          expected_loss = [
              expected_entropy_loss
              + expected_policy_gradient_loss
              + expected_baseline_loss
          ]
          self.assertAllClose(sess.run(loss), expected_loss, atol=1e-4)
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from absl.testing import parameterized
  import numpy
  from six.moves import xrange
  import tensorflow.compat.v1
  import tree
  from trfl import discrete_policy_gradient_ops
label_desc: Use image node_id 1 for calling the SequenceAdvantageActorCriticLossTest obj's underlying member method code with example usage: obj.testLossSequence(is_multi_actions, normalise_entropy) without return types