code: def _test_seq2slate_trainer_on_policy(
      self, policy_gradient_interval, output_arch, device
  ):
      batch_size = 32
      state_dim = 2
      candidate_num = 15
      candidate_dim = 4
      hidden_size = 16
      learning_rate = 1.0
      on_policy = True
      rank_seed = 111
      seq2slate_params = Seq2SlateParameters(on_policy=on_policy)
  
      seq2slate_net = create_seq2slate_transformer(
          state_dim,
          candidate_num,
          candidate_dim,
          hidden_size,
          output_arch,
      ).to(device)
      seq2slate_net_copy = copy.deepcopy(seq2slate_net).to(device)
      seq2slate_net_copy_copy = copy.deepcopy(seq2slate_net).to(device)
      trainer = create_trainer(
          seq2slate_net,
          learning_rate,
          seq2slate_params,
          policy_gradient_interval,
      )
      batch = create_on_policy_batch(
          seq2slate_net,
          batch_size,
          state_dim,
          candidate_num,
          candidate_dim,
          rank_seed,
          device,
      )
      training_data = DataLoader([batch], collate_fn=lambda x: x[0])
      pl_trainer = pl.Trainer(
          max_epochs=policy_gradient_interval,
          gpus=None if device == torch.device("cpu") else 1,
          logger=False,
      )
      pl_trainer.fit(trainer, training_data)
      seq2slate_net = trainer.seq2slate_net.to(device)
  
      # manual compute gradient
      torch.manual_seed(rank_seed)
      rank_output = seq2slate_net_copy(
          batch,
          mode=Seq2SlateMode.RANK_MODE,
          tgt_seq_len=candidate_num,
          greedy=False,
      )
      loss = -(
          torch.mean(
              torch.log(rank_output.ranked_per_seq_probs)
              * batch.slate_reward
          )
      )
      loss.backward()
      self.assert_correct_gradient(
          seq2slate_net_copy,
          seq2slate_net,
          policy_gradient_interval,
          learning_rate,
      )
  
      # another way to compute gradient manually
      torch.manual_seed(rank_seed)
      ranked_per_seq_probs = seq2slate_net_copy_copy(
          batch,
          mode=Seq2SlateMode.RANK_MODE,
          tgt_seq_len=candidate_num,
          greedy=False,
      ).ranked_per_seq_probs
      loss = -(
          torch.mean(
              ranked_per_seq_probs
              / ranked_per_seq_probs.detach()
              * batch.slate_reward
          )
      )
      loss.backward()
      self.assert_correct_gradient(
          seq2slate_net_copy_copy,
          seq2slate_net,
          policy_gradient_interval,
          learning_rate,
      )
  
imports: import copy
  import itertools
  import logging
  import random
  import unittest
  from itertools import permutations
  import numpy
  import numpy.testing
  import pytorch_lightning
  import reagent.core.types
  import torch
  from parameterized import parameterized
  from reagent.core.parameters import Seq2SlateParameters
  from reagent.core.parameters_seq2slate import IPSClamp, IPSClampMethod
  from reagent.model_utils.seq2slate_utils import Seq2SlateOutputArch
  from reagent.models.seq2slate import Seq2SlateMode, Seq2SlateTransformerNet
  from reagent.optimizer.union import classes, Optimizer__Union
  from reagent.samplers.frechet import FrechetSort
  from reagent.training.ranking.helper import ips_clamp
  from reagent.training.ranking.seq2slate_trainer import Seq2SlateTrainer
  from torch.utils.data import DataLoader
label_desc: Use image node_id 6 for calling the TestSeq2SlateTrainer obj's underlying member method code with example usage: obj._test_seq2slate_trainer_on_policy(policy_gradient_interval, output_arch, device) without return types