code: def test_update_model(self):
      trainer = self._construct_trainer()
      # can't update empty model
      with self.assertRaises(AssertionError):
          trainer.update_model()
      # _update_model called with permutation of traj_buffer contents update_epoch # times
      trainer = self._construct_trainer(
          new_params=PPOTrainerParameters(
              ppo_batch_size=1,
              update_epochs=2,
              update_freq=2,
              normalize=False,
          )
      )
      trainer.traj_buffer = [1, 2]
      trainer._update_model = mock.Mock()
      trainer.update_model()
      calls = [
          mock.call([1]),
          mock.call([2]),
          mock.call([1]),
          mock.call([2]),
      ]
      trainer._update_model.assert_has_calls(calls, any_order=True)
      # trainer empties buffer
      self.assertEqual(trainer.traj_buffer, [])
  
      # _update_model
      trainer = self._construct_trainer()
      value_net_opt_mock = mock.Mock()
      ppo_opt_mock = mock.Mock()
      trainer.get_optimizers = mock.Mock(
          return_value=[value_net_opt_mock, ppo_opt_mock]
      )
      trainer._trajectory_to_losses = mock.Mock(
          side_effect=[
              {
                  "ppo_loss": torch.tensor(1),
                  "value_net_loss": torch.tensor(2),
              },
              {
                  "ppo_loss": torch.tensor(3),
                  "value_net_loss": torch.tensor(4),
              },
          ]
      )
      trainer.manual_backward = mock.Mock()
      inp1 = PolicyGradientInput.input_prototype(
          batch_size=1, action_dim=1, state_dim=1
      )
      inp2 = PolicyGradientInput.input_prototype(
          batch_size=1, action_dim=1, state_dim=1
      )
  
      trainer._update_model([inp1, inp2])
  
      trainer._trajectory_to_losses.assert_has_calls(
          [mock.call(inp1), mock.call(inp2)]
      )
      value_net_opt_mock.zero_grad.assert_called()
      value_net_opt_mock.step.assert_called()
  
      ppo_opt_mock.zero_grad.assert_called()
      ppo_opt_mock.step.assert_called()
  
      trainer.manual_backward.assert_has_calls(
          [mock.call(torch.tensor(6)), mock.call(torch.tensor(4))]
      )
  
imports: import unittest
  from collections import defaultdict
  from unittest import mock
  import torch
  from reagent.core.types import PolicyGradientInput
  from reagent.evaluation.evaluator import get_metrics_to_score
  from reagent.gym.policies.policy import Policy
  from reagent.gym.policies.samplers.discrete_sampler import SoftmaxActionSampler
  from reagent.models.dueling_q_network import DuelingQNetwork
  from reagent.models.fully_connected_network import FloatFeatureFullyConnected
  from reagent.training.parameters import PPOTrainerParameters
  from reagent.training.ppo_trainer import PPOTrainer
  from reagent.workflow.types import RewardOptions
label_desc: Use image node_id 8 for calling the TestPPO obj's underlying member method code with example usage: obj.test_update_model() without return types