code: def test_rl_synthetic_tuning():
      def eval_fn(sample):
          a, b, c = sample["a"], sample["b"], sample["c"]
          return -(a - 1) * (a - 5) + b[0] - b[1] + sum(c)
  
      search_space = MutableDict(
          {
              "a": Categorical([5, 3, 1], label="a"),
              "b": MutableList(
                  [
                      Categorical([-1, 1, 3, 5], label="b1"),
                      Categorical([1, 0], label="b2"),
                  ]
              ),
              "c": CategoricalMultiple([1, 2], label="c"),
          }
      )
  
      best = max(map(eval_fn, search_space.grid()))
  
      replay_buffer = ReplayBuffer(1000)
      generator = TuningTrajectoryGenerator(
          search_space,
          partial(default_policy_fn, hidden_dim=16, lr=1e-3),
      )
      policy = generator.policy
  
      reward_curve = []
      for iter in range(100):
          rewards = []
          for _ in range(20):
              sample = generator.next_sample()
              sample_logits = generator.sample_logits
              reward = eval_fn(search_space.freeze(sample))
              rewards.append(reward)
              trajectory = generator.send_reward(reward)
              replay_buffer.update(trajectory)
  
          if iter > 10 and all(
              np.max(logits) > 0.9 for logits in sample_logits.values()
          ):
              break
  
          policy.update(0, replay_buffer, batch_size=10, repeat=10)
          reward_curve.append(np.mean(rewards))
      else:
          assert False, f"Failed to converge."
  
      assert _has_upward_trend(reward_curve)
  
      rewards = []
      for _ in range(5):
          sample = generator.next_sample()
          reward = eval_fn(search_space.freeze(sample))
          rewards.append(reward)
          if reward == best:
              break
      else:
          assert False, f"Cannot find the best sample, found: {rewards}"
  
file_path: nni/test/algo/nas/strategy/test_rl_impl.py
imports: from functools import partial
  import numpy
  import pytest
  from tianshou.data import ReplayBuffer, Batch
  from tianshou.policy import BasePolicy
  from nni.mutable import Categorical, Numerical, CategoricalMultiple, MutableDict, MutableList, ExpressionConstraint
  from nni.nas.strategy._rl_impl import TuningTrajectoryGenerator, default_policy_fn
label_desc: Use image node_id 4 for calling a global function with example usage: test_rl_synthetic_tuning() and returns: unknown