class_docstr: Encapsulates fields and methods for RL policy and value function estimation with shared parameters
code: def _build_model_for_step(self):
      # multiply with weight and apply mask on self.act_latent to generate
      self.act_step = step = tf.placeholder(
          shape=(), dtype=tf.int64, name="act_step"
      )
      with tf.variable_scope("pi", reuse=tf.AUTO_REUSE):
          from .util import ortho_init
  
          nin = self.act_latent.get_shape()[1].value
          w = tf.get_variable(
              "w", [nin, self.nh], initializer=ortho_init(0.01)
          )
          b = tf.get_variable(
              "b", [self.nh], initializer=tf.constant_initializer(0.0)
          )
          logits = tf.matmul(self.act_latent, w) + b
          piece = tf.slice(self.mask, [step, 0], [1, self.nh])
          re_piece = tf.reshape(piece, [-1])
          masked_logits = tf.math.multiply(logits, re_piece)
  
          npinf_piece = tf.slice(
              self.mask_npinf, [step, 0], [1, self.nh]
          )
          re_npinf_piece = tf.reshape(npinf_piece, [-1])
  
      def sample(logits, mask_npinf):
          new_logits = tf.math.add(logits, mask_npinf)
          u = tf.random_uniform(
              tf.shape(new_logits), dtype=logits.dtype
          )
          return tf.argmax(new_logits - tf.log(-1 * tf.log(u)), axis=-1)
  
      def neglogp(logits, x):
          # return tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=x)
          # Note: we can't use sparse_softmax_cross_entropy_with_logits because
          #       the implementation does not allow second-order derivatives...
          if x.dtype in {tf.uint8, tf.int32, tf.int64}:
              # one-hot encoding
              x_shape_list = x.shape.as_list()
              logits_shape_list = logits.get_shape().as_list()[:-1]
              for xs, ls in zip(x_shape_list, logits_shape_list):
                  if xs is not None and ls is not None:
                      assert (
                          xs == ls
                      ), "shape mismatch: {} in x vs {} in logits".format(
                          xs, ls
                      )
  
              x = tf.one_hot(x, logits.get_shape().as_list()[-1])
          else:
              # already encoded
              assert x.shape.as_list() == logits.shape.as_list()
  
          return tf.nn.softmax_cross_entropy_with_logits_v2(
              logits=logits, labels=x
          )
  
      self.act_action = sample(masked_logits, re_npinf_piece)
      self.act_neglogp = neglogp(masked_logits, self.act_action)
  
file_path: nni/nni/algorithms/hpo/ppo_tuner/policy.py
imports: import tensorflow.compat.v1
  from .distri import CategoricalPdType
  from .util import lstm_model, fc, observation_placeholder, adjust_shape
label_desc: Use image node_id 3 for calling the PolicyWithValue obj's underlying member method code with example usage: obj._build_model_for_step() and returns: tf, tf