code: def build_act_with_param_noise(
      q_func,
      ob_space,
      ac_space,
      stochastic_ph,
      update_eps_ph,
      sess,
      param_noise_filter_func=None,
  ):
      """
      Creates the act function with support for parameter space noise exploration (https://arxiv.org/abs/1706.01905):
  
      :param q_func: (DQNPolicy) the policy
      :param ob_space: (Gym Space) The observation space of the environment
      :param ac_space: (Gym Space) The action space of the environment
      :param stochastic_ph: (TensorFlow Tensor) the stochastic placeholder
      :param update_eps_ph: (TensorFlow Tensor) the update_eps placeholder
      :param sess: (TensorFlow session) The current TensorFlow session
      :param param_noise_filter_func: (function (TensorFlow Tensor): bool) function that decides whether or not a
          variable should be perturbed. Only applicable if param_noise is True. If set to None, default_param_noise_filter
          is used by default.
      :return: (function (TensorFlow Tensor, bool, float): TensorFlow Tensor, (TensorFlow Tensor, TensorFlow Tensor)
          act function to select and action given observation (See the top of the file for details),
          A tuple containing the observation placeholder and the processed observation placeholder respectively.
      """
      if param_noise_filter_func is None:
          param_noise_filter_func = default_param_noise_filter
  
      update_param_noise_threshold_ph = tf.placeholder(
          tf.float32, (), name="update_param_noise_threshold"
      )
      update_param_noise_scale_ph = tf.placeholder(
          tf.bool, (), name="update_param_noise_scale"
      )
      reset_ph = tf.placeholder(tf.bool, (), name="reset")
  
      eps = tf.get_variable(
          "eps", (), initializer=tf.constant_initializer(0)
      )
      param_noise_scale = tf.get_variable(
          "param_noise_scale",
          (),
          initializer=tf.constant_initializer(0.01),
          trainable=False,
      )
      param_noise_threshold = tf.get_variable(
          "param_noise_threshold",
          (),
          initializer=tf.constant_initializer(0.05),
          trainable=False,
      )
  
      # Unmodified Q.
      policy = q_func(sess, ob_space, ac_space, 1, 1, None)
      obs_phs = (policy.obs_ph, policy.processed_obs)
  
      # Perturbable Q used for the actual rollout.
      with tf.variable_scope("perturbed_model", reuse=False):
          perturbable_policy = q_func(
              sess, ob_space, ac_space, 1, 1, None, obs_phs=obs_phs
          )
  
      def perturb_vars(original_scope, perturbed_scope):
          """
          We have to wrap this code into a function due to the way tf.cond() works.
  
          See https://stackoverflow.com/questions/37063952/confused-by-the-behavior-of-tf-cond for a more detailed
          discussion.
  
          :param original_scope: (str or VariableScope) the original scope.
          :param perturbed_scope: (str or VariableScope) the perturbed scope.
          :return: (TensorFlow Operation)
          """
          all_vars = scope_vars(absolute_scope_name(original_scope))
          all_perturbed_vars = scope_vars(
              absolute_scope_name(perturbed_scope)
          )
          assert len(all_vars) == len(all_perturbed_vars)
          perturb_ops = []
          for var, perturbed_var in zip(all_vars, all_perturbed_vars):
              if param_noise_filter_func(perturbed_var):
                  # Perturb this variable.
                  operation = tf.assign(
                      perturbed_var,
                      var
                      + tf.random_normal(
                          shape=tf.shape(var),
                          mean=0.0,
                          stddev=param_noise_scale,
                      ),
                  )
              else:
                  # Do not perturb, just assign.
                  operation = tf.assign(perturbed_var, var)
              perturb_ops.append(operation)
          assert len(perturb_ops) == len(all_vars)
          return tf.group(*perturb_ops)
  
      # Set up functionality to re-compute `param_noise_scale`. This perturbs yet another copy
      # of the network and measures the effect of that perturbation in action space. If the perturbation
      # is too big, reduce scale of perturbation, otherwise increase.
      with tf.variable_scope("adaptive_model", reuse=False):
          adaptive_policy = q_func(
              sess, ob_space, ac_space, 1, 1, None, obs_phs=obs_phs
          )
      perturb_for_adaption = perturb_vars(
          original_scope="model", perturbed_scope="adaptive_model/model"
      )
      kl_loss = tf.reduce_sum(
          tf.nn.softmax(policy.q_values)
          * (
              tf.log(tf.nn.softmax(policy.q_values))
              - tf.log(tf.nn.softmax(adaptive_policy.q_values))
          ),
          axis=-1,
      )
      mean_kl = tf.reduce_mean(kl_loss)
  
      def update_scale():
          """
          update the scale expression
  
          :return: (TensorFlow Tensor) the updated scale expression
          """
          with tf.control_dependencies([perturb_for_adaption]):
              update_scale_expr = tf.cond(
                  mean_kl < param_noise_threshold,
                  lambda: param_noise_scale.assign(
                      param_noise_scale * 1.01
                  ),
                  lambda: param_noise_scale.assign(
                      param_noise_scale / 1.01
                  ),
              )
          return update_scale_expr
  
      # Functionality to update the threshold for parameter space noise.
      update_param_noise_thres_expr = param_noise_threshold.assign(
          tf.cond(
              update_param_noise_threshold_ph >= 0,
              lambda: update_param_noise_threshold_ph,
              lambda: param_noise_threshold,
          )
      )
  
      # Put everything together.
      perturbed_deterministic_actions = tf.argmax(
          perturbable_policy.q_values, axis=1
      )
      deterministic_actions = tf.argmax(policy.q_values, axis=1)
      batch_size = tf.shape(policy.obs_ph)[0]
      n_actions = (
          ac_space.nvec
          if isinstance(ac_space, MultiDiscrete)
          else ac_space.n
      )
      random_actions = tf.random_uniform(
          tf.stack([batch_size]),
          minval=0,
          maxval=n_actions,
          dtype=tf.int64,
      )
      chose_random = (
          tf.random_uniform(
              tf.stack([batch_size]),
              minval=0,
              maxval=1,
              dtype=tf.float32,
          )
          < eps
      )
      perturbed_stochastic_actions = tf.where(
          chose_random, random_actions, perturbed_deterministic_actions
      )
      stochastic_actions = tf.where(
          chose_random, random_actions, deterministic_actions
      )
  
      perturbed_output_actions = tf.cond(
          stochastic_ph,
          lambda: perturbed_stochastic_actions,
          lambda: deterministic_actions,
      )
      output_actions = tf.cond(
          stochastic_ph,
          lambda: stochastic_actions,
          lambda: deterministic_actions,
      )
      update_eps_expr = eps.assign(
          tf.cond(
              update_eps_ph >= 0, lambda: update_eps_ph, lambda: eps
          )
      )
      updates = [
          update_eps_expr,
          tf.cond(
              reset_ph,
              lambda: perturb_vars(
                  original_scope="model",
                  perturbed_scope="perturbed_model/model",
              ),
              lambda: tf.group(*[]),
          ),
          tf.cond(
              update_param_noise_scale_ph,
              lambda: update_scale(),
              lambda: tf.Variable(0.0, trainable=False),
          ),
          update_param_noise_thres_expr,
      ]
  
      _act = tf_util.function(
          inputs=[policy.obs_ph, stochastic_ph, update_eps_ph],
          outputs=output_actions,
          givens={update_eps_ph: -1.0, stochastic_ph: True},
          updates=[update_eps_expr],
      )
  
      _perturbed_act = tf_util.function(
          inputs=[
              policy.obs_ph,
              stochastic_ph,
              update_eps_ph,
              reset_ph,
              update_param_noise_threshold_ph,
              update_param_noise_scale_ph,
          ],
          outputs=perturbed_output_actions,
          givens={
              update_eps_ph: -1.0,
              stochastic_ph: True,
              reset_ph: False,
              update_param_noise_threshold_ph: False,
              update_param_noise_scale_ph: False,
          },
          updates=updates,
      )
  
      def act(
          obs,
          reset=None,
          update_param_noise_threshold=None,
          update_param_noise_scale=None,
          stochastic=True,
          update_eps=-1,
      ):
          """
          get the action from the current observation
  
          :param obs: (Any) Observation that can be feed into the output of make_obs_ph
          :param reset: (bool) reset the perturbed policy by sampling a new perturbation
          :param update_param_noise_threshold: (float) the desired threshold for the difference between
              non-perturbed and perturbed policy
          :param update_param_noise_scale: (bool) whether or not to update the scale of the noise for the next time
              it is re-perturbed
          :param stochastic: (bool) if set to False all the actions are always deterministic (default False)
          :param update_eps: (float) update epsilon a new value, if negative not update happens
              (default: no update)
          :return: (TensorFlow Tensor) tensor of dtype tf.int64 and shape (BATCH_SIZE,) with an action to be
              performed for every element of the batch.
          """
          if (
              reset is None
              or update_param_noise_threshold is None
              or update_param_noise_scale is None
          ):
              return _act(obs, stochastic, update_eps)
          else:
              return _perturbed_act(
                  obs,
                  stochastic,
                  update_eps,
                  reset,
                  update_param_noise_threshold,
                  update_param_noise_scale,
              )
  
      return act, obs_phs
  
file_path: stable-baselines/stable_baselines/deepq/build_graph.py
imports: import tensorflow
  from gym.spaces import MultiDiscrete
  from stable_baselines.common import tf_util
label_desc: Use image node_id 6 for calling a global function with example usage: build_act_with_param_noise(q_func, ob_space, ac_space, stochastic_ph, update_eps_ph, sess, param_noise_filter_func) and returns: act,  obs_phs, tf, update_scale_expr, _act, _perturbed_act