def build_act_with_param_noise(make_obs_ph, q_func, num_actions, scope="<STR_LIT>", reuse=None, param_noise_filter_func=None):
if param_noise_filter_func is None:<EOL><INDENT>param_noise_filter_func = default_param_noise_filter<EOL><DEDENT>with tf.variable_scope(scope, reuse=reuse):<EOL><INDENT>observations_ph = make_obs_ph("<STR_LIT>")<EOL>stochastic_ph = tf.placeholder(tf.bool, (), name="<STR_LIT>")<EOL>update_eps_ph = tf.placeholder(tf.float32, (), name="<STR_LIT>")<EOL>update_param_noise_threshold_ph = tf.placeholder(tf.float32, (), name="<STR_LIT>")<EOL>update_param_noise_scale_ph = tf.placeholder(tf.bool, (), name="<STR_LIT>")<EOL>reset_ph = tf.placeholder(tf.bool, (), name="<STR_LIT>")<EOL>eps = tf.get_variable("<STR_LIT>", (), initializer=tf.constant_initializer(<NUM_LIT:0>))<EOL>param_noise_scale = tf.get_variable("<STR_LIT>", (), initializer=tf.constant_initializer(<NUM_LIT>), trainable=False)<EOL>param_noise_threshold = tf.get_variable("<STR_LIT>", (), initializer=tf.constant_initializer(<NUM_LIT>), trainable=False)<EOL>q_values = q_func(observations_ph.get(), num_actions, scope="<STR_LIT>")<EOL>q_values_perturbed = q_func(observations_ph.get(), num_actions, scope="<STR_LIT>")<EOL>def perturb_vars(original_scope, perturbed_scope):<EOL><INDENT>all_vars = scope_vars(absolute_scope_name(original_scope))<EOL>all_perturbed_vars = scope_vars(absolute_scope_name(perturbed_scope))<EOL>assert len(all_vars) == len(all_perturbed_vars)<EOL>perturb_ops = []<EOL>for var, perturbed_var in zip(all_vars, all_perturbed_vars):<EOL><INDENT>if param_noise_filter_func(perturbed_var):<EOL><INDENT>op = tf.assign(perturbed_var, var + tf.random_normal(shape=tf.shape(var), mean=<NUM_LIT:0.>, stddev=param_noise_scale))<EOL><DEDENT>else:<EOL><INDENT>op = tf.assign(perturbed_var, var)<EOL><DEDENT>perturb_ops.append(op)<EOL><DEDENT>assert len(perturb_ops) == len(all_vars)<EOL>return tf.group(*perturb_ops)<EOL><DEDENT>q_values_adaptive = q_func(observations_ph.get(), num_actions, scope="<STR_LIT>")<EOL>perturb_for_adaption = perturb_vars(original_scope="<STR_LIT>", perturbed_scope="<STR_LIT>")<EOL>kl = tf.reduce_sum(tf.nn.softmax(q_values) * (tf.log(tf.nn.softmax(q_values)) - tf.log(tf.nn.softmax(q_values_adaptive))), axis=-<NUM_LIT:1>)<EOL>mean_kl = tf.reduce_mean(kl)<EOL>def update_scale():<EOL><INDENT>with tf.control_dependencies([perturb_for_adaption]):<EOL><INDENT>update_scale_expr = tf.cond(mean_kl < param_noise_threshold,<EOL>lambda: param_noise_scale.assign(param_noise_scale * <NUM_LIT>),<EOL>lambda: param_noise_scale.assign(param_noise_scale / <NUM_LIT>),<EOL>)<EOL><DEDENT>return update_scale_expr<EOL><DEDENT>update_param_noise_threshold_expr = param_noise_threshold.assign(tf.cond(update_param_noise_threshold_ph >= <NUM_LIT:0>,<EOL>lambda: update_param_noise_threshold_ph, lambda: param_noise_threshold))<EOL>deterministic_actions = tf.argmax(q_values_perturbed, axis=<NUM_LIT:1>)<EOL>batch_size = tf.shape(observations_ph.get())[<NUM_LIT:0>]<EOL>random_actions = tf.random_uniform(tf.stack([batch_size]), minval=<NUM_LIT:0>, maxval=num_actions, dtype=tf.int64)<EOL>chose_random = tf.random_uniform(tf.stack([batch_size]), minval=<NUM_LIT:0>, maxval=<NUM_LIT:1>, dtype=tf.float32) < eps<EOL>stochastic_actions = tf.where(chose_random, random_actions, deterministic_actions)<EOL>output_actions = tf.cond(stochastic_ph, lambda: stochastic_actions, lambda: deterministic_actions)<EOL>update_eps_expr = eps.assign(tf.cond(update_eps_ph >= <NUM_LIT:0>, lambda: update_eps_ph, lambda: eps))<EOL>updates = [<EOL>update_eps_expr,<EOL>tf.cond(reset_ph, lambda: perturb_vars(original_scope="<STR_LIT>", perturbed_scope="<STR_LIT>"), lambda: tf.group(*[])),<EOL>tf.cond(update_param_noise_scale_ph, lambda: update_scale(), lambda: tf.Variable(<NUM_LIT:0.>, trainable=False)),<EOL>update_param_noise_threshold_expr,<EOL>]<EOL>_act = U.function(inputs=[observations_ph, stochastic_ph, update_eps_ph, reset_ph, update_param_noise_threshold_ph, update_param_noise_scale_ph],<EOL>outputs=output_actions,<EOL>givens={update_eps_ph: -<NUM_LIT:1.0>, stochastic_ph: True, reset_ph: False, update_param_noise_threshold_ph: False, update_param_noise_scale_ph: False},<EOL>updates=updates)<EOL>def act(ob, reset=False, update_param_noise_threshold=False, update_param_noise_scale=False, stochastic=True, update_eps=-<NUM_LIT:1>):<EOL><INDENT>return _act(ob, stochastic, update_eps, reset, update_param_noise_threshold, update_param_noise_scale)<EOL><DEDENT>return act<EOL><DEDENT>
Creates the act function with support for parameter space noise exploration (https://arxiv.org/abs/1706.01905):

    Parameters
    ----------
    make_obs_ph: str -> tf.placeholder or TfInput
        a function that take a name and creates a placeholder of input with that name
    q_func: (tf.Variable, int, str, bool) -> tf.Variable
        the model that takes the following inputs:
            observation_in: object
                the output of observation placeholder
            num_actions: int
                number of actions
            scope: str
            reuse: bool
                should be passed to outer variable scope
        and returns a tensor of shape (batch_size, num_actions) with values of every action.
    num_actions: int
        number of actions.
    scope: str or VariableScope
        optional scope for variable_scope.
    reuse: bool or None
        whether or not the variables should be reused. To be able to reuse the scope must be given.
    param_noise_filter_func: tf.Variable -> bool
        function that decides whether or not a variable should be perturbed. Only applicable
        if param_noise is True. If set to None, default_param_noise_filter is used by default.

    Returns
    -------
    act: (tf.Variable, bool, float, bool, float, bool) -> tf.Variable
        function to select and action given observation.
`       See the top of the file for details.