softqlearning/softqlearning/algorithms/sql.py
def _create_svgd_update(self):
        """Create a minimization operation for policy update (SVGD)."""

        actions = self.policy.actions_for(
            observations=self._observations_ph,
            n_action_samples=self._kernel_n_particles,
            reuse=True)
        assert_shape(actions,
                     [None, self._kernel_n_particles, self._action_dim])

        # SVGD requires computing two empirical expectations over actions
        # (see Appendix C1.1.). To that end, we first sample a single set of
        # actions, and later split them into two sets: `fixed_actions` are used
        # to evaluate the expectation indexed by `j` and `updated_actions`
        # the expectation indexed by `i`.
        n_updated_actions = int(
            self._kernel_n_particles * self._kernel_update_ratio)
        n_fixed_actions = self._kernel_n_particles - n_updated_actions

        fixed_actions, updated_actions = tf.split(
            actions, [n_fixed_actions, n_updated_actions], axis=1)
        fixed_actions = tf.stop_gradient(fixed_actions)
        assert_shape(fixed_actions, [None, n_fixed_actions, self._action_dim])
        assert_shape(updated_actions,
                     [None, n_updated_actions, self._action_dim])

        svgd_target_values = self.qf.output_for(
            self._observations_ph[:, None, :], fixed_actions, reuse=True)

        # Target log-density. Q_soft in Equation 13:
        squash_correction = tf.reduce_sum(
            tf.log(1 - fixed_actions**2 + EPS), axis=-1)
        log_p = svgd_target_values + squash_correction

        grad_log_p = tf.gradients(log_p, fixed_actions)[0]
        grad_log_p = tf.expand_dims(grad_log_p, axis=2)
        grad_log_p = tf.stop_gradient(grad_log_p)
        assert_shape(grad_log_p, [None, n_fixed_actions, 1, self._action_dim])

        kernel_dict = self._kernel_fn(xs=fixed_actions, ys=updated_actions)

        # Kernel function in Equation 13:
        kappa = tf.expand_dims(kernel_dict["output"], dim=3)
        assert_shape(kappa, [None, n_fixed_actions, n_updated_actions, 1])

        # Stein Variational Gradient in Equation 13:
        action_gradients = tf.reduce_mean(
            kappa * grad_log_p + kernel_dict["gradient"], reduction_indices=1)
        assert_shape(action_gradients,
                     [None, n_updated_actions, self._action_dim])

        # Propagate the gradient through the policy network (Equation 14).
        gradients = tf.gradients(
            updated_actions,
            self.policy.get_params_internal(),
            grad_ys=action_gradients)

        surrogate_loss = tf.reduce_sum([
            tf.reduce_sum(w * tf.stop_gradient(g))
            for w, g in zip(self.policy.get_params_internal(), gradients)
        ])

        if self._train_policy:
            optimizer = tf.train.AdamOptimizer(self._policy_lr)
            svgd_training_op = optimizer.minimize(
                loss=-surrogate_loss,
                var_list=self.policy.get_params_internal())
            self._training_ops.append(svgd_training_op)
Create a minimization operation for policy update (SVGD).
Create a minimization operation for policy update (SVGD).
Create a minimization operation for policy update ( SVGD ) .
def _create_svgd_update(self):
        actions = self.policy.actions_for(
            observations=self._observations_ph,
            n_action_samples=self._kernel_n_particles,
            reuse=True)
        assert_shape(actions,
                     [None, self._kernel_n_particles, self._action_dim])
        n_updated_actions = int(
            self._kernel_n_particles * self._kernel_update_ratio)
        n_fixed_actions = self._kernel_n_particles - n_updated_actions
        fixed_actions, updated_actions = tf.split(
            actions, [n_fixed_actions, n_updated_actions], axis=1)
        fixed_actions = tf.stop_gradient(fixed_actions)
        assert_shape(fixed_actions, [None, n_fixed_actions, self._action_dim])
        assert_shape(updated_actions,
                     [None, n_updated_actions, self._action_dim])
        svgd_target_values = self.qf.output_for(
            self._observations_ph[:, None, :], fixed_actions, reuse=True)
        squash_correction = tf.reduce_sum(
            tf.log(1 - fixed_actions**2 + EPS), axis=-1)
        log_p = svgd_target_values + squash_correction
        grad_log_p = tf.gradients(log_p, fixed_actions)[0]
        grad_log_p = tf.expand_dims(grad_log_p, axis=2)
        grad_log_p = tf.stop_gradient(grad_log_p)
        assert_shape(grad_log_p, [None, n_fixed_actions, 1, self._action_dim])
        kernel_dict = self._kernel_fn(xs=fixed_actions, ys=updated_actions)
        kappa = tf.expand_dims(kernel_dict["output"], dim=3)
        assert_shape(kappa, [None, n_fixed_actions, n_updated_actions, 1])
        action_gradients = tf.reduce_mean(
            kappa * grad_log_p + kernel_dict["gradient"], reduction_indices=1)
        assert_shape(action_gradients,
                     [None, n_updated_actions, self._action_dim])
        gradients = tf.gradients(
            updated_actions,
            self.policy.get_params_internal(),
            grad_ys=action_gradients)
        surrogate_loss = tf.reduce_sum([
            tf.reduce_sum(w * tf.stop_gradient(g))
            for w, g in zip(self.policy.get_params_internal(), gradients)
        ])
        if self._train_policy:
            optimizer = tf.train.AdamOptimizer(self._policy_lr)
            svgd_training_op = optimizer.minimize(
                loss=-surrogate_loss,
                var_list=self.policy.get_params_internal())
            self._training_ops.append(svgd_training_op)
def _create_svgd_update ( self ) : actions = self . policy . actions_for ( observations = self . _observations_ph , n_action_samples = self . _kernel_n_particles , reuse = True ) assert_shape ( actions , [ None , self . _kernel_n_particles , self . _action_dim ] ) n_updated_actions = int ( self . _kernel_n_particles * self . _kernel_update_ratio ) n_fixed_actions = self . _kernel_n_particles - n_updated_actions fixed_actions , updated_actions = tf . split ( actions , [ n_fixed_actions , n_updated_actions ] , axis = 1 ) fixed_actions = tf . stop_gradient ( fixed_actions ) assert_shape ( fixed_actions , [ None , n_fixed_actions , self . _action_dim ] ) assert_shape ( updated_actions , [ None , n_updated_actions , self . _action_dim ] ) svgd_target_values = self . qf . output_for ( self . _observations_ph [ : , None , : ] , fixed_actions , reuse = True ) squash_correction = tf . reduce_sum ( tf . log ( 1 - fixed_actions ** 2 + EPS ) , axis = - 1 ) log_p = svgd_target_values + squash_correction grad_log_p = tf . gradients ( log_p , fixed_actions ) [ 0 ] grad_log_p = tf . expand_dims ( grad_log_p , axis = 2 ) grad_log_p = tf . stop_gradient ( grad_log_p ) assert_shape ( grad_log_p , [ None , n_fixed_actions , 1 , self . _action_dim ] ) kernel_dict = self . _kernel_fn ( xs = fixed_actions , ys = updated_actions ) kappa = tf . expand_dims ( kernel_dict [ "output" ] , dim = 3 ) assert_shape ( kappa , [ None , n_fixed_actions , n_updated_actions , 1 ] ) action_gradients = tf . reduce_mean ( kappa * grad_log_p + kernel_dict [ "gradient" ] , reduction_indices = 1 ) assert_shape ( action_gradients , [ None , n_updated_actions , self . _action_dim ] ) gradients = tf . gradients ( updated_actions , self . policy . get_params_internal ( ) , grad_ys = action_gradients ) surrogate_loss = tf . reduce_sum ( [ tf . reduce_sum ( w * tf . stop_gradient ( g ) ) for w , g in zip ( self . policy . get_params_internal ( ) , gradients ) ] ) if self . _train_policy : optimizer = tf . train . AdamOptimizer ( self . _policy_lr ) svgd_training_op = optimizer . minimize ( loss = - surrogate_loss , var_list = self . policy . get_params_internal ( ) ) self . _training_ops . append ( svgd_training_op )
Create a minimization operation for policy update (SVGD).
Create a minimization operation for policy update ( SVGD ) .
"""Create a minimization operation for policy update (SVGD).""" # SVGD requires computing two empirical expectations over actions # (see Appendix C1.1.). To that end, we first sample a single set of # actions, and later split them into two sets: `fixed_actions` are used # to evaluate the expectation indexed by `j` and `updated_actions` # the expectation indexed by `i`. # Target log-density. Q_soft in Equation 13: # Kernel function in Equation 13: # Stein Variational Gradient in Equation 13: # Propagate the gradient through the policy network (Equation 14).
self