softqlearning/softqlearning/algorithms/sql.py
def _create_td_update(self):
        """Create a minimization operation for Q-function update."""

        with tf.variable_scope('target'):
            # The value of the next state is approximated with uniform samples.
            target_actions = tf.random_uniform(
                (1, self._value_n_particles, self._action_dim), -1, 1)
            q_value_targets = self.qf.output_for(
                observations=self._next_observations_ph[:, None, :],
                actions=target_actions)
            assert_shape(q_value_targets, [None, self._value_n_particles])

        self._q_values = self.qf.output_for(
            self._observations_ph, self._actions_pl, reuse=True)
        assert_shape(self._q_values, [None])

        # Equation 10:
        next_value = tf.reduce_logsumexp(q_value_targets, axis=1)
        assert_shape(next_value, [None])

        # Importance weights add just a constant to the value.
        next_value -= tf.log(tf.cast(self._value_n_particles, tf.float32))
        next_value += self._action_dim * np.log(2)

        # \hat Q in Equation 11:
        ys = tf.stop_gradient(self._reward_scale * self._rewards_pl + (
            1 - self._terminals_pl) * self._discount * next_value)
        assert_shape(ys, [None])

        # Equation 11:
        bellman_residual = 0.5 * tf.reduce_mean((ys - self._q_values)**2)

        if self._train_qf:
            td_train_op = tf.train.AdamOptimizer(self._qf_lr).minimize(
                loss=bellman_residual, var_list=self.qf.get_params_internal())
            self._training_ops.append(td_train_op)

        self._bellman_residual = bellman_residual
Create a minimization operation for Q-function update.
Create a minimization operation for Q-function update.
Create a minimization operation for Q - function update .
def _create_td_update(self):
        with tf.variable_scope('target'):
            target_actions = tf.random_uniform(
                (1, self._value_n_particles, self._action_dim), -1, 1)
            q_value_targets = self.qf.output_for(
                observations=self._next_observations_ph[:, None, :],
                actions=target_actions)
            assert_shape(q_value_targets, [None, self._value_n_particles])
        self._q_values = self.qf.output_for(
            self._observations_ph, self._actions_pl, reuse=True)
        assert_shape(self._q_values, [None])
        next_value = tf.reduce_logsumexp(q_value_targets, axis=1)
        assert_shape(next_value, [None])
        next_value -= tf.log(tf.cast(self._value_n_particles, tf.float32))
        next_value += self._action_dim * np.log(2)
        ys = tf.stop_gradient(self._reward_scale * self._rewards_pl + (
            1 - self._terminals_pl) * self._discount * next_value)
        assert_shape(ys, [None])
        bellman_residual = 0.5 * tf.reduce_mean((ys - self._q_values)**2)
        if self._train_qf:
            td_train_op = tf.train.AdamOptimizer(self._qf_lr).minimize(
                loss=bellman_residual, var_list=self.qf.get_params_internal())
            self._training_ops.append(td_train_op)
        self._bellman_residual = bellman_residual
def _create_td_update ( self ) : with tf . variable_scope ( 'target' ) : target_actions = tf . random_uniform ( ( 1 , self . _value_n_particles , self . _action_dim ) , - 1 , 1 ) q_value_targets = self . qf . output_for ( observations = self . _next_observations_ph [ : , None , : ] , actions = target_actions ) assert_shape ( q_value_targets , [ None , self . _value_n_particles ] ) self . _q_values = self . qf . output_for ( self . _observations_ph , self . _actions_pl , reuse = True ) assert_shape ( self . _q_values , [ None ] ) next_value = tf . reduce_logsumexp ( q_value_targets , axis = 1 ) assert_shape ( next_value , [ None ] ) next_value -= tf . log ( tf . cast ( self . _value_n_particles , tf . float32 ) ) next_value += self . _action_dim * np . log ( 2 ) ys = tf . stop_gradient ( self . _reward_scale * self . _rewards_pl + ( 1 - self . _terminals_pl ) * self . _discount * next_value ) assert_shape ( ys , [ None ] ) bellman_residual = 0.5 * tf . reduce_mean ( ( ys - self . _q_values ) ** 2 ) if self . _train_qf : td_train_op = tf . train . AdamOptimizer ( self . _qf_lr ) . minimize ( loss = bellman_residual , var_list = self . qf . get_params_internal ( ) ) self . _training_ops . append ( td_train_op ) self . _bellman_residual = bellman_residual
Create a minimization operation for Q-function update.
Create a minimization operation for Q - function update .
"""Create a minimization operation for Q-function update.""" # The value of the next state is approximated with uniform samples. # Equation 10: # Importance weights add just a constant to the value. # \hat Q in Equation 11: # Equation 11:
self