lq/pgrl.py
def pg_vanilla_linpolicy(self, K0, N, batch_size, T, explore_mag=0.1,
                             step_size=0.1, beta1=0.9, beta2=0.999, epsilon=1.0e-8, safeguard=10):
        '''
        The vanilla policy gradient algorithm where the policy is considered to be linear. We use ADAM optimization.
        Different from the policy gradient, in each step rewards to go for that step is used.
        :param N: Number of rollouts
        :param batch_size: Number of stochastic gradient per minibatch
        :param T: Trajectory length
        :param explore_mag: Magnitude of the noise to explore
        :param step_size: The step size for ADAM optimization
        :param beta1: Beta1 in ADAM Optimization
        :param beta2: Beta2 in ADAM optimization
        :param epsilon: Epsilon in ADAM optimization
        :param safeguard: The maximum absolute value of entries of the controller gain
        :return: Controller gain by vanilla PG using ADAM
        '''
        # Initialize the controller
        Lin_gain = LinK(copy.copy(K0))

        # A heuristic baseline to decrease the variance
        Lin_gain.make_sampling_on(explore_mag)
        baseline = 0.0

        # Initializing Adam optimizer
        adam = ADAM(self.m, self.n, step_size=step_size, beta1=beta1, beta2=beta2, epsilon=epsilon)

        # start iteration
        for k in range(N):
            batch = np.zeros((self.m, self.n))
            reward = np.zeros(batch_size)

            # In each iteration, collect batches
            for j in range(batch_size):

                # Do one rollout
                states, actions, costs, _ = self.dyn.one_rollout(Lin_gain.sample_lin_policy, T)

                # Building the gradient of the loss with respect to gain
                actions_randomness = actions - Lin_gain.lin_policy(states)
                reward[j] = -np.sum(costs) / T
                for t in range(T):
                    reward_to_go = -np.sum(costs[t:]) / T
                    batch += (explore_mag ** (-2) * (reward_to_go - baseline) / batch_size) * \
                             np.outer(actions_randomness[t, :], states[t, :])

            # Update the baseline after collecting batches
            baseline = np.mean(reward)

            # Update the policy using ADAM after collecting batches in each iteration
            dK = adam.opt_onestep(batch)
            Lin_gain.K += dK
        return self.safeK(Lin_gain.K, safeguard)

        The vanilla policy gradient algorithm where the policy is considered to be linear. We use ADAM optimization.
        Different from the policy gradient, in each step rewards to go for that step is used.
        :param N: Number of rollouts
        :param batch_size: Number of stochastic gradient per minibatch
        :param T: Trajectory length
        :param explore_mag: Magnitude of the noise to explore
        :param step_size: The step size for ADAM optimization
        :param beta1: Beta1 in ADAM Optimization
        :param beta2: Beta2 in ADAM optimization
        :param epsilon: Epsilon in ADAM optimization
        :param safeguard: The maximum absolute value of entries of the controller gain
        :return: Controller gain by vanilla PG using ADAM
        
The vanilla policy gradient algorithm where the policy is considered to be linear. We use ADAM optimization.
Different from the policy gradient, in each step rewards to go for that step is used.
The vanilla policy gradient algorithm where the policy is considered to be linear . We use ADAM optimization . Different from the policy gradient in each step rewards to go for that step is used .
def pg_vanilla_linpolicy(self, K0, N, batch_size, T, explore_mag=0.1,
                             step_size=0.1, beta1=0.9, beta2=0.999, epsilon=1.0e-8, safeguard=10):
        Lin_gain = LinK(copy.copy(K0))
        Lin_gain.make_sampling_on(explore_mag)
        baseline = 0.0
        adam = ADAM(self.m, self.n, step_size=step_size, beta1=beta1, beta2=beta2, epsilon=epsilon)
        for k in range(N):
            batch = np.zeros((self.m, self.n))
            reward = np.zeros(batch_size)
            for j in range(batch_size):
                states, actions, costs, _ = self.dyn.one_rollout(Lin_gain.sample_lin_policy, T)
                actions_randomness = actions - Lin_gain.lin_policy(states)
                reward[j] = -np.sum(costs) / T
                for t in range(T):
                    reward_to_go = -np.sum(costs[t:]) / T
                    batch += (explore_mag ** (-2) * (reward_to_go - baseline) / batch_size) * \
                             np.outer(actions_randomness[t, :], states[t, :])
            baseline = np.mean(reward)
            dK = adam.opt_onestep(batch)
            Lin_gain.K += dK
        return self.safeK(Lin_gain.K, safeguard)
def pg_vanilla_linpolicy ( self , K0 , N , batch_size , T , explore_mag = 0.1 , step_size = 0.1 , beta1 = 0.9 , beta2 = 0.999 , epsilon = 1.0e-8 , safeguard = 10 ) : Lin_gain = LinK ( copy . copy ( K0 ) ) Lin_gain . make_sampling_on ( explore_mag ) baseline = 0.0 adam = ADAM ( self . m , self . n , step_size = step_size , beta1 = beta1 , beta2 = beta2 , epsilon = epsilon ) for k in range ( N ) : batch = np . zeros ( ( self . m , self . n ) ) reward = np . zeros ( batch_size ) for j in range ( batch_size ) : states , actions , costs , _ = self . dyn . one_rollout ( Lin_gain . sample_lin_policy , T ) actions_randomness = actions - Lin_gain . lin_policy ( states ) reward [ j ] = - np . sum ( costs ) / T for t in range ( T ) : reward_to_go = - np . sum ( costs [ t : ] ) / T batch += ( explore_mag ** ( - 2 ) * ( reward_to_go - baseline ) / batch_size ) * np . outer ( actions_randomness [ t , : ] , states [ t , : ] ) baseline = np . mean ( reward ) dK = adam . opt_onestep ( batch ) Lin_gain . K += dK return self . safeK ( Lin_gain . K , safeguard )
The vanilla policy gradient algorithm where the policy is considered to be linear.
The vanilla policy gradient algorithm where the policy is considered to be linear .
'''
        The vanilla policy gradient algorithm where the policy is considered to be linear. We use ADAM optimization.
        Different from the policy gradient, in each step rewards to go for that step is used.
        :param N: Number of rollouts
        :param batch_size: Number of stochastic gradient per minibatch
        :param T: Trajectory length
        :param explore_mag: Magnitude of the noise to explore
        :param step_size: The step size for ADAM optimization
        :param beta1: Beta1 in ADAM Optimization
        :param beta2: Beta2 in ADAM optimization
        :param epsilon: Epsilon in ADAM optimization
        :param safeguard: The maximum absolute value of entries of the controller gain
        :return: Controller gain by vanilla PG using ADAM
        ''' # Initialize the controller # A heuristic baseline to decrease the variance # Initializing Adam optimizer # start iteration # In each iteration, collect batches # Do one rollout # Building the gradient of the loss with respect to gain # Update the baseline after collecting batches # Update the policy using ADAM after collecting batches in each iteration
Controller gain by vanilla PG using ADAM
Controller gain by vanilla PG using ADAM
self
K0
N
Number of rollouts
Number of rollouts
batch_size
Number of stochastic gradient per minibatch
Number of stochastic gradient per minibatch
T
None
explore_mag
Magnitude of the noise to explore
Magnitude of the noise to explore
step_size
The step size for ADAM optimization
The step size for ADAM optimization
beta1
Beta1 in ADAM Optimization
Beta1 in ADAM Optimization
beta2
Beta2 in ADAM optimization
Beta2 in ADAM optimization
epsilon
Epsilon in ADAM optimization
Epsilon in ADAM optimization
safeguard
The maximum absolute value of entries of the controller gain
The maximum absolute value of entries of the controller gain