lq/pgrl.py
def pg_linpolicy(self, K0, N, batch_size, T, explore_mag=0.1,
                     step_size=0.1, beta1=0.9, beta2=0.999, epsilon=1.0e-8, safeguard=10):
        '''
        The policy gradient algorithm where the policy is considered to be linear. We use ADAM optimization.
        :param K0: The initial controller gain
        :param N: Number of iterations
        :param batch_size: number of batches per each step of optimization
        :param T: Trajectory length
        :param explore_mag: Magnitude of the noise to explore
        :param step_size: learning rate for Adam
        :param beta1: Adam related parameter
        :param beta2: Adam related parameter
        :param epsilon: Adam related parameter
        :param safeguard: The maximum absolute value of entries of the controller gain
        :return: The controller gain K
        '''

        # Initialize the controller
        Lin_gain = LinK(copy.copy(K0))
        Lin_gain.make_sampling_on(explore_mag)

        # A heuristic baseline to decrease the variance
        baseline = 0.0

        # Initializing Adam optimizer
        adam = ADAM(self.m, self.n, step_size=step_size, beta1=beta1, beta2=beta2, epsilon=epsilon)

        # start iteration
        for k in range(N):
            batch = np.zeros((self.m, self.n))
            reward = np.zeros(batch_size)

            # In each iteration, collect batches
            for j in range(batch_size):

                # Do one rollout
                states, actions, costs, _ = self.dyn.one_rollout(Lin_gain.sample_lin_policy, T)

                # Building the gradient of the loss with respect to gain
                actions_randomness = actions - Lin_gain.lin_policy(states)
                reward[j] = -np.sum(costs)/T
                batch += explore_mag**(-2) * ((reward[j] - baseline) / batch_size) * actions_randomness.T @ states

            # Update the baseline when batches are collected
            baseline = np.mean(reward)

            # Update the policy using ADAM
            dK = adam.opt_onestep(batch)
            Lin_gain.K += dK
        return self.safeK(Lin_gain.K, safeguard)

        The policy gradient algorithm where the policy is considered to be linear. We use ADAM optimization.
        :param K0: The initial controller gain
        :param N: Number of iterations
        :param batch_size: number of batches per each step of optimization
        :param T: Trajectory length
        :param explore_mag: Magnitude of the noise to explore
        :param step_size: learning rate for Adam
        :param beta1: Adam related parameter
        :param beta2: Adam related parameter
        :param epsilon: Adam related parameter
        :param safeguard: The maximum absolute value of entries of the controller gain
        :return: The controller gain K
        
The policy gradient algorithm where the policy is considered to be linear. We use ADAM optimization.
The policy gradient algorithm where the policy is considered to be linear . We use ADAM optimization .
def pg_linpolicy(self, K0, N, batch_size, T, explore_mag=0.1,
                     step_size=0.1, beta1=0.9, beta2=0.999, epsilon=1.0e-8, safeguard=10):
        Lin_gain = LinK(copy.copy(K0))
        Lin_gain.make_sampling_on(explore_mag)
        baseline = 0.0
        adam = ADAM(self.m, self.n, step_size=step_size, beta1=beta1, beta2=beta2, epsilon=epsilon)
        for k in range(N):
            batch = np.zeros((self.m, self.n))
            reward = np.zeros(batch_size)
            for j in range(batch_size):
                states, actions, costs, _ = self.dyn.one_rollout(Lin_gain.sample_lin_policy, T)
                actions_randomness = actions - Lin_gain.lin_policy(states)
                reward[j] = -np.sum(costs)/T
                batch += explore_mag**(-2) * ((reward[j] - baseline) / batch_size) * actions_randomness.T @ states
            baseline = np.mean(reward)
            dK = adam.opt_onestep(batch)
            Lin_gain.K += dK
        return self.safeK(Lin_gain.K, safeguard)
def pg_linpolicy ( self , K0 , N , batch_size , T , explore_mag = 0.1 , step_size = 0.1 , beta1 = 0.9 , beta2 = 0.999 , epsilon = 1.0e-8 , safeguard = 10 ) : Lin_gain = LinK ( copy . copy ( K0 ) ) Lin_gain . make_sampling_on ( explore_mag ) baseline = 0.0 adam = ADAM ( self . m , self . n , step_size = step_size , beta1 = beta1 , beta2 = beta2 , epsilon = epsilon ) for k in range ( N ) : batch = np . zeros ( ( self . m , self . n ) ) reward = np . zeros ( batch_size ) for j in range ( batch_size ) : states , actions , costs , _ = self . dyn . one_rollout ( Lin_gain . sample_lin_policy , T ) actions_randomness = actions - Lin_gain . lin_policy ( states ) reward [ j ] = - np . sum ( costs ) / T batch += explore_mag ** ( - 2 ) * ( ( reward [ j ] - baseline ) / batch_size ) * actions_randomness . T @ states baseline = np . mean ( reward ) dK = adam . opt_onestep ( batch ) Lin_gain . K += dK return self . safeK ( Lin_gain . K , safeguard )
The policy gradient algorithm where the policy is considered to be linear.
The policy gradient algorithm where the policy is considered to be linear .
'''
        The policy gradient algorithm where the policy is considered to be linear. We use ADAM optimization.
        :param K0: The initial controller gain
        :param N: Number of iterations
        :param batch_size: number of batches per each step of optimization
        :param T: Trajectory length
        :param explore_mag: Magnitude of the noise to explore
        :param step_size: learning rate for Adam
        :param beta1: Adam related parameter
        :param beta2: Adam related parameter
        :param epsilon: Adam related parameter
        :param safeguard: The maximum absolute value of entries of the controller gain
        :return: The controller gain K
        ''' # Initialize the controller # A heuristic baseline to decrease the variance # Initializing Adam optimizer # start iteration # In each iteration, collect batches # Do one rollout # Building the gradient of the loss with respect to gain # Update the baseline when batches are collected # Update the policy using ADAM
The controller gain K
The controller gain K
self
K0
The initial controller gain
The initial controller gain
N
Number of iterations
Number of iterations
batch_size
number of batches per each step of optimization
number of batches per each step of optimization
T
None
explore_mag
Magnitude of the noise to explore
Magnitude of the noise to explore
step_size
learning rate for Adam
learning rate for Adam
beta1
Adam related parameter
Adam related parameter
beta2
Adam related parameter
Adam related parameter
epsilon
Adam related parameter
Adam related parameter
safeguard
The maximum absolute value of entries of the controller gain
The maximum absolute value of entries of the controller gain