lq/pgrl.py
def random_search_linpolicy(self, K0, N, batch_size, T, explore_mag=0.04, step_size=0.05, safeguard=10):
        '''
        Random search.
        Gian initialization is not important
        :param N: Number of rollouts
        :param T: Time horizon
        :param explore_mag: Magnitude of the noise to explore
        :param step_size: the step size for learning
        :param batch_size: Number of directions per minibatches
        :param safeguard: The maximum absolute value of entries of the controller gain
        :return: Controller K by random search on LQ problem
        '''
        Lin_gain = LinK(copy.copy(K0))
        for k in range(N):
            reward_store = []
            batch = np.zeros((self.m, self.n))
            for j in range(batch_size):
                gain_randomness = np.random.randn(self.m, self.n)
                for sign in [-1, 1]:
                    K_test = LinK(Lin_gain.K + sign * explore_mag * gain_randomness)
                    _, _, costs, _ = self.dyn.one_rollout(K_test.lin_policy, T)
                    reward = - np.sum(costs) / T
                    batch += (reward * sign) * gain_randomness / 2 * explore_mag
                    reward_store.append(reward)

            dk = (step_size / np.std(reward_store) / batch_size) * batch
            Lin_gain.K += dk
            Lin_gain.K = np.minimum(np.maximum(Lin_gain.K, -safeguard), safeguard)
        return self.safeK(Lin_gain.K, safeguard)

        Random search.
        Gian initialization is not important
        :param N: Number of rollouts
        :param T: Time horizon
        :param explore_mag: Magnitude of the noise to explore
        :param step_size: the step size for learning
        :param batch_size: Number of directions per minibatches
        :param safeguard: The maximum absolute value of entries of the controller gain
        :return: Controller K by random search on LQ problem
        
Random search.
Gian initialization is not important
Random search . Gian initialization is not important
def random_search_linpolicy(self, K0, N, batch_size, T, explore_mag=0.04, step_size=0.05, safeguard=10):
        Lin_gain = LinK(copy.copy(K0))
        for k in range(N):
            reward_store = []
            batch = np.zeros((self.m, self.n))
            for j in range(batch_size):
                gain_randomness = np.random.randn(self.m, self.n)
                for sign in [-1, 1]:
                    K_test = LinK(Lin_gain.K + sign * explore_mag * gain_randomness)
                    _, _, costs, _ = self.dyn.one_rollout(K_test.lin_policy, T)
                    reward = - np.sum(costs) / T
                    batch += (reward * sign) * gain_randomness / 2 * explore_mag
                    reward_store.append(reward)
            dk = (step_size / np.std(reward_store) / batch_size) * batch
            Lin_gain.K += dk
            Lin_gain.K = np.minimum(np.maximum(Lin_gain.K, -safeguard), safeguard)
        return self.safeK(Lin_gain.K, safeguard)
def random_search_linpolicy ( self , K0 , N , batch_size , T , explore_mag = 0.04 , step_size = 0.05 , safeguard = 10 ) : Lin_gain = LinK ( copy . copy ( K0 ) ) for k in range ( N ) : reward_store = [ ] batch = np . zeros ( ( self . m , self . n ) ) for j in range ( batch_size ) : gain_randomness = np . random . randn ( self . m , self . n ) for sign in [ - 1 , 1 ] : K_test = LinK ( Lin_gain . K + sign * explore_mag * gain_randomness ) _ , _ , costs , _ = self . dyn . one_rollout ( K_test . lin_policy , T ) reward = - np . sum ( costs ) / T batch += ( reward * sign ) * gain_randomness / 2 * explore_mag reward_store . append ( reward ) dk = ( step_size / np . std ( reward_store ) / batch_size ) * batch Lin_gain . K += dk Lin_gain . K = np . minimum ( np . maximum ( Lin_gain . K , - safeguard ) , safeguard ) return self . safeK ( Lin_gain . K , safeguard )
Random search.
Random search .
'''
        Random search.
        Gian initialization is not important
        :param N: Number of rollouts
        :param T: Time horizon
        :param explore_mag: Magnitude of the noise to explore
        :param step_size: the step size for learning
        :param batch_size: Number of directions per minibatches
        :param safeguard: The maximum absolute value of entries of the controller gain
        :return: Controller K by random search on LQ problem
        '''
Controller K by random search on LQ problem
Controller K by random search on LQ problem
self
K0
N
Number of rollouts
Number of rollouts
batch_size
Number of directions per minibatches
Number of directions per minibatches
T
None
explore_mag
Magnitude of the noise to explore
Magnitude of the noise to explore
step_size
the step size for learning
the step size for learning
safeguard
The maximum absolute value of entries of the controller gain
The maximum absolute value of entries of the controller gain