scripts/grad_estimators.py
def estimate_grad_antithetic(self, expl_directions, rewards):
        """
        Central differences (also: antithetic) gradient estimator, with optional scaling by the stdev of the rewards.

        :param expl_directions: The set of perturbations.
        :param rewards: Rewards obtained from perturbed models, a tensor of size (num_expl_directions * 2), 
                        where the first half corresponds to exploration directions +epsilon, and the second half
                        to -epsilon.
        """
        rewards_pos, rewards_neg = torch.chunk(rewards, 2)   # split the two halves
        rewards = rewards.view((2,-1))

        if self.use_only_best:
            sort_idx = torch.argsort(torch.max(rewards, dim=0).values, descending=True)
            rewards_pos = rewards_pos[sort_idx][:self.use_only_best]
            rewards_neg = rewards_neg[sort_idx][:self.use_only_best] 
            expl_directions = expl_directions[sort_idx][:self.use_only_best]
        
        return (rewards_pos - rewards_neg) @ expl_directions / (len(rewards_pos) * 2 * self.normalizer(torch.cat((rewards_pos, rewards_neg))))

        Central differences (also: antithetic) gradient estimator, with optional scaling by the stdev of the rewards.

        :param expl_directions: The set of perturbations.
        :param rewards: Rewards obtained from perturbed models, a tensor of size (num_expl_directions * 2), 
                        where the first half corresponds to exploration directions +epsilon, and the second half
                        to -epsilon.
        
Central differences (also: antithetic) gradient estimator, with optional scaling by the stdev of the rewards.
Central differences ( also : antithetic ) gradient estimator with optional scaling by the stdev of the rewards .
def estimate_grad_antithetic(self, expl_directions, rewards):
        rewards_pos, rewards_neg = torch.chunk(rewards, 2)   
        rewards = rewards.view((2,-1))
        if self.use_only_best:
            sort_idx = torch.argsort(torch.max(rewards, dim=0).values, descending=True)
            rewards_pos = rewards_pos[sort_idx][:self.use_only_best]
            rewards_neg = rewards_neg[sort_idx][:self.use_only_best] 
            expl_directions = expl_directions[sort_idx][:self.use_only_best]
        return (rewards_pos - rewards_neg) @ expl_directions / (len(rewards_pos) * 2 * self.normalizer(torch.cat((rewards_pos, rewards_neg))))
def estimate_grad_antithetic ( self , expl_directions , rewards ) : rewards_pos , rewards_neg = torch . chunk ( rewards , 2 ) rewards = rewards . view ( ( 2 , - 1 ) ) if self . use_only_best : sort_idx = torch . argsort ( torch . max ( rewards , dim = 0 ) . values , descending = True ) rewards_pos = rewards_pos [ sort_idx ] [ : self . use_only_best ] rewards_neg = rewards_neg [ sort_idx ] [ : self . use_only_best ] expl_directions = expl_directions [ sort_idx ] [ : self . use_only_best ] return ( rewards_pos - rewards_neg ) @ expl_directions / ( len ( rewards_pos ) * 2 * self . normalizer ( torch . cat ( ( rewards_pos , rewards_neg ) ) ) )
Central differences (also: antithetic) gradient estimator, with optional scaling by the stdev of the rewards.
Central differences ( also : antithetic ) gradient estimator with optional scaling by the stdev of the rewards .
"""
        Central differences (also: antithetic) gradient estimator, with optional scaling by the stdev of the rewards.

        :param expl_directions: The set of perturbations.
        :param rewards: Rewards obtained from perturbed models, a tensor of size (num_expl_directions * 2), 
                        where the first half corresponds to exploration directions +epsilon, and the second half
                        to -epsilon.
        """ # split the two halves
self
expl_directions
The set of perturbations.
The set of perturbations .
rewards
Rewards obtained from perturbed models, a tensor of size (num_expl_directions * 2),
where the first half corresponds to exploration directions +epsilon, and the second half
to -epsilon.
Rewards obtained from perturbed models a tensor of size ( num_expl_directions * 2 ) where the first half corresponds to exploration directions + epsilon and the second half to - epsilon .