text: import numpy as np
  
  
  class AdaptiveParamNoiseSpec(object):
      def __init__(self, initial_stddev=0.1, desired_action_stddev=0.1, adoption_coefficient=1.01):
          self.initial_stddev = initial_stddev
          self.desired_action_stddev = desired_action_stddev
          self.adoption_coefficient = adoption_coefficient
  
          self.current_stddev = initial_stddev
  
      def adapt(self, distance):
          if distance > self.desired_action_stddev:
              # Decrease stddev.
              self.current_stddev /= self.adoption_coefficient
          else:
              # Increase stddev.
              self.current_stddev *= self.adoption_coefficient
  
      def get_stats(self):
          stats = {
              'param_noise_stddev': self.current_stddev,
          }
          return stats
  
      def __repr__(self):
          fmt = 'AdaptiveParamNoiseSpec(initial_stddev={}, desired_action_stddev={}, adoption_coefficient={})'
          return fmt.format(self.initial_stddev, self.desired_action_stddev, self.adoption_coefficient)
  
  
  class ActionNoise(object):
      def reset(self):
          pass
  
  
  class NormalActionNoise(ActionNoise):
      def __init__(self, mu, sigma):
          self.mu = mu
          self.sigma = sigma
  
      def __call__(self):
          return np.random.normal(self.mu, self.sigma)
  
      def __repr__(self):
          return 'NormalActionNoise(mu={}, sigma={})'.format(self.mu, self.sigma)
  
  
  # Based on http://math.stackexchange.com/questions/1287634/implementing-ornstein-uhlenbeck-in-matlab
  class OrnsteinUhlenbeckActionNoise(ActionNoise):
      def __init__(self, mu, sigma=0.2, sigma_min=0.05, n_steps_annealing=10e6, theta=.15, dt=1e-2, x0=None):
          self.theta = theta
          self.mu = mu
          self.sigma_initial = sigma
          self.sigma = sigma
          self.sigma_min = sigma_min
          self.dt = dt
          self.x0 = x0
          self.n_steps_annealing = n_steps_annealing
          self.n_steps = 0
          self.reset()
  
      def __call__(self):
          x = self.x_prev + self.theta * (self.mu - self.x_prev) * self.dt + self.sigma * np.sqrt(self.dt) * np.random.normal(size=self.mu.shape)
          self.x_prev = x
          self.n_steps += 1
          self.update_sigma_current()
          return x    
  
      def update_sigma_current(self):
          self.sigma = max(self.sigma_min, float(self.sigma_initial) + float(self.sigma_min - self.sigma_initial) *
                                  float(self.n_steps) / float(self.n_steps_annealing))
  
      def reset(self):
          self.x_prev = self.x0 if self.x0 is not None else np.zeros_like(self.mu)
  
  
      def __repr__(self):
          return 'OrnsteinUhlenbeckActionNoise(mu={}, sigma={})'.format(self.mu, self.sigma)
  