text: from __future__ import division
  import numpy as np
  
  from tfrl.util import *
  
  
  class Policy(object):
      """Abstract base class for all implemented policies.
  
      Each policy helps with selection of action to take on an environment.
  
      Do not use this abstract base class directly but instead use one of the concrete policies implemented.
      To implement your own policy, you have to implement the following methods:
  
      - `select_action`
  
      # Arguments
          agent (rl.core.Agent): Agent used
      """
      def _set_agent(self, agent):
          self.agent = agent
  
      @property
      def metrics_names(self):
          return []
  
      @property
      def metrics(self):
          return []
  
      def select_action(self, **kwargs):
          raise NotImplementedError()
  
      def get_config(self):
          """Return configuration of the policy
  
          # Returns
              Configuration as dict
          """
          return {}
  
  
  class LinearAnnealedPolicy(Policy):
      """Implement the linear annealing policy
  
      Linear Annealing Policy computes a current threshold value and
      transfers it to an inner policy which chooses the action. The threshold
      value is following a linear function decreasing over time."""
      def __init__(self, inner_policy, attr, value_max, value_min, value_test, nb_steps):
          if not hasattr(inner_policy, attr):
              raise ValueError('Policy does not have attribute "{}".'.format(attr))
  
          super(LinearAnnealedPolicy, self).__init__()
  
          self.inner_policy = inner_policy
          self.attr = attr
          self.value_max = value_max
          self.value_min = value_min
          self.value_test = value_test
          self.nb_steps = nb_steps
  
      def get_current_value(self):
          """Return current annealing value
  
          # Returns
              Value to use in annealing
          """
          if self.agent.training:
              # Linear annealed: f(x) = ax + b.
              a = -float(self.value_max - self.value_min) / float(self.nb_steps)
              b = float(self.value_max)
              value = max(self.value_min, a * float(self.agent.step) + b)
          else:
              value = self.value_test
          return value
  
      def select_action(self, **kwargs):
          """Choose an action to perform
  
          # Returns
              Action to take (int)
          """
          setattr(self.inner_policy, self.attr, self.get_current_value())
          return self.inner_policy.select_action(**kwargs)
  
      @property
      def metrics_names(self):
          """Return names of metrics
  
          # Returns
              List of metric names
          """
          return ['mean_{}'.format(self.attr)]
  
      @property
      def metrics(self):
          """Return metrics values
  
          # Returns
              List of metric values
          """
  
          return [getattr(self.inner_policy, self.attr)]
  
      def get_config(self):
          """Return configurations of LinearAnnealedPolicy
  
          # Returns
              Dict of config
          """
          config = super(LinearAnnealedPolicy, self).get_config()
          config['attr'] = self.attr
          config['value_max'] = self.value_max
          config['value_min'] = self.value_min
          config['value_test'] = self.value_test
          config['nb_steps'] = self.nb_steps
          config['inner_policy'] = get_object_config(self.inner_policy)
          return config
  
  class SoftmaxPolicy(Policy):
      """ Implement softmax policy for multinimial distribution
  
      Simple Policy
  
      - takes action according to the pobability distribution
  
      """
      def select_action(self, nb_actions, probs):
          """Return the selected action
  
          # Arguments
              probs (np.ndarray) : Probabilty for each action
  
          # Returns
              action
  
          """
          action = np.random.choice(range(nb_actions), p=probs)
          return action
  
  class EpsGreedyQPolicy(Policy):
      """Implement the epsilon greedy policy
  
      Eps Greedy policy either:
  
      - takes a random action with probability epsilon
      - takes current best action with prob (1 - epsilon)
      """
      def __init__(self, eps=.1):
          super(EpsGreedyQPolicy, self).__init__()
          self.eps = eps
  
      def select_action(self, q_values):
          """Return the selected action
  
          # Arguments
              q_values (np.ndarray): List of the estimations of Q for each action
  
          # Returns
              Selection action
          """
          assert q_values.ndim == 1
          nb_actions = q_values.shape[0]
  
          if np.random.uniform() < self.eps:
              action = np.random.randint(0, nb_actions)
          else:
              action = np.argmax(q_values)
          return action
  
      def get_config(self):
          """Return configurations of EpsGreedyQPolicy
  
          # Returns
              Dict of config
          """
          config = super(EpsGreedyQPolicy, self).get_config()
          config['eps'] = self.eps
          return config
  
  
  class GreedyQPolicy(Policy):
      """Implement the greedy policy
  
      Greedy policy returns the current best action according to q_values
      """
      def select_action(self, q_values):
          """Return the selected action
  
          # Arguments
              q_values (np.ndarray): List of the estimations of Q for each action
  
          # Returns
              Selection action
          """
          assert q_values.ndim == 1
          action = np.argmax(q_values)
          return action
  
  
  class BoltzmannQPolicy(Policy):
      """Implement the Boltzmann Q Policy
  
      Boltzmann Q Policy builds a probability law on q values and returns
      an action selected randomly according to this law.
      """
      def __init__(self, tau=1., clip=(-500., 500.)):
          super(BoltzmannQPolicy, self).__init__()
          self.tau = tau
          self.clip = clip
  
      def select_action(self, q_values):
          """Return the selected action
  
          # Arguments
              q_values (np.ndarray): List of the estimations of Q for each action
  
          # Returns
              Selection action
          """
          assert q_values.ndim == 1
          q_values = q_values.astype('float64')
          nb_actions = q_values.shape[0]
  
          exp_values = np.exp(np.clip(q_values / self.tau, self.clip[0], self.clip[1]))
          probs = exp_values / np.sum(exp_values)
          action = np.random.choice(range(nb_actions), p=probs)
          return action
  
      def get_config(self):
          """Return configurations of BoltzmannQPolicy
  
          # Returns
              Dict of config
          """
          config = super(BoltzmannQPolicy, self).get_config()
          config['tau'] = self.tau
          config['clip'] = self.clip
          return config
  
  
  class MaxBoltzmannQPolicy(Policy):
      """
      A combination of the eps-greedy and Boltzman q-policy.
  
      Wiering, M.: Explorations in Efficient Reinforcement Learning.
      PhD thesis, University of Amsterdam, Amsterdam (1999)
  
      https://pure.uva.nl/ws/files/3153478/8461_UBA003000033.pdf
      """
      def __init__(self, eps=.1, tau=1., clip=(-500., 500.)):
          super(MaxBoltzmannQPolicy, self).__init__()
          self.eps = eps
          self.tau = tau
          self.clip = clip
  
      def select_action(self, q_values):
          """Return the selected action
          The selected action follows the BoltzmannQPolicy with probability epsilon
          or return the Greedy Policy with probability (1 - epsilon)
  
          # Arguments
              q_values (np.ndarray): List of the estimations of Q for each action
  
          # Returns
              Selection action
          """
          assert q_values.ndim == 1
          q_values = q_values.astype('float64')
          nb_actions = q_values.shape[0]
  
          if np.random.uniform() < self.eps:
              exp_values = np.exp(np.clip(q_values / self.tau, self.clip[0], self.clip[1]))
              probs = exp_values / np.sum(exp_values)
              action = np.random.choice(range(nb_actions), p=probs)
          else:
              action = np.argmax(q_values)
          return action
  
      def get_config(self):
          """Return configurations of MaxBoltzmannQPolicy
  
          # Returns
              Dict of config
          """
          config = super(MaxBoltzmannQPolicy, self).get_config()
          config['eps'] = self.eps
          config['tau'] = self.tau
          config['clip'] = self.clip
          return config
  
  
  class BoltzmannGumbelQPolicy(Policy):
      """Implements Boltzmann-Gumbel exploration (BGE) adapted for Q learning
      based on the paper Boltzmann Exploration Done Right
      (https://arxiv.org/pdf/1705.10257.pdf).
  
      BGE is invariant with respect to the mean of the rewards but not their
      variance. The parameter C, which defaults to 1, can be used to correct for
      this, and should be set to the least upper bound on the standard deviation
      of the rewards.
  
      BGE is only available for training, not testing. For testing purposes, you
      can achieve approximately the same result as BGE after training for N steps
      on K actions with parameter C by using the BoltzmannQPolicy and setting
      tau = C/sqrt(N/K)."""
  
      def __init__(self, C=1.0):
          assert C > 0, "BoltzmannGumbelQPolicy C parameter must be > 0, not " + repr(C)
          super(BoltzmannGumbelQPolicy, self).__init__()
          self.C = C
          self.action_counts = None
  
      def select_action(self, q_values):
          """Return the selected action
  
          # Arguments
              q_values (np.ndarray): List of the estimations of Q for each action
  
          # Returns
              Selection action
          """
          # We can't use BGE during testing, since we don't have access to the
          # action_counts at the end of training.
          assert self.agent.training, "BoltzmannGumbelQPolicy should only be used for training, not testing"
  
          assert q_values.ndim == 1, q_values.ndim
          q_values = q_values.astype('float64')
  
          # If we are starting training, we should reset the action_counts.
          # Otherwise, action_counts should already be initialized, since we
          # always do so when we begin training.
          if self.agent.step == 0:
              self.action_counts = np.ones(q_values.shape)
          assert self.action_counts is not None, self.agent.step
          assert self.action_counts.shape == q_values.shape, (self.action_counts.shape, q_values.shape)
  
          beta = self.C/np.sqrt(self.action_counts)
          Z = np.random.gumbel(size=q_values.shape)
  
          perturbation = beta * Z
          perturbed_q_values = q_values + perturbation
          action = np.argmax(perturbed_q_values)
  
          self.action_counts[action] += 1
          return action
  
      def get_config(self):
          """Return configurations of BoltzmannGumbelQPolicy
  
          # Returns
              Dict of config
          """
          config = super(BoltzmannGumbelQPolicy, self).get_config()
          config['C'] = self.C
          return config
  