':param env: Environment
:param policy: Policy
:param qf: Q function
:param es: Exploration strategy
:param batch_size: Number of samples for each minibatch.
:param n_epochs: Number of epochs. Policy will be evaluated after each epoch.
:param epoch_length: How many timesteps for each epoch.
:param min_pool_size: Minimum size of the pool to start training.
:param replay_pool_size: Size of the experience replay pool.
:param discount: Discount factor for the cumulative return.
:param max_path_length: Discount factor for the cumulative return.
:param qf_weight_decay: Weight decay factor for parameters of the Q function.
:param qf_update_method: Online optimization method for training Q function.
:param qf_learning_rate: Learning rate for training Q function.
:param policy_weight_decay: Weight decay factor for parameters of the policy.
:param policy_update_method: Online optimization method for training the policy.
:param policy_learning_rate: Learning rate for training the policy.
:param eval_samples: Number of samples (timesteps) for evaluating the policy.
:param soft_target_tau: Interpolation parameter for doing the soft target update.
:param n_updates_per_sample: Number of Q function and policy updates per new sample obtained
:param scale_reward: The scaling factor applied to the rewards when training
:param include_horizon_terminal_transitions: whether to include transitions with terminal=True because the
horizon was reached. This might make the Q value back up less stable for certain tasks.
:param plot: Whether to visualize the policy performance after each eval_interval.
:param pause_for_plot: Whether to pause before continuing when plotting.
:return:'

def __init__(self, env, policy, qf, es, batch_size=32, n_epochs=200, epoch_length=1000, min_pool_size=10000, replay_pool_size=1000000, discount=0.99, max_path_length=250, qf_weight_decay=0.0, qf_update_method='adam', qf_learning_rate=0.001, policy_weight_decay=0, policy_update_method='adam', policy_learning_rate=0.001, eval_samples=10000, soft_target=True, soft_target_tau=0.001, n_updates_per_sample=1, scale_reward=1.0, include_horizon_terminal_transitions=False, plot=False, pause_for_plot=False):

    self.env = env
    self.policy = policy
    self.qf = qf
    self.es = es
    self.batch_size = batch_size
    self.n_epochs = n_epochs
    self.epoch_length = epoch_length
    self.min_pool_size = min_pool_size
    self.replay_pool_size = replay_pool_size
    self.discount = discount
    self.max_path_length = max_path_length
    self.qf_weight_decay = qf_weight_decay
    self.qf_update_method = parse_update_method(qf_update_method, learning_rate=qf_learning_rate)
    self.qf_learning_rate = qf_learning_rate
    self.policy_weight_decay = policy_weight_decay
    self.policy_update_method = parse_update_method(policy_update_method, learning_rate=policy_learning_rate)
    self.policy_learning_rate = policy_learning_rate
    self.eval_samples = eval_samples
    self.soft_target_tau = soft_target_tau
    self.n_updates_per_sample = n_updates_per_sample
    self.include_horizon_terminal_transitions = include_horizon_terminal_transitions
    self.plot = plot
    self.pause_for_plot = pause_for_plot
    self.qf_loss_averages = []
    self.policy_surr_averages = []
    self.q_averages = []
    self.y_averages = []
    self.paths = []
    self.es_path_returns = []
    self.paths_samples_cnt = 0
    self.scale_reward = scale_reward
    self.opt_info = None
