'Backpropagate prediction error from a randomly sampled experience batch.
A minibatch of experiences is randomly sampled from the model\'s experience
replay buffer and used to update the weights of the q_network and
target_q_network.'

def training_step(self):

    if ((self.num_times_train_called % self.dqn_hparams.train_every_nth) == 0):
        if (len(self.experience) < self.dqn_hparams.minibatch_size):
            return
        samples = random.sample(range(len(self.experience)), self.dqn_hparams.minibatch_size)
        samples = [self.experience[i] for i in samples]
        states = np.empty((len(samples), self.q_network.cell.state_size))
        new_states = np.empty((len(samples), self.target_q_network.cell.state_size))
        reward_new_states = np.empty((len(samples), self.reward_rnn.cell.state_size))
        observations = np.empty((len(samples), self.input_size))
        new_observations = np.empty((len(samples), self.input_size))
        action_mask = np.zeros((len(samples), self.num_actions))
        rewards = np.empty((len(samples),))
        lengths = np.full(len(samples), 1, dtype=int)
        for (i, (o, s, a, r, new_o, new_s, reward_s)) in enumerate(samples):
            observations[i, :] = o
            new_observations[i, :] = new_o
            states[i, :] = s
            new_states[i, :] = new_s
            action_mask[i, :] = a
            rewards[i] = r
            reward_new_states[i, :] = reward_s
        observations = np.reshape(observations, (len(samples), 1, self.input_size))
        new_observations = np.reshape(new_observations, (len(samples), 1, self.input_size))
        calc_summaries = ((self.iteration % 100) == 0)
        calc_summaries = (calc_summaries and (self.summary_writer is not None))
        if (self.algorithm == 'g'):
            (_, _, target_vals, summary_str) = self.session.run([self.prediction_error, self.train_op, self.target_vals, (self.summarize if calc_summaries else self.no_op1)], {self.reward_rnn.melody_sequence: new_observations, self.reward_rnn.initial_state: reward_new_states, self.reward_rnn.lengths: lengths, self.q_network.melody_sequence: observations, self.q_network.initial_state: states, self.q_network.lengths: lengths, self.target_q_network.melody_sequence: new_observations, self.target_q_network.initial_state: new_states, self.target_q_network.lengths: lengths, self.action_mask: action_mask, self.rewards: rewards})
        else:
            (_, _, target_vals, summary_str) = self.session.run([self.prediction_error, self.train_op, self.target_vals, (self.summarize if calc_summaries else self.no_op1)], {self.q_network.melody_sequence: observations, self.q_network.initial_state: states, self.q_network.lengths: lengths, self.target_q_network.melody_sequence: new_observations, self.target_q_network.initial_state: new_states, self.target_q_network.lengths: lengths, self.action_mask: action_mask, self.rewards: rewards})
        total_logs = (self.iteration * self.dqn_hparams.train_every_nth)
        if ((total_logs % self.output_every_nth) == 0):
            self.target_val_list.append(np.mean(target_vals))
        self.session.run(self.target_network_update)
        if calc_summaries:
            self.summary_writer.add_summary(summary_str, self.iteration)
        self.iteration += 1
    self.num_times_train_called += 1
