'Initializes the MelodyQNetwork class.
Args:
output_dir: Where the model will save its compositions (midi files).
dqn_hparams: A HParams object containing the hyperparameters of
the DQN algorithm, including minibatch size, exploration probability,
etc.
reward_mode: Controls which reward function can be applied. There are
several, including \'scale\', which teaches the model to play a scale,
and of course \'music_theory_all\', which is a music-theory-based reward
function composed of other functions.
reward_scaler: Controls the emphasis placed on the music theory rewards.
This value is the inverse of \'c\' in the academic paper.
exploration_mode: can be \'egreedy\' which is an epsilon greedy policy, or
it can be \'boltzmann\', in which the model will sample from its output
distribution to choose the next action.
priming_mode: Each time the model begins a new composition, it is primed
with either a random note (\'random_note\'), a random MIDI file from the
training data (\'random_midi\'), or a particular MIDI file
(\'single_midi\').
stochastic_observations: If False, the note that the model chooses to
play next (the argmax of its softmax probabilities) deterministically
becomes the next note it will observe. If True, the next observation
will be sampled from the model\'s softmax output.
algorithm: can be \'default\', \'psi\', \'g\' or \'pure_rl\', for different
learning algorithms
note_rnn_checkpoint_dir: The directory from which the internal
NoteRNNLoader will load its checkpointed LSTM.
note_rnn_checkpoint_file: A checkpoint file to use in case one cannot be
found in the note_rnn_checkpoint_dir.
note_rnn_type: If \'default\', will use the basic LSTM described in the
research paper. If \'basic_rnn\', will assume the checkpoint is from a
Magenta basic_rnn model.
note_rnn_hparams: A HParams object which defines the hyper parameters
used to train the MelodyRNN model that will be loaded from a checkpoint.
num_notes_in_melody: The length of a composition of the model
input_size: the size of the one-hot vector encoding a note that is input
to the model.
num_actions: The size of the one-hot vector encoding a note that is
output by the model.
midi_primer: A midi file that can be used to prime the model if
priming_mode is set to \'single_midi\'.
save_name: Name the model will use to save checkpoints.
output_every_nth: How many training steps before the model will print
an output saying the cumulative reward, and save a checkpoint.
training_file_list: A list of paths to tfrecord files containing melody
training data. This is necessary to use the \'random_midi\' priming mode.
summary_writer: A tf.summary.FileWriter used to log metrics.
initialize_immediately: if True, the class will instantiate its component
MelodyRNN networks and build the graph in the constructor.'

def __init__(self, output_dir, dqn_hparams=None, reward_mode='music_theory_all', reward_scaler=1.0, exploration_mode='egreedy', priming_mode='random_note', stochastic_observations=False, algorithm='q', note_rnn_checkpoint_dir=None, note_rnn_checkpoint_file=None, note_rnn_type='default', note_rnn_hparams=None, num_notes_in_melody=32, input_size=rl_tuner_ops.NUM_CLASSES, num_actions=rl_tuner_ops.NUM_CLASSES, midi_primer=None, save_name='rl_tuner.ckpt', output_every_nth=1000, training_file_list=None, summary_writer=None, initialize_immediately=True):

    self.graph = tf.Graph()
    with self.graph.as_default():
        self.input_size = input_size
        self.num_actions = num_actions
        self.output_every_nth = output_every_nth
        self.output_dir = output_dir
        self.save_path = os.path.join(output_dir, save_name)
        self.reward_scaler = reward_scaler
        self.reward_mode = reward_mode
        self.exploration_mode = exploration_mode
        self.num_notes_in_melody = num_notes_in_melody
        self.stochastic_observations = stochastic_observations
        self.algorithm = algorithm
        self.priming_mode = priming_mode
        self.midi_primer = midi_primer
        self.training_file_list = training_file_list
        self.note_rnn_checkpoint_dir = note_rnn_checkpoint_dir
        self.note_rnn_checkpoint_file = note_rnn_checkpoint_file
        self.note_rnn_hparams = note_rnn_hparams
        self.note_rnn_type = note_rnn_type
        if ((priming_mode == 'single_midi') and (midi_primer is None)):
            tf.logging.fatal('A    midi    primer    file    is    required    when    usingthe    single_midi    priming    mode.')
        if ((note_rnn_checkpoint_dir is None) or (not note_rnn_checkpoint_dir)):
            print('Retrieving    checkpoint    of    Note    RNN    from    Magenta    download    server.')
            urllib.urlretrieve('http://download.magenta.tensorflow.org/models/rl_tuner_note_rnn.ckpt', 'note_rnn.ckpt')
            self.note_rnn_checkpoint_dir = os.getcwd()
            self.note_rnn_checkpoint_file = os.path.join(os.getcwd(), 'note_rnn.ckpt')
        if (self.note_rnn_hparams is None):
            if (self.note_rnn_type == 'basic_rnn'):
                self.note_rnn_hparams = rl_tuner_ops.basic_rnn_hparams()
            else:
                self.note_rnn_hparams = rl_tuner_ops.default_hparams()
        if ((self.algorithm == 'g') or (self.algorithm == 'pure_rl')):
            self.reward_mode = 'music_theory_only'
        if (dqn_hparams is None):
            self.dqn_hparams = rl_tuner_ops.default_dqn_hparams()
        else:
            self.dqn_hparams = dqn_hparams
        self.discount_rate = tf.constant(self.dqn_hparams.discount_rate)
        self.target_network_update_rate = tf.constant(self.dqn_hparams.target_network_update_rate)
        self.optimizer = tf.train.AdamOptimizer()
        self.actions_executed_so_far = 0
        self.experience = deque(maxlen=self.dqn_hparams.max_experience)
        self.iteration = 0
        self.summary_writer = summary_writer
        self.num_times_store_called = 0
        self.num_times_train_called = 0
    self.reward_last_n = 0
    self.rewards_batched = []
    self.music_theory_reward_last_n = 0
    self.music_theory_rewards_batched = []
    self.note_rnn_reward_last_n = 0
    self.note_rnn_rewards_batched = []
    self.eval_avg_reward = []
    self.eval_avg_music_theory_reward = []
    self.eval_avg_note_rnn_reward = []
    self.target_val_list = []
    self.beat = 0
    self.composition = []
    self.composition_direction = 0
    self.leapt_from = None
    self.steps_since_last_leap = 0
    if (not exists(self.output_dir)):
        makedirs(self.output_dir)
    if initialize_immediately:
        self.initialize_internal_models_graph_session()
