code: # Currently this script is configured to use the note-generator model.
  
  from config import sequence_length, output_dir, note_generator_dir
  from helper import loadChorales, loadModelAndWeights, createPitchSpecificVocabularies, createDurationVocabularySpecific
  from music21 import note, instrument, stream, duration
  import numpy as np
  import os
  
  # disable GPU processing
  os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
  os.environ["CUDA_VISIBLE_DEVICES"] = ""
  # ----------------------------------------------
  
  from keras.utils import to_categorical
  
  # select the epoch to use when loading the weights of the model generator
  generator_epoch = 43
  
  # how many notes to generate ('end' marks are created along the way and the result is split into pieces)
  number_of_notes = 200
  
  # load chorales to create the vocabularies
  print('loading chorales...')
  notes = loadChorales()
  
  # create the vocabulary
  note_vocab, note_names_vocab, note_vocab_categorical = createPitchSpecificVocabularies([x[0] for (x, _) in notes])
  duration_vocab = createDurationVocabularySpecific([d for (_, d) in notes])
  duration_vocab_categorical = to_categorical(range(len(duration_vocab)))
  
  note_to_int = dict((note, number) for number, note in enumerate(note_vocab))
  int_to_note = dict((number, note) for number, note in enumerate(note_vocab))
  
  duration_to_int = dict((dur, number) for number, dur in enumerate(duration_vocab))
  
  duration_dim = duration_vocab.shape[0]
  pitch_dim = np.array(note_vocab).shape[0]
  
  print('loading networks...')
  dir_path = os.path.dirname(os.path.realpath(__file__))
  generator = loadModelAndWeights(os.path.join(dir_path, note_generator_dir, 'model.json'),
                                 os.path.join(dir_path, note_generator_dir, 'weights-{:02d}.hdf5'.format(generator_epoch)))
  
  # make a melody!!!
  pitch_input = np.eye(pitch_dim)[np.random.choice(pitch_dim, size=sequence_length)]
  duration_input = np.eye(duration_dim)[np.random.choice(duration_dim, size=sequence_length)]
  
  print('generating output...')
  
  # generate notes
  generator_output = []
  
  for _ in range(number_of_notes):
      # reshape inputs
      pi = np.reshape(pitch_input, (1, sequence_length, pitch_dim))
      di = np.reshape(duration_input, (1, sequence_length, duration_dim))
  
      # make prediction
      pitch_pred, dur_pred = generator.predict({'pitches_input': pi, 'durations_input': di}, verbose=0)
  
      generator_output.append((pitch_pred, dur_pred))
  
      pitch_input = np.vstack([pitch_input, pitch_pred])
      pitch_input = pitch_input[1:len(pitch_input)]
  
      duration_input = np.vstack([duration_input, dur_pred])
      duration_input = duration_input[1:len(duration_input)]
  
  
  output_notes = [(int_to_note[np.argmax(n)], duration_vocab[np.argmax(d)]) for (n, d) in generator_output]
  output_notes = np.array(output_notes)
  output_notes = np.reshape(output_notes, (-1, 2))
  
  # output_notes contains: pitch values in midi format (integers), 'rest' marks, 'end' marks
  
  # split the generated notes into pieces based on 'end' marks
  indices = []
  for (ind, (n, _)) in enumerate(output_notes):
      if n == 'end':
          indices.append(ind)
  indices = np.insert(np.reshape(indices, (-1)), 0, 0)
      
  pieces = [output_notes]
  if len(indices) > 1:
      pieces = ([ output_notes[(indices[j] + 1):indices[j + 1] ] for j in range(len(indices) - 1)])
  
  print('writing output to disk...')
  
  os.makedirs(os.path.join(dir_path, output_dir, 'note-generator'), exist_ok=True)
  
  # output pieces to midi files
  for index, notes in enumerate(pieces):
      midi_notes = []
      offset = 0
      for n, d in notes:
          # since a duration of 0 is included in the vocabulary (for the 'end' marks), the network may generate a 0 duration for other notes
          # naively correct and report this erroneous behaviour
          if abs(float(d)) < 0.001:
              print('found zero duration')
              d = '1.0'
          if n == 'rest':
              new_note = note.Rest()
              new_note.duration = duration.Duration(float(d))
              new_note.offset = offset
              new_note.storedInstrument = instrument.Piano()
              midi_notes.append(new_note)
          else:
              new_note = note.Note(int(n))
              new_note.duration = duration.Duration(float(d))
              new_note.offset = offset
              new_note.storedInstrument = instrument.Piano()
              midi_notes.append(new_note)
          offset += float(d)
          
      midi_stream = stream.Stream(midi_notes)
      midi_stream.write('midi', fp=os.path.join(dir_path, output_dir, 'note-generator', 'sample-{}.mid'.format(index))) # -*- coding: utf-8 -*-
  # source from https://github.com/keon/deep-q-learning/blob/master/dqn.py
  import random
  import gym
  import numpy as np
  from collections import deque
  from keras.models import Sequential
  from keras.layers import Dense
  from keras.optimizers import Adam
  
  EPISODES = 1000
  
  class DQNAgent:
      def __init__(self, state_size, action_size):
          self.state_size = state_size
          self.action_size = action_size
          self.memory = deque(maxlen=2000)
          self.gamma = 0.95    # discount rate
          self.epsilon = 1.0  # exploration rate
          self.epsilon_min = 0.01
          self.epsilon_decay = 0.995
          self.learning_rate = 0.001
          self.model = self._build_model()
  
      def _build_model(self):
          # Neural Net for Deep-Q learning Model
          model = Sequential()
          model.add(Dense(24, input_dim=self.state_size, activation='relu'))
          model.add(Dense(24, activation='relu'))
          model.add(Dense(self.action_size, activation='linear'))
          model.compile(loss='mse',
                        optimizer=Adam(lr=self.learning_rate))
          return model
  
      def remember(self, state, action, reward, next_state, done):
          self.memory.append((state, action, reward, next_state, done))
  
      def act(self, state):
          if np.random.rand() <= self.epsilon:
              return random.randrange(self.action_size)
          act_values = self.model.predict(state)
          return np.argmax(act_values[0])  # returns action
  
      def replay(self, batch_size):
          minibatch = random.sample(self.memory, batch_size)
          for state, action, reward, next_state, done in minibatch:
              target = reward
              if not done:
                  target = (reward + self.gamma *
                            np.amax(self.model.predict(next_state)[0]))
              target_f = self.model.predict(state)
              target_f[0][action] = target
              self.model.fit(state, target_f, epochs=1, verbose=0)
          if self.epsilon > self.epsilon_min:
              self.epsilon *= self.epsilon_decay
  
      def load(self, name):
          self.model.load_weights(name)
  
      def save(self, name):
          self.model.save_weights(name)
  
  
  if __name__ == "__main__":
      env = gym.make('MountainCar-v0')
      state_size = env.observation_space.shape[0]
      action_size = env.action_space.n
      agent = DQNAgent(state_size, action_size)
      # agent.load("./save/cartpole-dqn.h5")
      done = False
      batch_size = 32
  
      for e in range(EPISODES):
          state = env.reset()
          state = np.reshape(state, [1, state_size])        
          for time in range(500):
              # env.render()
              action = agent.act(state)
              env.render()
              next_state, reward, done, _ = env.step(action)
              reward = reward if not done else -10
              next_state = np.reshape(next_state, [1, state_size])
              agent.remember(state, action, reward, next_state, done)
              state = next_state
              if done:
                  print("episode: {}/{}, score: {}, e: {:.2}"
                        .format(e, EPISODES, time, agent.epsilon))
                  break
              if len(agent.memory) > batch_size:
                  agent.replay(batch_size)
          # if e % 10 == 0:
          #     agent.save("./save/cartpole-dqn.h5")
apis: numpy.vstack numpy.eye numpy.reshape numpy.random.choice numpy.argmax numpy.array
apis: numpy.reshape numpy.random.rand numpy.argmax