text: import collections
  import os
  import random
  
  import numpy as np
  import tensorflow as tf
  
  from network_helpers import load_network, get_stochastic_network_move, save_network
  
  
  def train_policy_gradients(game_spec,
                             create_network,
                             network_file_path,
                             save_network_file_path=None,
                             opponent_func=None,
                             number_of_games=10000,
                             print_results_every=1000,
                             learn_rate=1e-4,
                             batch_size=100,
                             randomize_first_player=True,
  						   plot_stats = False,
  						   clear_screen_every = 20):
      """Train a network using policy gradients
  
      Args:
          save_network_file_path (str): Optionally specifiy a path to use for saving the network, if unset then
              the network_file_path param is used.
          opponent_func (board_state, side) -> move: Function for the opponent, if unset we use an opponent playing
              randomly
          randomize_first_player (bool): If True we alternate between being the first and second player
          game_spec (games.base_game_spec.BaseGameSpec): The game we are playing
          create_network (->(input_layer : tf.placeholder, output_layer : tf.placeholder, variables : [tf.Variable])):
              Method that creates the network we will train.
          network_file_path (str): path to the file with weights we want to load for this network
          number_of_games (int): number of games to play before stopping
          print_results_every (int): Prints results to std out every x games, also saves the network
          learn_rate (float):
          batch_size (int):
  
      Returns:
          (variables used in the final network : list, win rate: float)
      """
  	
      save_network_file_path = save_network_file_path or network_file_path
      opponent_func = opponent_func or game_spec.get_random_player_func()
      reward_placeholder = tf.placeholder("float", shape=(None,))
      actual_move_placeholder = tf.placeholder("float", shape=(None, game_spec.outputs()))
  
      input_layer, output_layer, variables = create_network()
  
      policy_gradient = tf.log(
          tf.reduce_sum(tf.multiply(actual_move_placeholder, output_layer), axis=1)) * reward_placeholder
      train_step = tf.train.AdamOptimizer(learn_rate).minimize(-policy_gradient)
  
      with tf.Session() as session:
          if 'session' in locals() and session is not None:
              print('Close interactive session')
              session.close()
              session.run(tf.global_variables_initializer())
          
          if network_file_path and os.path.isfile(network_file_path):
              print("loading pre-existing network")
              load_network(session, variables, network_file_path)
  
          mini_batch_board_states, mini_batch_moves, mini_batch_rewards = [], [], []
          results = collections.deque(maxlen=print_results_every)
          mini_batch_illegal_moves = 0
          illegal_move = 0
          #for plotting
          episode_win = []
          episode_wrong_moves = []
          episode_count = []
  		
          def make_training_move(board_state, side):
              mini_batch_board_states.append(np.ravel(board_state) * side)
              move = get_stochastic_network_move(session, input_layer, output_layer, board_state, side)
              mini_batch_moves.append(move)
              #print("Training Move Called")
              return game_spec.flat_move_to_tuple(move.argmax())
  		
          for episode_number in range(1, number_of_games):
              # randomize if going first or second
              if (not randomize_first_player) or bool(random.getrandbits(1)):
                  reward,info = game_spec.play_game(make_training_move, opponent_func, log=False)
              else:
                  reward,info = game_spec.play_game(opponent_func, make_training_move, log=False)
                  reward = -reward
              results.append(reward)
              mini_batch_illegal_moves+=info['illegal_move']
  
              # we scale here so winning quickly is better winning slowly and loosing slowly better than loosing quick
              last_game_length = len(mini_batch_board_states) - len(mini_batch_rewards)
  
              reward /= float(last_game_length)
  
              mini_batch_rewards += ([reward] * last_game_length)
  
              if episode_number % batch_size == 0:
                  normalized_rewards = mini_batch_rewards - np.mean(mini_batch_rewards)
  
                  rewards_std = np.std(normalized_rewards)
                  if rewards_std != 0:
                      normalized_rewards /= rewards_std
                  else:
                      print("warning: got mini batch std of 0.")
  
                  np_mini_batch_board_states = np.array(mini_batch_board_states) \
                      .reshape(len(mini_batch_rewards), *input_layer.get_shape().as_list()[1:])
  
                  session.run(train_step, feed_dict={input_layer: np_mini_batch_board_states,
                                                     reward_placeholder: normalized_rewards,
                                                     actual_move_placeholder: mini_batch_moves})
  
                  # clear batches
                  del mini_batch_board_states[:]
                  del mini_batch_moves[:]
                  del mini_batch_rewards[:]
  				
  
              if episode_number % print_results_every == 0:
                  import matplotlib.pyplot as plt
                  episode_win.append(_win_rate(print_results_every, results)) 
                  episode_wrong_moves.append(mini_batch_illegal_moves/print_results_every) 
                  episode_count.append(episode_number)
                  
                  # red dashes, blue squares and green triangles
                  plt.plot(episode_count, episode_win, 'r', episode_count, episode_wrong_moves, 'g')
                  plt.show()
                  if( (episode_number // print_results_every)%20 == 0 ):
                      os.system('cls')				
  
                  print("episode: %s win_rate: %s illegal_moves: %s" % (episode_number, _win_rate(print_results_every, results),mini_batch_illegal_moves))
                  mini_batch_illegal_moves = 0
                  if network_file_path:
                      save_network(session, variables, save_network_file_path)
                      #print("file saved")
  
          if network_file_path:
              save_network(session, variables, save_network_file_path)
  
      return variables, _win_rate(print_results_every, results)
  
  
  def _win_rate(print_results_every, results):
      #print("Random Move Called")
      return 0.5 + sum(results) / (print_results_every * 2.)
  