text: # Copyright (c) Microsoft Corporation. All rights reserved.
  # Licensed under the MIT License.
  
  # Use `pip install pyswagger requests` to install pyswagger and requests
  from pyswagger import App
  from pyswagger.contrib.client.requests import Client
  
  # Use `pip install numpy pandas` to install numpy and pandas
  import numpy as np
  import pandas as pd
  
  import argparse
  import base64
  import hashlib
  import os
  import random
  import time
  
  # Below class QLearningTable is copy from MorvanZhou's tutorials
  # https://github.com/MorvanZhou/Reinforcement-learning-with-tensorflow/blob/master/contents/2_Q_Learning_maze/RL_brain.py
  class QLearningTable:
      def __init__(self, actions, learning_rate=0.01, reward_decay=0.9, e_greedy=0.9):
          self.actions = actions    # a list
          self.lr = learning_rate
          self.gamma = reward_decay
          self.epsilon = e_greedy
          self.q_table = pd.DataFrame(columns=self.actions, dtype=np.float64)
  
      def choose_action(self, state):
          self.check_state_exist(state)
          # action selection
          if np.random.uniform() < self.epsilon:
              # choose best action
              state_action = self.q_table.loc[state, :]
              # some actions may have the same value, randomly choose on in these actions
              action = np.random.choice(state_action[state_action == np.max(state_action)].index)
          else:
              # choose random action
              action = np.random.choice(self.actions)
          return action
  
      def learn(self, s, a, r, s_):
          self.check_state_exist(s_)
          q_predict = self.q_table.loc[s, a]
          if s_ != 'terminal':
              q_target = r + self.gamma * self.q_table.loc[s_, :].max()    # next state is not terminal
          else:
              q_target = r    # next state is terminal
          self.q_table.loc[s, a] += self.lr * (q_target - q_predict)    # update
  
      def check_state_exist(self, state):
          if state not in self.q_table.index:
              # append new state to q table
              self.q_table = self.q_table.append(
                  pd.Series(
                      [0]*len(self.actions),
                      index=self.q_table.columns,
                      name=state,
                  )
              )
  
  
  def action1(gArray):
      number = 28
      if len(gArray) != 0:
          number = gArray[-1]
      return number, number
  
  def action2(gArray):
      number = 28
      if len(gArray) != 0:
          number = gArray[-1]*0.618
      return number, number
  
  def action3(gArray):
      number = 28
      if len(gArray) != 0:
          number = np.average(gArray[-5:])
      return number, number
  
  def action4(gArray):
      number = 28
      if len(gArray) != 0:
          number = np.average(gArray[-5:])*0.618
      return number, number
  
  def action5(gArray):
      number = 28
      if len(gArray) != 0:
          number = np.average(gArray[-10:])
      return number, number
  
  def action6(gArray):
      number = 28
      if len(gArray) != 0:
          number = np.average(gArray[-10:])*0.618
      return number, number
  
  def action7(gArray):
      if len(gArray) == 0:
          return 28, 28
      if len(gArray) == 1:
          return gArray[0], gArray[0]
      number = gArray[-1] / gArray[-2] * gArray[-1]
      if number <= 0:
          number = 0.001
      if number >= 100:
          number = 100*0.618
      return number, number
  
  def action8(gArray):
      if len(gArray) == 0:
          return 28, 28
      number1=50
      number2=50/30*0.618+np.average(gArray[-5:])
      return number1, number2
  
  actions=[]
  actions.append(action1)
  actions.append(action2)
  actions.append(action3)
  actions.append(action4)
  actions.append(action5)
  actions.append(action6)
  actions.append(action7)
  actions.append(action8)
  
  n_actions = len(actions)
  RL = QLearningTable(actions=list(range(n_actions)))
  
  def getState(gArray):
      if len(gArray) == 0 or len(gArray) == 1:
          return '0_0'
      else:
          sub = np.array(gArray[-10:])
          sub1 = sub[:-1]
          sub2 = sub[1:]
          dif = sub1 - sub2
          up = sum(1 for e in dif if e < 0)
          down = sum(1 for e in dif if e > 0)
          return '{}_{}'.format(up, down)
  
  lastState=None
  lastAction=None
  
  def GeneratePredictionNumbers(goldenNumberList, lastScore, numberCount):
      global lastState
      global lastAction
  
      state = getState(goldenNumberList)
  
      if lastState != None and lastAction != None:
          RL.learn(lastState, lastAction, lastScore, state)
  
      action = RL.choose_action(state)
      number1, number2 = actions[action](goldenNumberList)
  
      lastState = state
      lastAction = action
  
      return number1, number2
  
  # Init swagger client
  host = 'https://goldennumber.aiedu.msra.cn/'
  jsonpath = '/swagger/v1/swagger.json'
  app = App._create_(host + jsonpath)
  client = Client()
  
  # Make sure all the parameters have the right value
  def perProcess(roomid, userid, usertoken):
      userInfoFile = 'userinfo.txt'
      nickname = None
      tokenFile = 'token.txt'
  
      # if not specify userid, try read userid locally
      if userid is None:
          if os.path.isfile(userInfoFile):
              with open(userInfoFile) as f:
                  userid = f.read().split(',')[0]
  
      # verify userid is valide or not
      if userid:
          userResp = client.request(
                  app.op['User'](
                      uid = userid
                  ))
          if userResp.status == 400:
              print(f'Verify user ID failed: {userResp.data.message}')
              userid = None
          else:
              userid = userResp.data.userId
              nickname = userResp.data.nickName
              print(f'Use an exist player: {nickname}, User ID: {userid}')
  
      # create user if userid is empty or invalide
      if not userid:
          # random nickname
          nickname = 'AI Player ' + str(random.randint(0, 9999))
          userResp = client.request(
              app.op['NewUser'](
                  nickName = nickname
              ))
          assert userResp.status == 200
          userid = userResp.data.userId
          nickname = userResp.data.nickName
          print(f'Create a new player: {nickname}, User ID: {userid}')
  
      # save user information locally
      with open(userInfoFile, "w") as f:
          f.write("%s,%s" % (userid, nickname))
  
      if roomid is None:
          # Input the roomid if there is no roomid in args
          roomid = input("Input room id: ")
          try:
              roomid = int(roomid)
              print(f'You are using room {roomid}')
          except:
              roomid = 0
              print('Parse room id failed, default use room 0')
  
      if not usertoken and os.path.isfile(tokenFile):
          with open(tokenFile) as f:
              usertoken = f.read()
  
      return roomid, userid, usertoken
  
  def main(roomid, userid, usertoken):
      # Make sure all the parameters have the right value
      roomid, userid, usertoken = perProcess(roomid, userid, usertoken)
  
      while True:
          stateResp = client.request(
              app.op['State'](
                  uid = userid,
                  roomid = roomid
              ))
          if stateResp.status != 200:
              print('Network issue, query again after 1 second')
              time.sleep(1)
              continue
          state = stateResp.data
  
          if state.state == 2:
              print('The game has finished')
              break
  
          if state.state == 1:
              print('The game has not started, query again after 1 second')
              time.sleep(1)
              continue
  
          if state.hasSubmitted:
              print('Already submitted this round, wait for next round')
              if state.maxUserCount == 0:
                  time.sleep(state.leftTime + 1)
              else:
                  # One round can be finished when all players submitted their numbers if the room have set the max count of users, need to check the state every second.
                  time.sleep(1)
              continue
  
          print('\r\nThis is round ' + str(state.finishedRoundCount + 1))
  
          todayGoldenListResp = client.request(
              app.op['TodayGoldenList'](
                  roomid = roomid
              ))
          if todayGoldenListResp.status != 200:
              print('Network issue, query again after 1 second')
              time.sleep(1)
              continue
          todayGoldenList = todayGoldenListResp.data
          if len(todayGoldenList.goldenNumberList) != 0:
              print('Last golden number is: ' + str(todayGoldenList.goldenNumberList[-1]))
  
          lastRoundResp = client.request(
              app.op['History'](
                  roomid = roomid,
                  count = 1
              ))
          if lastRoundResp.status != 200:
              print('Network issue, query again after 1 second')
              time.sleep(1)
              continue
          lastScore = 0
          if len(lastRoundResp.data.rounds) > 0:
              scoreArray = [user for user in lastRoundResp.data.rounds[0].userNumbers if user.userId == userid]
              if len(scoreArray) == 1:
                  lastScore = scoreArray[0].score
          print('Last round score: {}'.format(lastScore))
  
          number1, number2 = GeneratePredictionNumbers(todayGoldenList.goldenNumberList, lastScore, state.numbers)
  
          computedToken = ''
          if state.enabledToken:
              mergedString = userid + state.roundId + usertoken
              computedToken = base64.b64encode(hashlib.sha256(mergedString.encode('utf-8')).digest()).decode('utf-8')
  
          if state.numbers == 2:
              submitRsp = client.request(
                  app.op['Submit'](
                      uid = userid,
                      rid = state.roundId,
                      n1 = str(number1),
                      n2 = str(number2),
                      token = computedToken
                  ))
              if submitRsp.status == 200:
                  print('You submit numbers: ' + str(number1) + ', ' + str(number2))
              else:
                  print('Error: ' + submitRsp.data.message)
                  time.sleep(1)
  
          else:
              submitRsp = client.request(
                  app.op['Submit'](
                      uid = userid,
                      rid = state.roundId,
                      n1 = str(number1),
                      token = computedToken
                  ))
              if submitRsp.status == 200:
                  print('You submit number: ' + str(number1))
              else:
                  print('Error: ' + submitRsp.data.message)
                  time.sleep(1)
  
  if __name__ == '__main__':
      parser = argparse.ArgumentParser()
      parser.add_argument('--roomid', type=int, help='Room ID', required=False)
      parser.add_argument('--userid', type=str, help='User ID', required=False)
      parser.add_argument('--token', type=str, help='User token', required=False)
      args = parser.parse_args()
      main(args.roomid, args.userid, args.token)