text: import numpy as np
  import pandas as pd
  import time
  
  
  np.random.seed(2)
  
  
  N_STATES = 6    # the length of the 1 dimensional world
  ACTION = ['left', 'right']    # available actions
  EPSILON = 0.9    # greedy police
  ALPHA = 0.1      # learning rate
  LAMBDA = 0.9     # discount factor
  MAX_EPISODES = 13    # maximum episodes
  FRESH_TIME = 0.3     # fresh time for one move
  
  
  def build_q_table(n_states, actions):
    table = pd.DataFrame(
        np.zeros((n_states, len(actions))),
        columns=actions
    )
    # print(table)
    return table
  
  
  def choose_action(state, q_table):
    state_actions = q_table.iloc[state, :]
    if (np.random.uniform() > EPSILON) or (state_actions.any() == 0):
      action_name = np.random.choice(ACTION)
    else:
      action_name = state_actions.idxmax()
    return action_name
  
  
  
  def get_env_feedback(S, A):
    if A == 'right':
      if S == N_STATES - 2:
        S_ = 'terminal'
        R = 1
      else:
        S_ = S + 1
        R = 0
    else:
      R = 0
      if S == 0:
        S_ = S
      else:
        S_ = S - 1
    return S_, R
  
  
  def update_env(S, episode, step_counter):
    env_list = ['-'] * (N_STATES - 1) + ['T']
    if S == 'terminal':
      interaction = 'Episode %s: total_step = %s' % (episode + 1, step_counter)
      print('\r{}'.format(interaction), end='')
      time.sleep(2)
      print('\r                                ', end='')
    else:
      env_list[S] = 'o'
      interaction = ''.join(env_list)
      print('\r{}'.format(interaction), end='')
      time.sleep(FRESH_TIME)
  
  
  def rl():
    q_table = build_q_table(N_STATES, ACTION)
    for episode in range(MAX_EPISODES):
      step_counter = 0
      S = 0
      is_terminated = False
      update_env(S, episode, step_counter)
      while not is_terminated:
        A = choose_action(S, q_table)
        S_, R = get_env_feedback(S, A)
        q_predict = q_table.loc[S, A]
        if S_ != 'terminal':
          q_target = R + LAMBDA * q_table.loc[S_, :].max()
        else:
          q_target = R
          is_terminated = True
  
        q_table.loc[S, A] += ALPHA * (q_target - q_predict)
        S = S_
  
        update_env(S, episode, step_counter + 1)
        step_counter += 1
  
    return q_table
  
  
  if __name__ == "__main__":
      q_table = rl()
      print('\r\nQ-table:\n')
      print(q_table)