text: import numpy as np
  from numpy.lib.shape_base import dsplit
  
  
  class BlackjackGame:
      def __init__(self) -> None:
          # states 2-21 are considered for state values and returns arrays
          self.state_values = dict((x, 0) for x in range(2, 22))
  
          # returns are a dict of k=state, value=[total return for state, n_occurences of state]
          self.returns = dict((x, [0, 0]) for x in range(2, 22))
  
          self.player_policy = {}
          # parameters: player sum (s), dealer's visible card (d), usable ace (a)
          # stored in dictionary where key is (s,d,a) and value is T|F
          for s in range(2, 22):
              for d in range(2, 12):
                  for a in [True, False]:
                      if s < 20:
                          self.player_policy[(s, d, a)] = True
                      else:
                          self.player_policy[(s, d, a)] = False
  
      def playGame(self):
          # trace is an ordered list which captures the players states, actions, and rewards (SARSA...)
          game_trace = []
  
          player_hand = list(np.random.randint(low=2, high=12, size=(2,)))
          # use one ace if the starting hand is [ace, ace]
          if 11 in player_hand:
              player_hand[player_hand.index(11)] = 1
  
          dealer_hand = list(np.random.randint(low=2, high=12, size=(2,)))
          dealer_visible_card = dealer_hand[0]
  
          # players turn
          while True:
              game_trace.append(sum(player_hand))
  
              # returns T|F from policy
              player_hit = self.player_policy[(
                  sum(player_hand), dealer_visible_card, 11 in player_hand)]
              game_trace.append(player_hit)
              if player_hit:
                  player_hand.append(np.random.randint(low=2, high=12))
  
                  if sum(player_hand) > 21:
                      # check for usable ace
                      if 11 in player_hand:
                          player_hand[player_hand.index(11)] = 1
                      else:
                          # player goes bust and dealer need not play
                          game_trace.append(-1)
                          return game_trace
              else:
                  # player sticks
                  break
  
          # dealer's turn
          while True:
              dealer_hit = sum(dealer_hand) < 17
              if dealer_hit:
                  dealer_hand.append(np.random.randint(low=2, high=12))
              else:
                  if 11 in dealer_hand:
                      # usable ace
                      dealer_hand[dealer_hand.index(11)] = 1
                      continue
                  # dealer sticks
                  break
  
          # decide game
          p_sum = sum(player_hand)
          d_sum = sum(dealer_hand)
  
          # if the player went bust, the function has already returned
          if d_sum > 21:
              # dealer went bust, so player wins
              game_trace.append(1)
              return game_trace
  
          if p_sum <= 21 and d_sum <= 21:
              # neither went bust
              if p_sum > d_sum:
                  # player wins
                  game_trace.append(1)
              elif d_sum > p_sum:
                  # dealer wins
                  game_trace.append(-1)
              else:
                  # draw
                  game_trace.append(0)
  
          return game_trace
  
      def simulate(self):
          count = 0
          while True:
              game_trace = self.playGame()
              if count % 10**5 == 0:
                  self.prettyPrintValues(count)
              count += 1
  
              # there is only one reward per episode with is w/l/d = 1/-1/0 so that is also the return
              ret = game_trace.pop()
              for i in range(len(game_trace)//2):
                  s = game_trace[i*2]
                  a = game_trace[i*2+1]
  
                  new_sum = self.returns[s][0] + ret
                  n_visits = self.returns[s][1] + 1
  
                  # update returns dict
                  self.returns[s] = [new_sum, n_visits]
  
                  # update state value
                  self.state_values[s] = new_sum / n_visits
  
      def prettyPrintValues(self, count):
          print(f"{count}:")
          for k,v in self.state_values.items():
              if k == 2:
                  continue
              print(f"{k} = {round((v+1)/2, 3)}", end="\t")
              histo = int((v+1) * 20) * "*"
              print(histo)
          print()
  
  
  
  if __name__ == "__main__":
      game = BlackjackGame()
      game.simulate()
  