# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: 'Python 3.8.10 64-bit (''dennybritz-reinforcement-learning-vHFc5M9T'':
#       pipenv)'
#     name: python3
# ---

# ### This is Example 4.3. Gambler’s Problem from Sutton's book.
#
# A gambler has the opportunity to make bets on the outcomes of a sequence of coin flips. 
# If the coin comes up heads, he wins as many dollars as he has staked on that flip; 
# if it is tails, he loses his stake. The game ends when the gambler wins by reaching his goal of $100, 
# or loses by running out of money. 
#
# On each flip, the gambler must decide what portion of his capital to stake, in integer numbers of dollars. 
# This problem can be formulated as an undiscounted, episodic, finite MDP. 
#
# The state is the gambler’s capital, s ∈ {1, 2, . . . , 99}.
# The actions are stakes, a ∈ {0, 1, . . . , min(s, 100 − s)}. 
# The reward is zero on all transitions except those on which the gambler reaches his goal, when it is +1.
#
# The state-value function then gives the probability of winning from each state. A policy is a mapping from levels of capital to stakes. The optimal policy maximizes the probability of reaching the goal. Let p_h denote the probability of the coin coming up heads. If p_h is known, then the entire problem is known and it can be solved, for instance, by value iteration.
#

import numpy as np
import sys
import matplotlib.pyplot as plt
if "../" not in sys.path:
  sys.path.append("../") 


#
# ### Exercise 4.9 (programming)
#
# Implement value iteration for the gambler’s problem and solve it for p_h = 0.25 and p_h = 0.55.
#
#

def value_iteration_for_gamblers(p_h, theta=0.0001, discount_factor=1.0):
    """
    Args:
        p_h: Probability of the coin coming up heads
    """
    V = np.zeros(101)
    """
    일반적으로 terminal state의 value는 0으로 정의함.
    책에서 V[100]=1로 해보라 해서 해봤더니, p_h에 따라서 결과가 달라지는 경우도 있음.
    """
    # V[100] = 1
    policy = np.zeros(100)


    def rewards(next_state):
        if next_state == 100:
            return 1
        else:
            return 0
    
    def one_step_lookahead(s, V):
        """
        Helper function to calculate the value for all action in a given state.
        
        Args:
            s: The gambler’s capital. Integer. (current state)
            V: The vector that contains values at each state. 
            rewards: The reward function.
                        
        Returns:
            A vector containing the expected value of each action. 
            Its length equals to the number of actions.
        """
        
        # Implement!
        A = []
        for action in range(1,min(s, 100-s)+1): 
            """
            state s 에서 취할 수 있는 action은 [0, min(s, 100-s)]
            근데 solution 에서는 [1,min(s,100-s)]라고 함.
            두 경우에서 value는 같은데, 0도 가능한 action 이라고 하면 
            optimal policy에서 optimal action 을 0이라고 하는 경우가 꽤 있음.
            """
            value_lose = rewards(s-action) + discount_factor * V[s-action]
            value_win = rewards(s+action) + discount_factor * V[s+action]
            action_value = (1-p_h) * value_lose + p_h * value_win # action = 0 일 경우 value 변화 없음
            A.append(action_value)
            
        return A
    

    # Implement!

    # update V
    while True:
        delta = 0
        for state in range(1,100): # state : 1-99 에 대해 value 들을 업데이트 (1과 100은 terminal state)
            v = V[state]
            action_value_list = one_step_lookahead(state,V)
            V[state] = np.max(action_value_list)
            delta = max(delta, np.abs(v-V[state]))
        
        if delta < theta :
            break
    
    # confirm greedy policy
    for state in range(1,100): # terminal state 를 제외한 state들에 대해 policy를 업데이트
        optimal_action = np.argmax(one_step_lookahead(state,V)) + 1
        policy[state] = optimal_action


    return policy, V

# +
policy, v = value_iteration_for_gamblers(0.4)

print("Optimized Policy:")
print(policy)
print("")

print("Optimized Value Function:")
print(v)
print("")

# +
# Plotting Final Policy (action stake) vs State (Capital)

# Implement!
plt.bar(range(1,100), policy.tolist()[1:])
plt.xlabel('state')
plt.ylabel('optimal action')
plt.show()

# +
# Plotting State Value vs State (Capital)

# Implement!
plt.plot(range(1,100),v[1:100])
plt.xlabel('state')
plt.ylabel('value')
plt.show()

# -



StarcoderdataJupyterScriptsDedupFiltered