# coding:utf-8

import time
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import sklearn.datasets
import sklearn.linear_model
import matplotlib

matplotlib.rcParams['figure.figsize'] = (10.0, 8.0)

np.random.seed(0)
X, y = sklearn.datasets.make_moons(200, noise=0.20)
plt.scatter(X[:,0], X[:,1], s=40, c=y, cmap=plt.cm.Spectral)

# plt.show()
clf = sklearn.linear_model.LogisticRegressionCV()
clf.fit(X, y)

# Helper function to plot a decision boundary.
# If you don't fully understand this function don't worry, it just generates
# the contour plot below.
def plot_decision_boundary(pred_func):
    # Set min and max values and give it some padding
    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
    h = 0.01
    # Generate a grid of points with distance h between them
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max,
        h))
    # Predict the function value for the whole gid
    Z = pred_func(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    # Plot the contour and training examples
    plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
    plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Spectral)



plot_decision_boundary(lambda x: clf.predict(x))
plt.title("Logistic Regression")
#plt.show()

num_examples = len(X) # training set size
nn_input_dim = 2 # input layer dimensionality
nn_output_dim = 2 # output layer dimensionality

# Gradient descent parameters (I picked these by hand)
epsilon = 0.01 # learning rate for gradient descent
reg_lambda = 0.01 # regularization strength


# Helper function to evaluate the total loss on the dataset
def calculate_loss(model):
    W1, b1, W2, b2 = model['W1'], model['b1'], model['W2'], model['b2']
    # Forward propagation to calculate our predictions
    z1 = X.dot(W1) + b1
    a1 = np.tanh(z1)
    z2 = a1.dot(W2) + b2
    exp_scores = np.exp(z2)
    probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)
    # Calculating the loss

    #print(11111111)
    #print(probs)
    
    #time.sleep(10)
    corect_logprobs = -np.log(probs[range(num_examples), y])
    data_loss = np.sum(corect_logprobs)
    # Add regulatization term to loss (optional)
    # L2 regulatization
    data_loss += reg_lambda/2 * (np.sum(np.square(W1)) + np.sum(np.square(W2)))
    return 1./num_examples * data_loss

def predict(model, x):
    W1, b1, W2, b2 = model['W1'], model['b1'], model['W2'], model['b2']
    # Forward propagation
    z1 = x.dot(W1) + b1
    a1 = np.tanh(z1)
    z2 = a1.dot(W2) + b2
    exp_scores = np.exp(z2)
    probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)
    return np.argmax(probs, axis=1)

# This function learns parameters for the neural network and returns the model.
# - nn_hdim: Number of nodes in the hidden layer
# - num_passes: Number of passes through the training data for gradient descent
# - print_loss: If True, print the loss every 1000 iterations
def build_model(nn_hdim, num_passes=20000, print_loss=False):
    # Initialize the parameters to random values. We need to learn these.
    np.random.seed(0)
    W1 = np.random.randn(nn_input_dim, nn_hdim) / np.sqrt(nn_input_dim)
    b1 = np.zeros((1, nn_hdim))
    W2 = np.random.randn(nn_hdim, nn_output_dim) / np.sqrt(nn_hdim)
    b2 = np.zeros((1, nn_output_dim))
    # This is what we return at the end
    model = {}
    # Gradient descent. For each batch...
    for i in range(0, num_passes):
        # Forward propagation
        z1 = X.dot(W1) + b1
        a1 = np.tanh(z1)
        z2 = a1.dot(W2) + b2
        exp_scores = np.exp(z2)
        probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)
        # Backpropagation
        delta3 = probs
        delta3[range(num_examples), y] -= 1
        dW2 = (a1.T).dot(delta3)
        db2 = np.sum(delta3, axis=0, keepdims=True)
        delta2 = delta3.dot(W2.T) * (1 - np.power(a1, 2))
        dW1 = np.dot(X.T, delta2)
        db1 = np.sum(delta2, axis=0)
        # Add regularization terms (b1 and b2 don't have regularization terms)
        dW2 += reg_lambda * W2
        dW1 += reg_lambda * W1
        # Gradient descent parameter update
        W1 += -epsilon * dW1
        b1 += -epsilon * db1
        W2 += -epsilon * dW2
        b2 += -epsilon * db2
        # Assign new parameters to the model
        model = { 'W1': W1, 'b1': b1, 'W2': W2, 'b2': b2}
        # Optionally print the loss.
        # This is expensive because it uses the whole dataset, so we don't want to do it too often.
        if print_loss and i % 1000 == 0:
            print("Loss after iteration %i: %f" %(i, calculate_loss(model)))
    return model


def test_1():
    # Build a model with a 3-dimensional hidden layer
    model = build_model(3, print_loss=True)
    # Plot the decision boundary
    plot_decision_boundary(lambda x: predict(model, x))
    plt.title("Decision Boundary for hidden layer size 3")
    plt.show()    

def test_2():
    plt.figure(figsize=(16, 32))
    hidden_layer_dimensions = [1, 2, 3, 4, 5, 20, 50]
    for i, nn_hdim in enumerate(hidden_layer_dimensions):
        plt.subplot(5, 2, i+1)
        plt.title('Hidden Layer size %d' % nn_hdim)
        model = build_model(nn_hdim)
        plot_decision_boundary(lambda x: predict(model, x))
    plt.show()


if __name__ == '__main__':
    #print(y)
    #print(12121)
    #print(X)
    test_1()

[((13, 0, 13, 17), 'numpy.random.seed', 'np.random.seed', ({(13, 15, 13, 16): '(0)'}, {}), '(0)', True, 'import numpy as np\n'), ((14, 7, 14, 51), 'sklearn.datasets.make_moons', 'sklearn.datasets.make_moons', (), '', False, 'import sklearn\n'), ((15, 0, 15, 60), 'matplotlib.pyplot.scatter', 'plt.scatter', (), '', True, 'import matplotlib.pyplot as plt\n'), ((18, 6, 18, 49), 'sklearn.linear_model.LogisticRegressionCV', 'sklearn.linear_model.LogisticRegressionCV', ({}, {}), '()', False, 'import sklearn\n'), ((42, 0, 42, 32), 'matplotlib.pyplot.title', 'plt.title', ({(42, 10, 42, 31): '"""Logistic Regression"""'}, {}), "('Logistic Regression')", True, 'import matplotlib.pyplot as plt\n'), ((36, 4, 36, 49), 'matplotlib.pyplot.contourf', 'plt.contourf', (), '', True, 'import matplotlib.pyplot as plt\n'), ((37, 4, 37, 60), 'matplotlib.pyplot.scatter', 'plt.scatter', (), '', True, 'import matplotlib.pyplot as plt\n'), ((59, 9, 59, 20), 'numpy.tanh', 'np.tanh', ({(59, 17, 59, 19): 'z1'}, {}), '(z1)', True, 'import numpy as np\n'), ((61, 17, 61, 27), 'numpy.exp', 'np.exp', ({(61, 24, 61, 26): 'z2'}, {}), '(z2)', True, 'import numpy as np\n'), ((70, 16, 70, 39), 'numpy.sum', 'np.sum', ({(70, 23, 70, 38): 'corect_logprobs'}, {}), '(corect_logprobs)', True, 'import numpy as np\n'), ((80, 9, 80, 20), 'numpy.tanh', 'np.tanh', ({(80, 17, 80, 19): 'z1'}, {}), '(z1)', True, 'import numpy as np\n'), ((82, 17, 82, 27), 'numpy.exp', 'np.exp', ({(82, 24, 82, 26): 'z2'}, {}), '(z2)', True, 'import numpy as np\n'), ((84, 11, 84, 35), 'numpy.argmax', 'np.argmax', (), '', True, 'import numpy as np\n'), ((92, 4, 92, 21), 'numpy.random.seed', 'np.random.seed', ({(92, 19, 92, 20): '(0)'}, {}), '(0)', True, 'import numpy as np\n'), ((94, 9, 94, 31), 'numpy.zeros', 'np.zeros', ({(94, 18, 94, 30): '(1, nn_hdim)'}, {}), '((1, nn_hdim))', True, 'import numpy as np\n'), ((96, 9, 96, 37), 'numpy.zeros', 'np.zeros', ({(96, 18, 96, 36): '(1, nn_output_dim)'}, {}), '((1, nn_output_dim))', True, 'import numpy as np\n'), ((137, 4, 137, 58), 'matplotlib.pyplot.title', 'plt.title', ({(137, 14, 137, 57): '"""Decision Boundary for hidden layer size 3"""'}, {}), "('Decision Boundary for hidden layer size 3')", True, 'import matplotlib.pyplot as plt\n'), ((138, 4, 138, 14), 'matplotlib.pyplot.show', 'plt.show', ({}, {}), '()', True, 'import matplotlib.pyplot as plt\n'), ((141, 4, 141, 32), 'matplotlib.pyplot.figure', 'plt.figure', (), '', True, 'import matplotlib.pyplot as plt\n'), ((148, 4, 148, 14), 'matplotlib.pyplot.show', 'plt.show', ({}, {}), '()', True, 'import matplotlib.pyplot as plt\n'), ((30, 25, 30, 51), 'numpy.arange', 'np.arange', ({(30, 35, 30, 40): 'x_min', (30, 42, 30, 47): 'x_max', (30, 49, 30, 50): 'h'}, {}), '(x_min, x_max, h)', True, 'import numpy as np\n'), ((30, 53, 31, 10), 'numpy.arange', 'np.arange', ({(30, 63, 30, 68): 'y_min', (30, 70, 30, 75): 'y_max', (31, 8, 31, 9): 'h'}, {}), '(y_min, y_max, h)', True, 'import numpy as np\n'), ((62, 25, 62, 66), 'numpy.sum', 'np.sum', (), '', True, 'import numpy as np\n'), ((83, 25, 83, 66), 'numpy.sum', 'np.sum', (), '', True, 'import numpy as np\n'), ((93, 9, 93, 47), 'numpy.random.randn', 'np.random.randn', ({(93, 25, 93, 37): 'nn_input_dim', (93, 39, 93, 46): 'nn_hdim'}, {}), '(nn_input_dim, nn_hdim)', True, 'import numpy as np\n'), ((93, 50, 93, 71), 'numpy.sqrt', 'np.sqrt', ({(93, 58, 93, 70): 'nn_input_dim'}, {}), '(nn_input_dim)', True, 'import numpy as np\n'), ((95, 9, 95, 48), 'numpy.random.randn', 'np.random.randn', ({(95, 25, 95, 32): 'nn_hdim', (95, 34, 95, 47): 'nn_output_dim'}, {}), '(nn_hdim, nn_output_dim)', True, 'import numpy as np\n'), ((95, 51, 95, 67), 'numpy.sqrt', 'np.sqrt', ({(95, 59, 95, 66): 'nn_hdim'}, {}), '(nn_hdim)', True, 'import numpy as np\n'), ((103, 13, 103, 24), 'numpy.tanh', 'np.tanh', ({(103, 21, 103, 23): 'z1'}, {}), '(z1)', True, 'import numpy as np\n'), ((105, 21, 105, 31), 'numpy.exp', 'np.exp', ({(105, 28, 105, 30): 'z2'}, {}), '(z2)', True, 'import numpy as np\n'), ((111, 14, 111, 51), 'numpy.sum', 'np.sum', (), '', True, 'import numpy as np\n'), ((113, 14, 113, 33), 'numpy.dot', 'np.dot', ({(113, 21, 113, 24): 'X.T', (113, 26, 113, 32): 'delta2'}, {}), '(X.T, delta2)', True, 'import numpy as np\n'), ((114, 14, 114, 36), 'numpy.sum', 'np.sum', (), '', True, 'import numpy as np\n'), ((144, 8, 144, 30), 'matplotlib.pyplot.subplot', 'plt.subplot', ({(144, 20, 144, 21): '(5)', (144, 23, 144, 24): '(2)', (144, 26, 144, 29): '(i + 1)'}, {}), '(5, 2, i + 1)', True, 'import matplotlib.pyplot as plt\n'), ((145, 8, 145, 51), 'matplotlib.pyplot.title', 'plt.title', ({(145, 18, 145, 50): "('Hidden Layer size %d' % nn_hdim)"}, {}), "('Hidden Layer size %d' % nn_hdim)", True, 'import matplotlib.pyplot as plt\n'), ((106, 29, 106, 70), 'numpy.sum', 'np.sum', (), '', True, 'import numpy as np\n'), ((73, 40, 73, 53), 'numpy.square', 'np.square', ({(73, 50, 73, 52): 'W1'}, {}), '(W1)', True, 'import numpy as np\n'), ((73, 64, 73, 77), 'numpy.square', 'np.square', ({(73, 74, 73, 76): 'W2'}, {}), '(W2)', True, 'import numpy as np\n'), ((112, 41, 112, 56), 'numpy.power', 'np.power', ({(112, 50, 112, 52): 'a1', (112, 54, 112, 55): '(2)'}, {}), '(a1, 2)', True, 'import numpy as np\n')]