# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
data = pd.read_csv("/kaggle/input/gender-classification-dataset/gender_classification_v7.csv")
data.columns
data.info()
data.head(10)
data.describe()
#data.drop("nose_wide",inplace = True, axis=1)
#data.drop("nose_long",inplace = True, axis = 1)
#data.drop("lips_thin",inplace = True, axis = 1)
#data.drop("distance_nose_to_lip_long",inplace = True, axis = 1)
data
"""
### Train - Test Split
"""
y = data.gender.values

x = data[["long_hair","forehead_width_cm","forehead_height_cm","nose_wide","nose_long","lips_thin","distance_nose_to_lip_long"]].values
from sklearn.model_selection import train_test_split

x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.2, random_state = 42)

x_train = x_train.T
x_test = x_test.T
y_train = y_train.T
y_test = y_test.T

print("x_train : ",x_train)
print("x_test : ",x_test)
print("y_train : ",y_train)
print("y_test : ",y_test)
"""
### Logistic Regression
"""
from sklearn.model_selection import train_test_split
x_test, x_train, y_test, y_train = train_test_split(x, y, test_size = 0.15, random_state = 42)
from sklearn import linear_model

logreg = linear_model.LogisticRegression(random_state = 42, max_iter = 150)

print("Test Accuracy : ", (logreg.fit(x_train, y_train).score(x_test, y_test)))
print("Train Accuracy : ", (logreg.fit(x_train, y_train).score(x_train, y_train)))
"""
### KNN Classification
"""
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier

x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.2, random_state = 42)
knn = KNeighborsClassifier(n_neighbors = 9)
knn.fit(x_train,y_train)
prediction = knn.predict(x_test)

from sklearn.metrics import confusion_matrix
import seaborn as sns
f, ax = plt.subplots(figsize = (5,5))
cm = confusion_matrix(y_test, knn.predict(x_test))
sns.heatmap(cm,annot = True, linewidth = 0.5, linecolor = "red", fmt = ".0f", ax = ax)
plt.show()
print("Confusion Matrix : ", cm)
print(" {}nn Score: {} ".format(3, knn.score(x_test,y_test)))

score_list = []
for each in range(1,15):
    knn2 = KNeighborsClassifier(n_neighbors = each)
    knn2.fit(x_train, y_train)
    score_list.append(knn2.score(x_test, y_test))

plt.plot(range(1,15), score_list)
plt.xlabel("k values")
plt.ylabel("Accuracy")
plt.show()
"""
### SVM (Support Vector Machine)
"""
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.15, random_state = 1)

from sklearn.svm import SVC
svm = SVC(random_state = 1)
svm.fit(x_train, y_train)
print("Accuracy of SVM Algo : ", svm.score(x_test, y_test))

from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, svm.predict(x_test))

print("Confusion Matrix : ", cm)
import seaborn as sns
f, ax = plt.subplots(figsize = (5,5))
sns.heatmap(cm, annot = True, linewidth = 0.5, linecolor = "red", fmt = ".0f", ax = ax)
plt.show()
"""
### Naive Bayes
"""
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.3, random_state = 1)

from sklearn.naive_bayes import GaussianNB
nb = GaussianNB()
nb.fit(x_train, y_train)

from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, nb.predict(x_test))

print("Accuracy of Naive Bayes Algo : ", nb.score(x_test, y_test))
print("Confusion Matrix : ", cm)

import seaborn as sns
f, ax = plt.subplots(figsize = (5,5))
sns.heatmap(cm, annot = True, linewidth = 0.5, linecolor = "red", fmt = ".0f", ax = ax)
plt.show()
"""
### Decision Tree Learning
"""
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.15, random_state = 1)

from sklearn.tree import DecisionTreeClassifier
dt = DecisionTreeClassifier()
dt.fit(x_train, y_train)

from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, dt.predict(x_test))
print("Score : ", dt.score(x_test, y_test))
print("Confusion Matrix : ", cm)

import seaborn as sns
f ,ax = plt.subplots(figsize = (5,5))
sns.heatmap(cm, annot = True, linewidth = 0.5, linecolor = "red", fmt = ".0f", ax = ax)
plt.show()
"""
### Random Forest Classification
"""
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.15, random_state = 42)

from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier(n_estimators = 100, random_state = 1)
rf.fit(x_train, y_train)

from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, rf.predict(x_test))

print("Random Forest Algo Result : ", rf.score(x_test, y_test))
print("Confusion Matrix : ", cm)

import seaborn as sns
f, ax = plt.subplots(figsize = (5,5))
sns.heatmap(cm, annot = True, linewidth = 0.5, linecolor = "red", fmt = ".0f", ax = ax)
plt.show()
"""
### Without Sklearn Library
"""
from sklearn.model_selection import train_test_split

x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.1, random_state = 42)

x_train = x_train.T
x_test = x_test.T
y_train = y_train.T
y_test = y_test.T

print("x_train : ",x_train)
print("x_test : ",x_test)
print("y_train : ",y_train)
print("y_test : ",y_test)
def initialize_weights_and_bias(dimension):
    w = np.full((dimension,1),0.01)
    b = 0.0
    return w,b
#Calculation of Z
def sigmoid(z):
    y_head = 1/(1+np.exp(-z))
    return y_head
#Forward Propagation
def forward_propagation(w, b, x_train, y_train):
    z = np.dot(w.T, x_train,) + b
    y_head = sigmoid(z)
    loss = -y_train * np.log(y_head) - (1 - y_train) * np.log(1 - y_head)
    cost = (np.sum(loss)) / x_train.shape[1] #x_train.shape[1] is for scaling(optimizing)
    
    return cost
#In Backward Propagation, We will use y_head that found in Forward Propagation. 
#For this reason, instead of writing Backward Propagation method, Let's combine Forward and Backward Propagation

def forward_backward_propagation(w, b, x_train, y_train):
    #Forward Propagation
    z = np.dot(w.T, x_train) + b
    y_head = sigmoid(z)
    loss = -y_train * np.log(y_head) - (1 - y_train) * np.log(1 - y_head)
    cost = (np.sum(loss)) / x_train.shape[1] #x_train.shape[1] is for scaling(optimizing)
    
    #Backward Propagation
    derivative_weight = (np.dot(x_train, ((y_head - y_train).T))) / x_train.shape[1] # x_train.shape[1] another scaling
    derivative_bias = np.sum(y_head - y_train) / x_train.shape[1] # x_train.shape[1] another scaling
    
    gradients = {"derivative_weight" : derivative_weight, "derivative_bias" : derivative_bias}
    
    return cost, gradients
#In Backward Propagation, We will use y_head that found in Forward Propagation. 
#For this reason, instead of writing Backward Propagation method, Let's combine Forward and Backward Propagation

def forward_backward_propagation(w, b, x_train, y_train):
    #Forward Propagation
    z = np.dot(w.T, x_train) + b
    y_head = sigmoid(z)
    loss = -y_train * np.log(y_head) - (1 - y_train) * np.log(1 - y_head)
    cost = (np.sum(loss)) / x_train.shape[1] #x_train.shape[1] is for scaling(optimizing)
    
    #Backward Propagation
    derivative_weight = (np.dot(x_train, ((y_head - y_train).T))) / x_train.shape[1] # x_train.shape[1] another scaling
    derivative_bias = np.sum(y_head - y_train) / x_train.shape[1] # x_train.shape[1] another scaling
    
    gradients = {"derivative_weight" : derivative_weight, "derivative_bias" : derivative_bias}
    
    return cost, gradients
#Updating (learning) Parameters

def update(w, b, x_train, y_train, learning_rate, number_of_iteration):
    cost_list = []
    cost_list2 = []
    index = []
    
    #Updating (learning) parameters is number_of_iteration times
    for i in range(number_of_iteration):
        
        #Make forward and backward propagation and find cost and gradients
        cost, gradients = forward_backward_propagation(w, b, x_train, y_train)
        cost_list.append(cost)
        
        #Let's Update
        w = w - learning_rate * gradients["derivative_weight"]
        b = b - learning_rate * gradients["derivative_bias"]
        
        if i % 10 == 0:
            cost_list2.append(cost)
            index.append(i)
            print ("Cost after iteration %i: %f" %(i, cost))
        
        #We update(learn) parameters weights and bias
        parameters = {"weight": w,"bias": b}
        plt.plot(index,cost_list2)
        plt.xticks(index,rotation='vertical')
        plt.xlabel("Number of Iterarion")
        plt.ylabel("Cost")
        plt.show()
        
        return parameters, gradients, cost_list
#Prediction
def predict(w, b, x_test):
    
    #x_test is a input for forward propagation
    z = sigmoid(np.dot(w.T, x_test) + b)
    Y_prediction = np.zeros((1, x_test.shape[1]))
    
    # if z is bigger than 0.5, our prediction is Male (y_head = 1)
    # if z is lower tan 0.5, our prediction is Female (y_head = 0)
    for i in range(z.shape[1]):
        if z[0,i] <= 0.5:
            Y_prediction[0,i] = 0
        else:
            Y_prediction[0,i] = 1
    return Y_prediction
#We have made our prediction
#Now, we're putting them together

def logistic_regression(x_train, y_train, x_test, y_test, learning_rate, num_iteration):
    #Initialize
    dimension = x_train.shape[0] #Which 5001
    w, b = initialize_weights_and_bias(dimension)
    
    #Do not change learning rate
    parameters, gradients, cost_list = update(w, b, x_train, y_train, learning_rate, num_iteration)
    
    y_prediction_test = predict(parameters["weight"], parameters["bias"], x_test)
    y_prediction_train = predict(parameters["weight"], parameters["bias"], x_train)
    
    #Print Train/Test Errors
    print("Train Accuracy : ", (100 - np.mean(np.abs(y_prediction_train - y_train) * 100)))
    print("Test Accuracy : ", (100 - np.mean(np.abs(y_prediction_test - y_test) * 100)))
    
logistic_regression(x_train, y_train, x_test, y_test, learning_rate = 30, num_iteration = 150)