"""
# Logistic regression
Logistic regression is a statistical model that in its basic form uses a logistic function to model a binary dependent variable, although many more complex extensions exist. In regression analysis, logistic regression (or logit regression) is estimating the parameters of a logistic model (a form of binary regression).
"""
"""
# Providing Workflow Of the Logistic Regression
"""
"""
![logistic%20flow_with%20ROC](https://raw.githubusercontent.com/DASHANANT/ML_Algorithms/main/Logistic%20Regression/logistic%20flow_with%20ROC.png)
"""
"""
# How Sigmoid Function is used for logistic-regression?

It is used to Map the results into 0 to 1 Domain - which we called as probability of event being to specific class 
then the classification is done using a threshold which is decided by ROC AUC Curve

To get the Knowledge ROC AUC curve look at this notebook
[understanding-roc-auc-curve](https://www.kaggle.com/anantdashpute/understanding-roc-auc-curve)
"""
"""
![logistic-regression-in-machine-learning](https://static.javatpoint.com/tutorial/machine-learning/images/logistic-regression-in-machine-learning.png)
"""
"""
# Importing Libraires
"""
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn import linear_model
from numpy.linalg import inv,det,multi_dot,norm
from sklearn.metrics import confusion_matrix
from itertools import combinations
import warnings
warnings.filterwarnings('ignore')
"""
# Model Class Logistic Regression
"""
class LogisticRegression:  
    def __init__(self):
        self.x=None
        self.y=None        
        self.weights = None
        self.bias = None
        self.cost= [ ]        
        
    def Error_logit(self,x,y):
        r,c=x.shape
        x=np.array(x)
        y=np.array(y).reshape(1,r)
        for i in range(len(self.bias)):
            fx = np.dot(x, self.weights[i]) + self.bias[i]         
        y_pred = self.sigmoid_func(fx)        
        y_cal = np.array([1 if i > 0.5 else 0 for i in y_pred]).reshape(1,r)  
        
        MSE=(1/len(x))* np.sum(np.square(y_cal - y))       
        return MSE   

    def kfold_logistic(self,x,y):
        kf = KFold(n_splits=5) 
        x=np.array(x)
        y=np.array(y)      
        
        MinErr=[ ]
        alpha_range = [10**i for i in range(-6,2)]
        for j in alpha_range :
            l=[ ]
            for train_index,test_index in kf.split(x,y):
                #print(x.size,y.size)
                self.Logistic_Train(x,y,j,1000,10**-10)                
                pred= self.Error_logit(x[test_index],y[test_index])
                l.append(pred)
                
            
            MinErr.append(sum(l)/len(l))
            print(f"Learning rate: {j} mean error is : {sum(l)/len(l)}")
        
        k= np.argmin(MinErr)        
        optimal_alpha=(alpha_range[k])
        print("optimal Learning rate is-->{}".format(optimal_alpha))
        return optimal_alpha
   
         
    def sigmoid_func(self,z):
        return 1/(1+np.exp(-(z)))
    
    def Logistic_Train(self,x,y,alpha,itr,eps):  
        r,c=x.shape
        x=np.array(x)
        y=np.array(y).reshape(r,1)
        w=np.zeros((c,1))
        w_list=[w]
        bias=[ ]
        cost_list=[ ]
        w0=0
        for i in range(itr):
            fx = np.dot(x,w)+ w0
            h0=self.sigmoid_func(fx)
            Err = h0 - y  
            w = w - (alpha/r)* np.dot(x.T,Err)            
            w0 = w0 - alpha * np.sum(Err)            
            cost = - (np.sum(np.log(h0)*y+ np.log(1-h0)*(1-y)))            
            cost_list.append(cost)            
            w_list.append(w)
            bias.append(w0)
            if cost < eps:
                break 
                
        self.weights={ }
        self.bias={ }  
        K=np.unique(y)                          #for hyperplane parameters
        m=[*range(len(K))]
        
        # for multiclass classification , we run the binary classification with permutations of p(M,2)
        
        p=list(combinations(m, 2))         
        for i in range(len(p)):           
            self.weights[i]=w
            self.bias[i]=w0
            
        self.cost=cost_list        
        

    def confusion_mat(self,y_test,y_pred):        
        cm = confusion_matrix(y_test,y_pred)
        return cm   

    def Logistic_Test(self,x,y):
        r,c=x.shape
        x=np.array(x)
        y=np.array(y).reshape(r,1)
        for i in range(len(self.bias)):
            z= np.dot(x, self.weights[i]) + self.bias[i]
        y_pred = self.sigmoid_func(z)      
        y_cal = np.array([1 if i > 0.5 else 0 for i in y_pred]).reshape(r,1) 
        MSE=(1/len(x))* np.sum(np.square(y_cal - y))          
        print('Mean Square Error is-->{}'.format(MSE))
        cm=self.confusion_mat(y,y_cal)        
        self.performance(cm)
"""
# Plotting Function for Hyperplane of Logistic Classification
"""
def plot_decison_boundary(x_test,y_test, weight, bias):
        r,c = x_test.shape
        if(c<3):           
            #  2D hyperplane plotting
            x=x_test.iloc[:,0]
            for i in range(len(bias)):
                y_cal=-(bias[i]+ x *weight[i][0])/weight[i][1]
                plt.plot(x,y_cal)
            C=['red' if l==1 else 'blue' for l in y_test]
            plt.scatter(x_test.iloc[:,[0]], x_test.iloc[:,[1]], color= C ,label="Actual data")
            plt.xlabel('x_1',fontsize = 10)
            plt.ylabel('x_2',fontsize = 10)
            plt.legend(loc='best')
            plt.show()
            
        elif(c==3):
            # 3D hyperplane plotting
            fig=plt.figure(figsize=(8, 6))
            ax = fig.add_subplot(111, projection = '3d')
            C=['red' if l==1 else 'blue' for l in y_test]                    
            x1=x_test.iloc[:,0]          
            x2=x_test.iloc[:,1]
            tmp = np.linspace(-40,40,3)
            for i in range(len(bias)):
                x1,x2 = np.meshgrid(tmp,tmp)
                z = lambda x1,x2: -(bias[i]+(x1*weight[i][0])+(x2*weight[i][1]))/weight[i][2]
                ax.plot_surface(x1, x2, z(x1, x2))  
                    
            
            ax.scatter3D(x_test.iloc[:,[0]], x_test.iloc[:,[1]], x_test.iloc[:,[2]], color= C ,label="Actual data")
            ax.set_xlabel('X1',fontsize = 10)
            ax.set_ylabel('X2',fontsize = 10)
            ax.set_zlabel('X3',fontsize = 10)
            plt.legend(loc='best')
            plt.show()
        
        
        else:
            # plot is in higher dimenstion
            pass