import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

churn = pd.read_csv("telecom_churn_data.csv")

churn.head(10)

#To increase the display width of the data (i.e. #columns,rows)
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

churn.info(verbose=True)

churn.info(verbose=False)

churn.describe(include=[np.number])

churn.describe(include=[np.object])

ObjectVars = list(churn.columns[churn.dtypes == 'object'])
print(ObjectVars)

#Seperating the date, ID and numerical columns
churn_date=['last_date_of_month_6',
             'last_date_of_month_7',
             'last_date_of_month_8',
             'last_date_of_month_9',
             'date_of_last_rech_6',
             'date_of_last_rech_7',
             'date_of_last_rech_8',
             'date_of_last_rech_9',
             'date_of_last_rech_data_6',
             'date_of_last_rech_data_7',
             'date_of_last_rech_data_8',
             'date_of_last_rech_data_9']
churn_id = ["mobile_number","circle_id" ]
churn_num = [col for col in churn.columns if col not in churn_date + churn_id]

churn_num

# when you look at the CSV file you will realise that the FB and night pack users are actually categorical columns
#coded 0 and 1 for yes and no, so lets seperate them as well
churn_cat = ['fb_user_6',
 'fb_user_7',
 'fb_user_8',
 'fb_user_9','night_pck_user_6',
 'night_pck_user_7',
 'night_pck_user_8',
 'night_pck_user_9']
#new num columns will be as following
churn_num = [col for col in churn_num if col not in churn_cat]

churn_num

round(churn.isnull().sum()/len(churn.index),4)*100

"""
Explanation: <a href="https://colab.research.google.com/github/HemantTiwariGitHub/AndroidNDSunshineProgress/blob/master/Telecom_churn.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>
Reading Data
End of explanation
"""


# categorical columns
churn[churn_cat] = churn[churn_cat].apply(lambda x: x.fillna(-1))

round(churn.isnull().sum()/len(churn.index),4)*100

churn.head(5)

#we can impute the recharge columns such as total, avg and max with 0 as null valiue idicates no recharge
i = ["total_rech_data_6"
,"total_rech_data_7"
,"total_rech_data_8"
,"total_rech_data_9"
,"av_rech_amt_data_6"
,"av_rech_amt_data_7"
,"av_rech_amt_data_8"
,"av_rech_amt_data_9"
,"max_rech_data_6"
,"max_rech_data_7"
,"max_rech_data_8"
,"max_rech_data_9"]
churn[i]=churn[i].apply(lambda x: x.fillna(0))

churn[i].isna

round(churn.isnull().sum()/len(churn.index),4)*100 > 70

j=["date_of_last_rech_data_6",'date_of_last_rech_data_7',
      'date_of_last_rech_data_8','date_of_last_rech_data_9']
churn[j]=churn[j].apply(lambda x: x.fillna(0))

"""
Explanation: Missing value treatement
End of explanation
"""


#now lets remove these columns
removecolumn = churn.columns[100*(churn.isnull().sum()/len(churn.index)) > 70]
print(removecolumn)

churn=churn.drop(removecolumn,axis=1)
churn.shape

#lets drop id columns as well
churn=churn.drop(churn_id,axis=1)
churn.shape

#identifying columns with missing values >0
x=churn.columns[100*(churn.isnull().sum()/len(churn.index)) > 0]
x

#imputing with 0
churn[x]=churn[x].apply(lambda x: x.fillna(0))

round(churn.isnull().sum()/len(churn.index),4)*100 

churn.nunique()

#dropping columns with only one unique value as they don'y add value
churn = churn.drop(['loc_og_t2o_mou', 'std_og_t2o_mou', 'loc_ic_t2o_mou'],axis=1)
churn.shape

"""
Explanation: There are many columns with more than 70% null values
These columns don't add any value to the data analysis, so lets remove them
End of explanation
"""


#data recharge
churn['total_data_rech_6'] = churn['total_rech_data_6'] * churn['av_rech_amt_data_6']
churn['total_data_rech_7'] = churn['total_rech_data_7'] * churn['av_rech_amt_data_7']

#total recharge
#Hemant : does total_rech_amt doesnt already include total_data_rech ?
churn['amt_data_6'] = churn[['total_rech_amt_6','total_data_rech_6']].sum(axis=1)
churn['amt_data_7'] = churn[['total_rech_amt_7','total_data_rech_7']].sum(axis=1)

#average of both months to filter the customers
churn['AVG_amt_data_6_7'] = churn[['amt_data_6','amt_data_7']].mean(axis=1)
churn[['total_rech_amt_6','total_rech_amt_7','AVG_amt_data_6_7']].quantile(np.linspace(.1, 1, 9, 0))

churn_highvalue = churn.loc[churn.AVG_amt_data_6_7 >= churn.AVG_amt_data_6_7.quantile(0.7), :]
churn_highvalue = churn_highvalue.reset_index(drop=True)
churn_highvalue.shape

"""
Explanation: Identifying High value customers
we can filter high value customers based on the amount of recharge they have done in the good months (6&7)<br>
total recharge amount = data recharge + normal recharge amount<br>
data recharge = total_rech_data*av_rech_amt_data
End of explanation
"""


#tagging churn vs non churn 
# Hemant , changed to AND 
churn_highvalue['churn_tag'] = np.where(
                            ((churn_highvalue['total_ic_mou_9'] == 0.00) & (churn_highvalue['total_og_mou_9'] == 0.00))
                            & ((churn_highvalue['vol_2g_mb_9'] == 0.00) & (churn_highvalue['vol_3g_mb_9'] == 0.00))
                            , 1, 0)

# change data type to category
churn_highvalue.churn_tag = churn_highvalue.churn_tag.astype("category")

# print churn ratio
print("Churn Ratio:")
print(churn_highvalue.churn_tag.value_counts()*100/churn_highvalue.shape[0])

"""
Explanation: Tagging churn vs non-churn
customers whose outgoing and incoming calls minutes = 0 <br>
and useage of data 2g and 3g is 0, those customers are tagged as churned or 1
End of explanation
"""


#removing all the 9th month columns
churn_highvalue = churn_highvalue.drop(churn_highvalue.filter(regex='_9|sep', axis = 1).columns, axis=1)

# delete variables created to filter high-value customers
churn_highvalue = churn_highvalue.drop(['total_data_rech_6', 'total_data_rech_7',
                                      'amt_data_6', 'amt_data_7', 'AVG_amt_data_6_7'], axis=1)
churn_highvalue.shape

round((churn_highvalue.shape[0]/churn.shape[0]),4)*100

"""
Explanation: Before proceeding lets drop all the 9th month columns and, the columns of 6,7 that are not required
End of explanation
"""


churn_highvalue.info(verbose="true")

churn_v1 = churn_highvalue
churn_v1

"""
Explanation: 30% of customers are high value
End of explanation
"""


def plotCategoricalVariables(columnName,dataFrame):
    sns.set(style="whitegrid")
    ax = (dataFrame.groupby(columnName).size()/len(dataFrame.index)).plot.bar(color=sns.color_palette('dark', 15))
    ax.set(xlabel= columnName, ylabel='Count')

    plt.show()

 plt.figure(figsize=(7,4))
plotCategoricalVariables("churn_tag", churn_v1)


churn_v1.info(verbose=True)

"""
Explanation: EDA
End of explanation
"""


#Neeed balancing by synthetic generation of new churn records

Y = churn_v1.pop("churn_tag")
X = churn_v1.select_dtypes(exclude=['object'])

Y.head()

# Test Train Split should happen before balancing should be applied only on train data
from sklearn.model_selection import train_test_split

X_train, X_test, Y_train, Y_test = train_test_split(X,Y, train_size=0.8,test_size=0.2,random_state=1)

"""
Explanation: The dataset is imbalanced , only 8% churn customers.
We need class balancing.
End of explanation
"""


#Before Class Balancing
Y_DF = pd.DataFrame(Y_train)

plotCategoricalVariables("churn_tag", Y_DF)

# ADASYN Balancing , ADASYN is more advanced than SMOTE
from imblearn.over_sampling import ADASYN 

adaSyn = ADASYN()
X_Train_Bal, Y_Train_Bal = adaSyn.fit_sample(X_train, Y_train)
print(X_train.shape)
print(X_Train_Bal.shape)

Y_Train_Bal_DF = pd.DataFrame(Y_Train_Bal, columns = {"churn_tag"})
plotCategoricalVariables("churn_tag", Y_Train_Bal_DF)

"""
Explanation: Balancing the DataSet
End of explanation
"""


from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X_Train_Bal_Standardized = scaler.fit_transform(X_Train_Bal)
X_Test_Standardized = scaler.transform(X_test)

"""
Explanation: AdaSyn Balanced Training Set
Standardization Of Data
End of explanation
"""


from sklearn.decomposition import PCA
pca = PCA(random_state=100)
pca.fit(X_Train_Bal_Standardized)

pca.components_

pca.explained_variance_ratio_

plt.bar(range(1,len(pca.explained_variance_ratio_)+1), pca.explained_variance_ratio_)

var_cumu = np.cumsum(pca.explained_variance_ratio_)


fig = plt.figure(figsize = (15,10))
plt.plot(range(1,len(var_cumu)+1), var_cumu)



"""
Explanation: Applying PCA
End of explanation
"""


# for  better model , we will take 40 features 

pca40Components = PCA(n_components = 40, random_state=100)
X_Train_Bal_Standardized_PCA40 = pca40Components.fit_transform(X_Train_Bal_Standardized)
X_Test_Standardized_PCA40 = pca40Components.transform(X_Test_Standardized)
print(X_Train_Bal_Standardized_PCA40.shape)
print(X_Test_Standardized_PCA40.shape)

"""
Explanation: 40 Features explain ~ 85 % variance and 60 features ~ 90 % , with 100 features variance explained is almost 100 %
End of explanation
"""


from sklearn.model_selection import KFold
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import cross_val_score
from sklearn.tree import DecisionTreeClassifier
from sklearn import metrics

searchParamGrid = {
    'max_depth': range(5,15,3),
    'min_samples_leaf': range(50, 500, 50),
    'min_samples_split': range(100, 500, 100),
}



dtc=DecisionTreeClassifier(random_state=100)
grid_search = GridSearchCV(estimator = dtc, param_grid = searchParamGrid, 
                          cv = 5, verbose=2, n_jobs = 4)

grid_search.fit(X_Train_Bal_Standardized_PCA40,Y_Train_Bal )

print(grid_search.best_score_)
print(grid_search.best_params_)

dtc_best = DecisionTreeClassifier(max_depth=11,
                             min_samples_leaf=50, 
                             min_samples_split=100,
                             random_state=100)

dtc_best.fit(X_Train_Bal_Standardized_PCA40,Y_Train_Bal )

Y_TestPreds = dtc_best.predict(X_Test_Standardized_PCA40)

from sklearn.metrics import classification_report
print(classification_report(Y_test,Y_TestPreds))

"""
Explanation: Now the Data is Balanced, Standardized and of Reduced Dimensionality
We need to apply 
3 Models for good Prediction
1 Model for Interpretability

Decision Tree
End of explanation
"""


GBC = GradientBoostingClassifier(random_state=100)
    

from sklearn.ensemble import GradientBoostingClassifier
import xgboost as xgb
from xgboost import XGBClassifier
from xgboost import plot_importance

GBC.fit(X_Train_Bal_Standardized_PCA40,Y_Train_Bal)

Y_TestPreds = GBC.predict(X_Test_Standardized_PCA40)

print(classification_report(Y_test,Y_TestPreds))

"""
Explanation: Recall is important as we want to identify all churn cases.
Using Grid Search with Decision Tree we get Test Recall of 0.73 

Gradient Boost
End of explanation
"""


# parameter grid
param_grid = {"learning_rate": [0.07],
              "subsample": [0.8],
              "n_estimators" : [150, 200],
              "max_depth" : [5,10]
             }

folds = 3
grid_search_GBC = GridSearchCV(GBC, 
                               cv = folds,
                               param_grid=param_grid, 
                               scoring = 'roc_auc', 
                               return_train_score=True,                         
                               verbose = 2,
                               n_jobs =5)

grid_search_GBC.fit(X_Train_Bal_Standardized_PCA40,Y_Train_Bal)

print(grid_search_GBC.best_score_)
print(grid_search_GBC.best_params_)

GBC_best = GradientBoostingClassifier(learning_rate=0.07,
                             max_depth=10, 
                             n_estimators=200,
                             subsample =0.8,
                             random_state=100)
GBC_best.fit(X_Train_Bal_Standardized_PCA40,Y_Train_Bal)

Y_TestPreds = GBC_best.predict(X_Test_Standardized_PCA40)
print(classification_report(Y_test,Y_TestPreds))

"""
Explanation: Without Any Tuning Gradient Boost Got Recall of .80 on Churn Detection
End of explanation
"""


model = XGBClassifier()
model.fit(X_Train_Bal_Standardized_PCA40,Y_Train_Bal)

Y_TestPreds = model.predict(X_Test_Standardized_PCA40)
print(classification_report(Y_test,Y_TestPreds))

"""
Explanation: XGBoost
End of explanation
"""


import statsmodels.api as sm

logm1 = sm.GLM(Y_Train_Bal,(sm.add_constant(X_Train_Bal_Standardized)), family = sm.families.Binomial())
logm1.fit().summary()

from sklearn.linear_model import LogisticRegression
logreg = LogisticRegression(max_iter=500)

from sklearn.feature_selection import RFE
rfe = RFE(logreg, 40)             
rfe = rfe.fit(X_Train_Bal_Standardized, Y_Train_Bal)

rfe.support_

Importance = list(zip(X_train.columns, rfe.support_, rfe.ranking_))

col = X_train.columns[rfe.support_]

Y_Pred = rfe.predict(X_Test_Standardized)

print(classification_report(Y_test,Y_TestPreds))

print(Importance)

def sortingVal(row):
  return row[2]

Importance.sort( key=sortingVal)

print (Importance)

# Top 10 Predictors for Churn
for item in Importance[:10]:
    print(item[0])

"""
Explanation: XG Boost also gives recall of 0.80 for Churn Prediction
Interpretability
We use the Logistic Regression here w/o PCA to determine insights.
End of explanation
"""


