# -*- coding: utf-8 -*-
"""Feature_Selection_Categorical_Chi_Squared.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1LrcTaAKZx0I_8yeLjG2E-cNs_4aTOzI4

# How to choose categorical columns in dataset

## Load Data
"""

import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
sns.set()
import warnings
warnings.simplefilter(action='ignore')

churn_master_df = pd.read_csv('https://raw.githubusercontent.com/srivatsan88/YouTubeLI/master/dataset/churn_data_st.csv')

churn_df = churn_master_df.copy()

churn_df.head()

"""customerID - Customer ID

gender - Whether the customer is a male or a female

SeniorCitizen - Whether the customer is a senior citizen or not (1, 0)

Partner - Whether the customer has a partner or not (Yes, No)

Dependents - Whether the customer has dependents or not (Yes, No)

tenure - Number of months the customer has stayed with the company

PhoneService - Whether the customer has a phone service or not (Yes, No)

MultipleLines - Whether the customer has multiple lines or not (Yes, No, No phone service)

InternetService - Customer’s internet service provider (DSL, Fiber optic, No)

OnlineSecurity - Whether the customer has online security or not (Yes, No, No internet service)

OnlineBackup - Whether the customer has online backup or not (Yes, No, No internet service)

DeviceProtection - Whether the customer has device protection or not (Yes, No, No internet service)

TechSupport - Whether the customer has tech support or not (Yes, No, No internet service)

StreamingTV - Whether the customer has streaming TV or not (Yes, No, No internet service)

StreamingMovies - Whether the customer has streaming movies or not (Yes, No, No internet service)

Contract - The contract term of the customer (Month-to-month, One year, Two year)

PaperlessBilling - Whether the customer has paperless billing or not (Yes, No)

PaymentMethod - The customer’s payment method (Electronic check, Mailed check, Bank transfer (automatic), Credit card (automatic))

MonthlyCharges - The amount charged to the customer monthly

TotalCharges - The total amount charged to the customer

Churn - Whether the customer churned or not (Yes or No)

## Feature Selection
"""

churn_df.info()

print("Number of Rows: %s "%(churn_df.shape[0]))
print("Number of Columns: %s "%(churn_df.shape[1]))
print("\n Features: ", churn_df.columns.to_list())
print("\n Missing Values: %s"%(churn_df.isnull().any()))
print("\n Unique values: %s"%(churn_df.nunique()))

"""## Null Hypothesis: -
There is no relationship between the two categorical values.

## Alternate Hypothesis: -
There is a relationship between the two categorical values.

Use p value = 0.05 to make the decision

Questions to answer: -

- Is there any relationship between users who have churned to genders of the user ?

- Is there any relationship between users who opted for paperless billing to users who have churned ?

# Chi-Squared Test: -

There are multiple relationship tests that are available. But here we use the chi-squared test.

This test can be applied to any 2 categorical columns. It need not be supervised/unsupervised.

Chi-squared test shows the significance of the relationship between two categorical data.
"""

churn_df['gender'].value_counts()

fig, ax = plt.subplots()
ax = sns.countplot(x="gender", hue="Churn", data=churn_df)

for p in ax.patches:
    height = p.get_height()
    ax.text(p.get_x() + p.get_width()/2., height+3, '{:1.2f}'.format(height/churn_df.shape[0]), ha="center")

"""## Is there any relationship between users who have churned to genders of the user ?

Statistics alone do not prove anything. All tests are relative likelihood

- To do a chi-squared test. We need to create a contigency table.

- Also called cross-tab.
"""

gender_churn_ct = pd.crosstab(index=churn_df["gender"], columns=churn_df["Churn"])

gender_churn_ct

print(gender_churn_ct.iloc[0].values)
print(gender_churn_ct.iloc[1].values)

from scipy import stats

"""scipy.stats.chi2_contingency(observed, correction=True, lambda_=None)

Chi-square test of independence of variables in a contingency table.

This function computes the chi-square statistic and p-value for the hypothesis test of independence of the observed frequencies in the contingency table [1] observed. 

The expected frequencies are computed based on the marginal sums under the assumption of independence; see scipy.stats.contingency.expected_freq.

The number of degrees of freedom is (expressed using numpy functions and attributes):
"""

(chi2, p, dof, _) = stats.chi2_contingency([gender_churn_ct.iloc[0].values, gender_churn_ct.iloc[1].values])

print("chi2: ",chi2)
print("p-value: ",p)
print("Degree of freedom",dof)

"""chi ^ 2 statistics =

X ^ 2 = sum((observed - Expected) ^ 2 / Expected)

Higher the chi2 value more is the correlation between the two variables.

dof is number of categories - 1

p-value > 0.05
hence we do not accept the alternate hypothesis and choose the null hypothesis.

That means we can drop the gender column it is not very useful.
"""

pd.crosstab(index=churn_df['gender'], columns=churn_df['Churn'], margins=True)

"""## Is there any relationship between users who opted for paperless billing to users who have churned ?"""

churn_df['PaperlessBilling'].value_counts()

fig, ax = plt.subplots()
ax = sns.countplot(x="PaperlessBilling", hue="Churn", data=churn_df)

for p in ax.patches:
    height = p.get_height()
    ax.text(p.get_x() + p.get_width()/2., height+3, '{:1.2f}'.format(height/churn_df.shape[0]), ha="center")

pb_churn_ct = pd.crosstab(index=churn_df['PaperlessBilling'], columns=churn_df['Churn'])

pb_churn_ct

print(pb_churn_ct.iloc[0])
print(pb_churn_ct.iloc[1])

(chi2, p, dof, _) = stats.chi2_contingency([pb_churn_ct.iloc[0].values, pb_churn_ct.iloc[1].values])

print("chi2: ",chi2)
print("p-value: ",p)
print("Degree of freedom",dof)

"""Practically it may not make much sense.

But here p-value < 0.05 hence we reject the null hypothesis in favour of alternate hypothesis.

That is there is a significant relationship in this data.
"""