pip install lifelines
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
from scipy import stats
import warnings
warnings.filterwarnings('ignore')
from lifelines import KaplanMeierFitter
from lifelines.statistics import logrank_test
from lifelines.statistics import multivariate_logrank_test
from lifelines.statistics import pairwise_logrank_test
from lifelines import CoxPHFitter

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))
df = pd.read_csv('/kaggle/input/telecom-churn/telecom_churn.csv').reset_index()
df.head()
df.info()
"""
**Columns**:
* Churn - 1 if customer cancelled service, 0 if not
* AccountWeeks - number of weeks customer has had active account
* ContractRenewal - 1 if customer recently renewed contract, 0 if not
* DataPlan - 1 if customer has data plan, 0 if not
* DataUsage - gigabytes of monthly data usage
* CustServCalls - number of calls into customer service
* DayMins - average daytime minutes per month
* DayCalls - average number of daytime calls
* MonthlyCharge - average monthly bill
* OverageFee - largest overage fee in last 12 months
* RoamMins - average number of roaming minutes
"""
"""
**Create functions**
"""
def countplot(data, title, title_x, annotate_x, palette):
    
    '''
    : param data: categorical data.
    : param title: chart title.
    : param title_x: title location by x.
    : param annotate_x: annotate location by x.
    : param palette: colors.
    : return bar chart   
    '''
    
    fig = plt.figure(figsize = (8, 4))

    ax = fig.add_axes([0, 0, 1, 1])

    sns.countplot(
        y = data,
        palette = palette,
        order = data.value_counts().index,
        edgecolor = 'black',
        lw = 2)

    sns.despine(bottom = True)

    plt.xticks([])
    plt.yticks(
        fontsize = 15,
        color = 'black',
        family = 'sans-serif')

    plt.xlabel('')
    plt.ylabel('')

    for p in ax.patches:
        width = p.get_width()
        height = p.get_height()
        x, y = p.get_xy() 
        ax.annotate('{:.0f}'.format(width), (annotate_x + width, y + height*0.5), ha='center', fontsize=15, color='black', family='sans-serif')

    plt.title(
        label = title,
        fontsize = 20,
        x = title_x,
        y = 1.1,
        family = 'sans-serif',
        fontweight='bold')    

    plt.show()
def stack_hist(categorial_data, data, palette, x_label, title, title_x):
    
    '''
    : param categorial_data: categorical data.
    : param data: numeric data.
    : param palette: colors.
    : param x_label: x_label name.
    : param title: chart title.
    : param title_x: title location by x.
    : return stacked histogram   
    '''
    
    plt.figure(figsize = (13, 6))

    sns.histplot(
        x=data,
        hue=categorial_data.astype('str'),
        palette=palette,
        multiple='stack',
        edgecolor='black')

    sns.despine()

    plt.xticks(
        fontsize=15,
        color='black',
        family='sans-serif')
    plt.yticks(
        fontsize=15,
        color='black',
        family='sans-serif')

    plt.xlabel(
        xlabel=x_label,
        fontsize=17,
        color='black',
        family='sans-serif',
        fontweight='bold')
    plt.ylabel(
        ylabel='Count',
        fontsize=17,
        color='black',
        family='sans-serif',
        fontweight='bold')

    plt.title(
        label=title,
        fontsize=20,
        x=title_x,
        y=1.1,
        family='sans-serif',
        fontweight='bold')

    plt.legend([]).set_visible(False)

    plt.show()
def stack_bar(data, column_1, column_2, palette, title, title_x):
    
    '''
    : param data: data set.
    : param column_1: first categorical data.
    : param column_2: second categorical data.
    : param palette: colors.
    : param title: chart title.
    : param title_x: title location by x.
    : return stacked bar chart   
    '''
    
    group = data \
    .groupby([column_1, column_2],as_index = False) \
    .agg({'index' : 'count'}) \
    .rename(columns={'index':'count'})

    fig = plt.figure(figsize=(8, 4))

    ax = fig.add_axes([0, 0, 1, 1])

    sns.histplot(
        x=group.iloc[:,0].astype('str'),
        hue=group.iloc[:,1],
        weights=group.iloc[:,2],
        multiple='stack',
        ax=ax,
        palette=palette,
        shrink=0.8,
        edgecolor='black',
        lw=2)

    sns.despine(left=True)

    plt.yticks([])
    plt.xticks(
        fontsize = 15,
        color = 'black',
        family = 'sans-serif')

    plt.xlabel('')
    plt.ylabel('')

    plt.legend([]).set_visible(False)

    for p in ax.patches:
        width, height = p.get_width(), p.get_height()
        x, y = p.get_xy() 
        ax.text(x+width/2, 
                y+height/2, 
                '{:.0f}'.format(height), 
                horizontalalignment='center', 
                verticalalignment='center',
                fontsize=11,
                color='white',
                fontweight='bold',
                family='sans-serif')

    plt.title(
        label=title,
        fontsize=20,
        x=title_x,
        y=1.1,
        family='sans-serif',
        fontweight='bold')

    plt.show()
def get_bootstrap(data_column_1, data_column_2, boot_it = 1000, statistic = np.mean, bootstrap_conf_level = 0.99):

    '''
    :param data_column_1: first group
    :param data_column_2: second group
    :param boot_it: number of bootstrap subsamples (by default - 1000)
    :param statistic: statistic (by default - mean)
    :param bootstrap_conf_level: significance level
    :return p-value.
    '''
    
    boot_len = max([len(data_column_1), len(data_column_2)])
    boot_data = []
    for i in (range(boot_it)): 
        samples_1 = data_column_1.sample(
            boot_len, 
            replace = True 
        ).values
        
        samples_2 = data_column_2.sample(
            boot_len, 
            replace = True
        ).values
        
        boot_data.append(statistic(samples_1-samples_2)) 
        
    pd_boot_data = pd.DataFrame(boot_data)
        
    left_quant = (1 - bootstrap_conf_level)/2  
    right_quant = 1 - (1 - bootstrap_conf_level) / 2  
    ci = pd_boot_data.quantile([left_quant, right_quant]) 
        
    p_1 = stats.norm.cdf(                 
        x = 0, 
        loc = np.mean(boot_data), 
        scale = np.std(boot_data)
    )
    p_2 = stats.norm.cdf(
        x = 0, 
        loc = -np.mean(boot_data), 
        scale = np.std(boot_data)
    )
    p_value = min(p_1, p_2) * 2       
    
    print(f'p-value - {p_value}')
def chi_2(alpha, data_1, data_2):
    '''
    :param alpha: significance level
    :param data_1: first categorical data
    :param alpha: second categorical data
    :return result (reject/no reject H0)
    '''
    print('Test chi2:')
    alpha = alpha
    p = stats.chi2_contingency(pd.crosstab(data_1, data_2))[1]
    if p < 0.05:
        print('Reject H0')
    else:
        print('No reject H0')
countplot(
    data=df['Churn'],
    title='Churn count',
    title_x=0,
    annotate_x=110,
    palette=['#4869D6', '#FF7373'])

print(f"Churn percentage - {round(len(df[df['Churn']==1]) / len(df)*100, 2)}%")
stack_hist(
    categorial_data=df['Churn'],
    data=df['AccountWeeks'],
    palette=['#4869D6', '#FF7373'],
    x_label='AccountWeeks',
    title='Duration by churn',
    title_x=0.1
    )
print(f'''AccountWeeks mean (No Churn) - {round(df[df['Churn']==0]['AccountWeeks'].mean())}
AccountWeeks mean (Churn) - {round(df[df['Churn']==1]['AccountWeeks'].mean())}''')
print()
print(f'''std (No Churn) - {round(df[df['Churn']==0]['AccountWeeks'].std())}
std (Churn) - {round(df[df['Churn']==1]['AccountWeeks'].std())}''')
print()
get_bootstrap(
    data_column_1=df[df['Churn']==0]['AccountWeeks'], 
    data_column_2=df[df['Churn']==1]['AccountWeeks'], 
    boot_it = 1000,
    statistic = np.mean, 
    bootstrap_conf_level = 0.99 
)
"""
The duration distribution is close to normal. The distribution of the No Churn users is more variable.

The average duration of the No Churn users is less than 2 weeks, but the differences are not statistically significant.
"""
stack_bar(
    data=df,
    column_1='ContractRenewal',
    column_2='Churn',
    palette=['#4869D6', '#FF7373'],
    title='Churn count by renewal contract',
    title_x=0.35)

chi_2(
    alpha=0.01,
    data_1=df['Churn'],
    data_2=df['ContractRenewal'])
"""
The Churn ratio among users who did not renew the contract does not differ much (~ 42%). However, the percentage of Churn users among those who extended the contract is significantly less than the share of remaining users.

The chi-square test showed that the variable has an effect on Churn.
"""
stack_bar(
    data=df,
    column_1='DataPlan',
    column_2='Churn',
    palette=['#4869D6', '#FF7373'],
    title='Churn count by DataPlan',
    title_x=0.2)

chi_2(
    alpha=0.01,
    data_1=df['Churn'],
    data_2=df['DataPlan'])
"""
The percentage of Churn users is higher among those who do not have an Internet connection.

The chi-square test showed that the variable has an effect on Churn.
"""
stack_hist(
    categorial_data=df['Churn'],
    data=df['DataUsage'],
    palette=['#4869D6', '#FF7373'],
    x_label='DataUsage',
    title='DataUsage by churn',
    title_x=0.1
    )
print(f'''Average datause (No Churn) - {round(df[df['Churn']==0]['DataUsage'].mean(), 2)}
Average datause (Churn) - {round(df[df['Churn']==1]['DataUsage'].mean(), 2)}''')
print()
get_bootstrap(
    data_column_1=df[df['Churn']==0]['DataUsage'], 
    data_column_2=df[df['Churn']==1]['DataUsage'], 
    boot_it = 1000,
    statistic = np.mean, 
    bootstrap_conf_level = 0.99 
)
"""
Most users do not exceed 1 GB.

On average, Churn users use less traffic, and these differences are statistically significant.
"""
stack_bar(
    data=df,
    column_1='CustServCalls',
    column_2='Churn',
    palette=['#4869D6', '#FF7373'],
    title='Churn count by CustServCalls',
    title_x=0.35)

chi_2(
    alpha=0.01,
    data_1=df['Churn'],
    data_2=df['CustServCalls'])
"""
Most users have made one request to technical support. Only a small part of users made more than 4 calls to technical support.
"""
stack_hist(
    categorial_data=df['Churn'],
    data=df['DayCalls'],
    palette=['#4869D6', '#FF7373'],
    x_label='DayCalls',
    title='DayCalls by churn',
    title_x=0.1
    )
print(f'''Average daycalls (No Churn) - {round(df[df['Churn']==0]['DayCalls'].mean())}
Average daycalls (Churn) - {round(df[df['Churn']==1]['DayCalls'].mean())}''')
print()
print(f'''std daycalls (No Churn) - {round(df[df['Churn']==0]['DayCalls'].std())}
std daycalls (Churn) - {round(df[df['Churn']==1]['DayCalls'].std())}''')
print()
get_bootstrap(
    data_column_1=df[df['Churn']==0]['DayCalls'], 
    data_column_2=df[df['Churn']==1]['DayCalls'], 
    boot_it = 1000,
    statistic = np.mean, 
    bootstrap_conf_level = 0.99 
)
"""
The distribution of the number of minutes per day is close to normal with small outliers. The distribution of Churn users is more variable.

On average, the Churn users make 1 more call per day, the differences are not statistically significant
"""
stack_hist(
    categorial_data=df['Churn'],
    data=df['MonthlyCharge'],
    palette=['#4869D6', '#FF7373'],
    x_label='MonthlyCharge',
    title='MonthlyCharge by churn',
    title_x=0.1
    )
print(f'''Average monthlycharge (No Churn) - {round(df[df['Churn']==0]['MonthlyCharge'].mean(), 2)}
Average monthlycharge (Churn) - {round(df[df['Churn']==1]['MonthlyCharge'].mean(), 2)}''')
print()
print(f'''std monthlycharge (No Churn) - {round(df[df['Churn']==0]['MonthlyCharge'].std())}
std monthlycharge (Churn) - {round(df[df['Churn']==1]['MonthlyCharge'].std())}''')
print()
get_bootstrap(
    data_column_1=df[df['Churn']==0]['MonthlyCharge'], 
    data_column_2=df[df['Churn']==1]['MonthlyCharge'], 
    boot_it = 1000,
    statistic = np.mean, 
    bootstrap_conf_level = 0.99 
)
"""
On average, Churn users pay more per month, and the differences are statistically significant.
"""
stack_hist(
    categorial_data=df['Churn'],
    data=df['OverageFee'],
    palette=['#4869D6', '#FF7373'],
    x_label='OverageFee',
    title='OverageFee by churn',
    title_x=0.1
    )
print(f'''Average overagefee (No Churn) - {round(df[df['Churn']==0]['OverageFee'].mean(), 2)}
Average overagefee (Churn) - {round(df[df['Churn']==1]['OverageFee'].mean(), 2)}''')
print()
print(f'''std overagefee (No Churn) - {round(df[df['Churn']==0]['OverageFee'].std(), 2)}
std overagefee (Churn) - {round(df[df['Churn']==1]['OverageFee'].std(), 2)}''')
print()
get_bootstrap(
    data_column_1=df[df['Churn']==0]['OverageFee'], 
    data_column_2=df[df['Churn']==1]['OverageFee'], 
    boot_it = 1000,
    statistic = np.mean, 
    bootstrap_conf_level = 0.99 
)
"""
The distribution is close to normal.

On average, the Churn users have the highest overpayment for 12 months more, and the differences are statistically significant
"""
stack_hist(
    categorial_data=df['Churn'],
    data=df['RoamMins'],
    palette=['#4869D6', '#FF7373'],
    x_label='RoamMins',
    title='RoamMins by churn',
    title_x=0.1
    )
print(f'''Average roammins (No Churn) - {round(df[df['Churn']==0]['RoamMins'].mean(), 2)}
Average roammins (Churn) - {round(df[df['Churn']==1]['RoamMins'].mean(), 2)}''')
print()
print(f'''std roammins (No Churn) - {round(df[df['Churn']==0]['RoamMins'].std(), 2)}
std roammins (Churn) - {round(df[df['Churn']==1]['RoamMins'].std(), 2)}''')
print()
get_bootstrap(
    data_column_1=df[df['Churn']==0]['RoamMins'], 
    data_column_2=df[df['Churn']==1]['RoamMins'], 
    boot_it = 1000,
    statistic = np.mean, 
    bootstrap_conf_level = 0.99 
)
"""
The distribution is close to normal.

On average, the Churn users used more roaming minutes, and the differences are statistically significant.
"""
"""
# Survival analysis.
"""
plt.figure(figsize=(13,8))

kmf = KaplanMeierFitter()
kmf.fit(
    durations=df['AccountWeeks'], 
    event_observed=df['Churn'])

kmf.plot()

plt.vlines(
    x=kmf.median_survival_time_,
    ymin=0,
    ymax=0.8,
    ls=':',
    color='crimson',
    lw=3)

plt.ylim(0,1)

plt.yticks(np.arange(0.1, 1.1, 0.1))

sns.despine()

plt.legend().set_visible(False)

plt.xticks(
    fontsize=15,
    color='black',
    family='sans-serif')
plt.yticks(
    fontsize=15,
    color='black',
    family='sans-serif')

plt.xlabel(
    xlabel='AccountWeeks',
    fontsize=17,
    color='black',
    family='sans-serif',
    fontweight='bold')
plt.ylabel(
    ylabel='Percentage',
    fontsize=17,
    color='black',
    family='sans-serif',
    fontweight='bold')

plt.title(
    label='Survival function',
    fontsize=20,
    x=0,
    y=1.1,
    family='sans-serif',
    fontweight='bold')

plt.show()

print(f'''50% of customers will refuse services before {int(kmf.median_survival_time_)} weeks''')
def surv_func(data, categorial_data, duration, event, title_x):

    '''
    :param data: data set
    :param categorial_data: column name with categorical data
    :param duration: column name with duration
    :param event: column name with event
    :param title_x: title location by x
    :return Survival function.
    '''
    
    median_surv_time_list = []
    labels = []

    plt.figure(figsize=(13,8))
    ax = plt.subplot()

    for i in data[categorial_data].unique():
        kmf = KaplanMeierFitter()
        kmf.fit(
            durations=data[data[categorial_data]== i][duration], 
            event_observed=data[data[categorial_data]== i][event],
            label=f'{i}')
        kmf.survival_function_.plot(ax=ax)
        median_surv_time = kmf.median_survival_time_
        median_surv_time_list.append(median_surv_time)
        labels.append(i)

    data_median = pd.DataFrame(index=labels, data=median_surv_time_list, columns=['median'])

    plt.xticks(
        fontsize=15,
        color='black',
        family='sans-serif')
    plt.yticks(
        fontsize=15,
        color='black',
        family='sans-serif')

    plt.xlabel(
        xlabel=duration,
        fontsize=17,
        color='black',
        family='sans-serif',
        fontweight='bold')
    plt.ylabel(
        ylabel='Percentage',
        fontsize=17,
        color='black',
        family='sans-serif',
        fontweight='bold')

    plt.title(
        label=f'Survival function by {categorial_data}',
        fontsize=20,
        x=title_x,
        y=1.1,
        family='sans-serif',
        fontweight='bold')

    sns.despine()

    plt.show()

    print(data_median)
surv_func(
    data=df,
    categorial_data='ContractRenewal',
    duration='AccountWeeks',
    event='Churn',
    title_x=0.1)
"""
We can also check whether the differences are statistically significant.
"""
results = logrank_test(
    durations_A=df[df['ContractRenewal']== 0]['AccountWeeks'],
    durations_B=df[df['ContractRenewal']== 1]['AccountWeeks'],
    event_observed_A=df[df['ContractRenewal']== 0]['Churn'],
    event_observed_B=df[df['ContractRenewal']== 1]['Churn'])
results.print_summary()
"""
The differences are statistically significant.

* 50% of customers who did not renew the contract will leave before 136 weeks;

* 50% of clients who have extended the contract will leave before 212 weeks.
"""
surv_func(
    data=df,
    categorial_data='DataPlan',
    duration='AccountWeeks',
    event='Churn',
    title_x=0.1)
results = logrank_test(
    durations_A=df[df['DataPlan']== 0]['AccountWeeks'],
    durations_B=df[df['DataPlan']== 1]['AccountWeeks'],
    event_observed_A=df[df['DataPlan']== 0]['Churn'],
    event_observed_B=df[df['DataPlan']== 1]['Churn'])
results.print_summary()
"""
From the data presented, it can be seen that among users who have the Internet connected, the survival rate has not fallen below ~ 65%.

50% of users who are not connected to the Internet leave before 193 weeks.

The differences are statistically significant.
"""
"""
In EDA, I found out that only a small proportion of users made more than 4 calls to technical support. Therefore, to analyze the survival rate and the possibility of interpreting the results, we will take into account only users who have made 4 or fewer requests.
"""
surv_func(
    data=df.query('CustServCalls <= 4'),
    categorial_data='CustServCalls',
    duration='AccountWeeks',
    event='Churn',
    title_x=0.1)
"""
The worst results among those who made 4 calls in support.
"""
"""
Since we have 2+ groups, we will apply a multivariate_logrank_test that tests the hypothesis that there is a statistically significant difference for at least one of the groups.
"""
result = multivariate_logrank_test(
    event_durations=df.query('CustServCalls <= 4')['AccountWeeks'],
    event_observed=df.query('CustServCalls <= 4')['Churn'],
    groups=df.query('CustServCalls <= 4')['CustServCalls'])
result.print_summary()
"""
The test showed us that at least one of the groups is statistically significantly different. To find out between which groups there is a statistically significant difference, I will use pairwise_logrank_test.
"""
result = pairwise_logrank_test(
    event_durations=df.query('CustServCalls <= 4')['AccountWeeks'],
    event_observed=df.query('CustServCalls <= 4')['Churn'],
    groups=df.query('CustServCalls <= 4')['CustServCalls'])
result.print_summary()
"""
Since the level we set is alpha = 0.01, there are statistically significant differences between the following groups:
* 0 and 4
* 1 and 4
* 2 and 4
* 3 and 4
"""
"""
# Cox
"""
"""
Using Cox regression, it is possible to determine the significance of predictors, how they affect and predict the probability of survival.
"""
df.drop('index', axis=1, inplace=True)
coxph = CoxPHFitter(alpha = 0.01)
coxph.fit(df, duration_col='AccountWeeks', event_col='Churn')
coxph.print_summary()
plt.figure(figsize = (13, 8))

coxph.plot()
"""
Next features have the significance:
* ContractRenewal
* CustServCalls
* RoamMins
"""
"""
For predict let's create data set with some No Churn users
"""
some_users = df[df['Churn']==1].reset_index().iloc[165:169,:]
round(coxph.predict_survival_function(some_users, conditional_after=some_users['AccountWeeks']), 4)
"""
From the data we can see what is the probability that these users will not stop using the service
"""