import pandas as pd
import numpy as np
%pylab inline
pylab.style.use('ggplot')
import seaborn as sns

gender_data = pd.DataFrame(data=[[60, 54, 46, 41], 
                                 [40, 44, 53, 57]],
                          index=['female', 'male'],
                          columns=['High School', 'Bachelors', 'Masters', 'Ph.d.'])

gender_data

"""
Explanation: Chi-Square Test of Independence
The Chi-square test of independence tests if two categorical variables are independent.
Recall that we can summarize two categorical variables within a two-way table, also called a r × c contingency table, where r = number of rows, c = number of columns. Our question of interest is "Are the two variables independent?" This question is set up using the following hypothesis statements:
Null Hypothesis: The two categorical variables are independent.
Alternative Hypothesis: The two categorical variables are dependent.
The chi-square test statistic is calculated by using the formula:
$$\chi^2 = \sum \dfrac {(O-E)^2} {E}$$
where $O$ represents the observed frequency. $E$ is the expected frequency under the null hypothesis and computed by:
$$ E = \dfrac {row \space total \times column \space total} {sample \space size}$$
We will calculate the P-value of the chi-square test statistic from a $\chi^2$ distribution with degree of freedom $(r - 1) * (c - 1)$. We will fail to reject the null hypothesis if the P-value is greater than the specified significance level.
Example
Is gender independent of education level? A random sample of 395 people were surveyed and each person was asked to report the highest education level they obtained. The data that resulted from the survey is summarized in the following table:
End of explanation
"""


gender_long = pd.melt(gender_data.reset_index(), 
                      id_vars='index', 
                      var_name='education', 
                      value_name='n_samples').rename(columns={'index': 'gender'})

gender_long

g = sns.factorplot(kind='bar', x='education', y='n_samples', hue="gender", data=gender_long)

"""
Explanation: Visualizing the Data
End of explanation
"""


row_totals = gender_data.sum(axis=1)
row_totals

column_totals = gender_data.sum(axis=0)
column_totals.to_frame().T

"""
Explanation: Calculate Totals
End of explanation
"""


expected_frequencies = pd.concat(
    {c: row_totals for c in gender_data.columns}, 
    axis=1).mul(column_totals).div(gender_data.sum(axis=1).sum())

expected_frequencies

"""
Explanation: Calculate Expected Frequencies
End of explanation
"""


t_stat = gender_data.sub(
    expected_frequencies).pow(2.0).div(
    expected_frequencies).sum(axis=1).sum()

t_stat

df = (gender_data.shape[0]-1) * (gender_data.shape[1]-1)
df

from scipy.stats import chi2
chi2_dist = chi2(df=df)
p_val = chi2_dist.sf(t_stat)

sig_level = 0.05
print('P-val of chi^2 test at significance level {:.2f} is {:.4f}'.format(sig_level, p_val))

"""
Explanation: Calculate Chi-Square test statistic
End of explanation
"""


