# Run this cell :)
1+2

"""
Explanation: Introduction to Statistics

Summarizing data. 
Plotting data.
Confidence intervals.
Statistical tests.

About this Notebook
In this notebook, we download a dataset with data about customers. Then, we calculate statistical measures and plot distributions. Finally, we perform statistical tests.
End of explanation
"""


# Uncomment next command if you need to install a missing module
#!pip install statsmodels
import matplotlib.pyplot as plt
import pandas as pd
try:
    import statsmodels.api as sm
except:
    !pip install statsmodels
import numpy as np
%matplotlib inline

"""
Explanation: Importing Needed packages
Statsmodels is a Python module that allows users to explore data, estimate statistical models, and perform statistical tests.
End of explanation
"""


import sys 
print(sys.version)

"""
Explanation: Print the current version of Python:
End of explanation
"""


import sys 
if sys.platform.startswith('linux'):
    !ls
elif sys.platform.startswith('freebsd'):
    !ls
elif sys.platform.startswith('darwin'):
    !ls
elif sys.platform.startswith('win'):
    !dir    

"""
Explanation: Downloading Data
Run system commands using ! (platform dependant)
End of explanation
"""


if sys.platform.startswith('linux'):
    !wget -O /resources/customer_dbase_sel.csv http://analytics.romanko.ca/data/customer_dbase_sel.csv

"""
Explanation: To download the data, we will use !wget (on DataScientistWorkbench)
End of explanation
"""


url = "http://analytics.romanko.ca/data/customer_dbase_sel.csv"
df = pd.read_csv(url)

## On DataScientistWorkbench you can read from /resources directory
#df = pd.read_csv("/resources/customer_dbase_sel.csv")

# display first 5 rows of the dataset
df.head()

"""
Explanation: Understanding the Data
customer_dbase_sel.csv:
We have downloaded an extract from IBM SPSS sample dataset with customer data, customer_dbase_sel.csv, which contains customer-specific data such as age, income, credit card spendings, commute type and time, etc. Dataset source

custid e.g. 0648-AIPJSP-UVM (customer id)
gender e.g. Female or Male
age e.g. 26
debtinc e.g. 11.1 (debt to income ratio in %)
card e.g. Visa, Mastercard (type of primary credit card)
carditems e.g. 1, 2, 3 ... (# of primary credit card purchases in the last month)
cardspent e.g 228.27 (amount in \$ spent on the primary credit card last month)
commute e.g. Walk, Car, Bus (commute type)
commutetime e.g. 22 (time in minutes to commute to work)
income e.g. 16.00 (income in thousands \$ per year)
edcat e.g. College degree, Post-undergraduate degree (education level)

Reading the data in
End of explanation
"""


# Summarize the data
df.describe()

# Number of rows and columns in the data
df.shape

# Display column names
df.columns

"""
Explanation: Data Exploration
End of explanation
"""


# To label data into high-income and low-income
df['income_category'] = df['annual_income'].map(lambda x: 1 if x>30000 else 0)
df[['annual_income','income_category']].head()

"""
Explanation: Labeling Data
income > 30000  --> High-income --> 1
income < 30000  --> Low-income  --> 0
End of explanation
"""


viz = df[['cardspent','debtinc','carditems','commutetime']]
viz.head()

"""
Explanation: Data Exploration
Select 4 data columns for visualizing:
End of explanation
"""


viz.describe()

"""
Explanation: Compute descriptive statistics for the data:
End of explanation
"""


df[['commutetime']].dropna().count()

"""
Explanation: Drop NaN (Not-a-Number) observations:
End of explanation
"""


print( df[np.isnan(df["commutetime"])] )

"""
Explanation: Print observations with NaN commutetime:
End of explanation
"""


viz.hist()
plt.show()

df[['cardspent']].hist()
plt.show()

df[['commutetime']].hist()
plt.show()

"""
Explanation: Visualize data:
End of explanation
"""


from scipy import stats

"""
Explanation: Confidence Intervals
For computing confidence intervals and performing simple statistical tests, we will use the stats sub-module of scipy:
End of explanation
"""


mu, sigma = np.mean(df[['debtinc']]), np.std(df[['debtinc']])
print ("mean = %G, st. dev = %g" % (mu, sigma))


N = len(df[['debtinc']])
N

"""
Explanation: Confidence intervals tell us how close we think the mean is to the true value, with a certain level of confidence.
We compute mean mu, standard deviation sigma and the number of observations N in our sample of the debt-to-income ratio:
End of explanation
"""


conf_int = stats.norm.interval( 0.95, loc = mu, scale = sigma/np.sqrt(N) )
conf_int

print ("95%% confidence interval for the mean of debt to income ratio = [%g %g]") % (conf_int[0], conf_int[1])

"""
Explanation: The 95% confidence interval for the mean of N draws from a Normal distribution with mean mu and standard deviation sigma is
End of explanation
"""


adf=df[['gender','cardspent','debtinc']]
print(adf['gender'])

"""
Explanation: Statistical Tests
Select columns by name:
End of explanation
"""


gender_data = adf.groupby('gender')
print (gender_data.mean())

"""
Explanation: Compute means for cardspent and debtinc for the male and female populations:
End of explanation
"""


adf[adf['gender'] == 'Female']['cardspent'].mean()

"""
Explanation: Compute mean for cardspent for female population only:
End of explanation
"""


female_card = adf[adf['gender'] == 'Female']['cardspent']
male_card = adf[adf['gender'] == 'Male']['cardspent']
tc, pc = stats.ttest_ind(female_card, male_card)
print ("t-test: t = %g  p = %g" % (tc, pc))

"""
Explanation: We have seen above that the mean cardspent and debtinc in the male and female populations were different. To test if this is significant, we do a 2-sample t-test with scipy.stats.ttest_ind():
End of explanation
"""


female_debt = adf[adf['gender'] == 'Female']['debtinc']
male_debt   = adf[adf['gender'] == 'Male']['debtinc']
td, pd      = stats.ttest_ind(female_debt, male_debt)
print ("t-test: t = %g  p = %g" % (td, pd))

"""
Explanation: In the case of amount spent on primary credit card, we conclude that men tend to charge more on their primary card (p-value = 2e-6 &lt; 0.05, statistically significant).
End of explanation
"""


adf.boxplot(column='cardspent', by='gender', grid=False, showfliers=False)
plt.show()

"""
Explanation: In the case of debt-to-income ratio, we conclude that there is no significant difference between men and women (p-value = 0.758 &gt; 0.05, not statistically significant).
Plot Data
Plot statistical measures for amounts spent on primary credit card
Use boxplot to compare medians, 25% and 75% percentiles, 12.5% and 87.5% percentiles:
End of explanation
"""


gend = list(['Female', 'Male'])
for i in [1,2]:
    y = adf.cardspent[adf.gender==gend[i-1]].dropna()        
    # Add some random "jitter" to the x-axis
    x = np.random.normal(i, 0.04, size=len(y))
    plt.plot(x, y, 'r.', alpha=0.2)
plt.boxplot([female_card,male_card],labels=gend)
plt.ylabel("cardspent")
plt.ylim((-50,850))    
plt.show()

"""
Explanation: Plot observations with boxplot:
End of explanation
"""


plt.scatter(df.age, df.annual_income)
plt.xlabel("Age")
plt.ylabel("Income")
plt.show()

"""
Explanation: Plot age vs. income data to find some interesting relationships.
End of explanation
"""


