# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/aapte11/DS-Sprint-01-Dealing-With-Data/blob/master/LS_DS_114_Making_Data_backed_Assertions.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="Okfr_uhwhS1X" colab_type="text"
# # Lambda School Data Science - Making Data-backed Assertions
#
# This is, for many, the main point of data science - to create and support reasoned arguments based on evidence. It's not a topic to master in a day, but it is worth some focused time thinking about and structuring your approach to it.

# + [markdown] id="9dtJETFRhnOG" colab_type="text"
# ## Lecture - generating a confounding variable
#
# The prewatch material told a story about a hypothetical health condition where both the drug usage and overall health outcome were related to gender - thus making gender a confounding variable, obfuscating the possible relationship between the drug and the outcome.
#
# Let's use Python to generate data that actually behaves in this fashion!

# + id="WiBkgmPJhmhE" colab_type="code" outputId="955ced12-8c35-4315-e396-6d9881d07957" colab={"base_uri": "https://localhost:8080/", "height": 1008}
import random
dir(random)  # Reminding ourselves what we can do here

# + id="Ks5qFtpnq-q5" colab_type="code" outputId="5c364669-a52b-475a-8012-fc6e69ca352b" colab={"base_uri": "https://localhost:8080/", "height": 33}
# Let's think of another scenario:
# We work for a company that sells accessories for mobile phones.
# They have an ecommerce site, and we are supposed to analyze logs
# to determine what sort of usage is related to purchases, and thus guide
# website development to encourage higher conversion.

# The hypothesis - users who spend longer on the site tend
# to spend more. Seems reasonable, no?

# But there's a confounding variable! If they're on a phone, they:
# a) Spend less time on the site, but
# b) Are more likely to be interested in the actual products!

# Let's use namedtuple to represent our data

from collections import namedtuple
# purchased and mobile are bools, time_on_site in seconds
User = namedtuple('User', ['purchased','time_on_site', 'mobile'])

example_user = User(False, 12, False)
print(example_user)

# + id="lfPiHNG_sefL" colab_type="code" outputId="20621927-65d3-4add-b1e3-82401e7d8df3" colab={"base_uri": "https://localhost:8080/", "height": 53}
# And now let's generate 1000 example users
# 750 mobile, 250 not (i.e. desktop)
# A desktop user has a base conversion likelihood of 10%
# And it goes up by 1% for each 15 seconds they spend on the site
# And they spend anywhere from 10 seconds to 10 minutes on the site (uniform)
# Mobile users spend on average half as much time on the site as desktop
# But have twice as much base likelihood of buying something

users = []

for _ in range(250):
  # Desktop users
  time_on_site = random.uniform(10, 600)
  purchased = random.random() < 0.1 + (time_on_site // 1500)
  users.append(User(purchased, time_on_site, False))
  
for _ in range(750):
  # Mobile users
  time_on_site = random.uniform(5, 300)
  purchased = random.random() < 0.2 + (time_on_site // 1500)
  users.append(User(purchased, time_on_site, True))
  
random.shuffle(users)
print(users[:10])

# + id="9gDYb5qGuRzy" colab_type="code" outputId="a37526ba-7e4e-4323-b852-65166e2bd085" colab={"base_uri": "https://localhost:8080/", "height": 191}
# Let's put this in a dataframe so we can look at it more easily
import pandas as pd
user_data = pd.DataFrame(users)
user_data.head()

# + id="sr6IJv77ulVl" colab_type="code" outputId="9cd4e153-159b-4931-f4f3-907ecd3c592d" colab={"base_uri": "https://localhost:8080/", "height": 181}
# Let's use crosstabulation to try to see what's going on
pd.crosstab(user_data['purchased'], user_data['time_on_site'])

# + id="hvAv6J3EwA9s" colab_type="code" outputId="04b17307-0098-457d-84e0-b71e99c69ff0" colab={"base_uri": "https://localhost:8080/", "height": 133}
# OK, that's not quite what we want
# Time is continuous! We need to put it in discrete buckets
# Pandas calls these bins, and pandas.cut helps make them

time_bins = pd.cut(user_data['time_on_site'], 5)  # 5 equal-sized bins
pd.crosstab(user_data['purchased'], time_bins)

# + id="pjcXnJw0wfaj" colab_type="code" outputId="ad0fb6a4-9b42-43a7-f615-5752d1bfb3e4" colab={"base_uri": "https://localhost:8080/", "height": 133}
# We can make this a bit clearer by normalizing (getting %)
pd.crosstab(user_data['purchased'], time_bins, normalize='all')

# + id="C3GzvDxlvZMa" colab_type="code" outputId="40853a66-c2f3-44f1-f303-7592a9f1938c" colab={"base_uri": "https://localhost:8080/", "height": 133}
# That seems counter to our hypothesis
# More time on the site seems to have fewer purchases

# But we know why, since we generated the data!
# Let's look at mobile and purchased
pd.crosstab(user_data['purchased'], user_data['mobile'], normalize='all')

# + id="KQb-wU60xCum" colab_type="code" colab={}
# Yep, mobile users are more likely to buy things
# But we're still not seeing the *whole* story until we look at all 3 at once

# Live/stretch goal - how can we do that?

# + id="RPiiRU9Zbdts" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 452} outputId="04559abd-c4ba-42c2-ddf0-7b55be0a2101"
my_colors = ['k','gray','silver','w']

pd.crosstab(time_bins, [user_data['purchased'], user_data['mobile']], normalize='index').plot(kind = 'bar', stacked = True, color = my_colors)

# Normalizing by index makes interpretation of the y axis make sense. 

# + id="4SY9OOolU-Jx" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 363} outputId="26f39a02-41fd-4f2c-e9a7-ce37f43b6cde"
pd.crosstab(time_bins, [user_data['purchased'], user_data['mobile']], normalize='all').plot(kind = 'barh', stacked = False, color = my_colors)

# + [markdown] id="lOqaPds9huME" colab_type="text"
# ## Assignment - what's going on here?
#
# Consider the data in `persons.csv` (already prepared for you, in the repo for the week). It has four columns - a unique id, followed by age (in years), weight (in lbs), and exercise time (in minutes/week) of 1200 (hypothetical) people.
#
# Try to figure out which variables are possibly related to each other, and which may be confounding relationships.

# + id="TGUS79cOhPWj" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 191} outputId="cadbbe13-c4d4-4a8a-df61-560b6dda8cdc"
# TODO - your code here
# Use what we did live in lecture as an example

# HINT - you can find the raw URL on GitHub and potentially use that
# to load the data with read_csv, or you can upload it yourself

# path = 'DS-Sprint-01-Dealing-With-Data/module4-databackedassertions/persons.csv' XXX
# path_alt = 'https://github.com/aapte11/DS-Sprint-01-Dealing-With-Data/blob/master/module4-databackedassertions/persons.csv' XXX

path_alt2 = 'https://raw.githubusercontent.com/aapte11/DS-Sprint-01-Dealing-With-Data/master/module4-databackedassertions/persons.csv'

# Always upload from raw file
      
df_persons = pd.read_csv(path_alt2, index_col=0)

df_persons.head()


# + id="HwCt_DSgil9i" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 1065} outputId="8b2255bb-4b4d-4d76-83dc-fc34be489971"
df_persons.plot.scatter('age', 'weight')
df_persons.plot.scatter('age', 'exercise_time', color = 'r')
df_persons.plot.scatter('weight','exercise_time', color = 'g')

# + id="yKs9F0yKjI0T" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 133} outputId="34a9b51c-d5cb-4b62-c4f8-251d4cdd2d22"
df_persons.corr()

# + id="PvEMq2ohD9Ko" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 472} outputId="e02381d6-c79f-43c0-c4a0-5d32bfb88635"
print(df_persons.age.unique())
print (df_persons.weight.unique())
print (df_persons.exercise_time.unique())



# + id="F4PqSR6DGvt5" colab_type="code" colab={}
age_bins = pd.cut(df_persons.age, 5)
weight_bins = pd.cut(df_persons.weight, 5)
exer_bins = pd.cut(df_persons.exercise_time, 5)

import seaborn as sns

df1 = pd.crosstab(age_bins, weight_bins, normalize = 'all')
df2 = pd.crosstab(age_bins, exer_bins, normalize='all')
df3 = pd.crosstab(exer_bins, weight_bins, normalize='all')




# + id="5nIqT76bQ3an" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 376} outputId="16a57123-aaa8-429f-aa73-fd9f6e41307b"
sns.heatmap(df1, annot=True).set_title('Age Bins vs Weight Bins');

# + id="J54_0FM4PyAi" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 376} outputId="37a94213-75a9-4021-b827-dc04c6673649"
sns.heatmap(df2, annot=True).set_title('Age Bins vs Exercise Time Bins');

# + id="a0USpN1VPyQ3" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 376} outputId="9864d0f5-5c92-404e-e143-009ae284b774"
sns.heatmap(df3, annot=True).set_title('Exercise Time Bins vs Weight Bins');

# + [markdown] id="BT9gdS7viJZa" colab_type="text"
# ### Assignment questions
#
# After you've worked on some code, answer the following questions in this text block:
#
# 1.  What are the variable types in the data?
#
# **Age, weight and exercise time are all continuous variables even though in this dataset they seem to be put into bins.**
#
#
# 2.  What are the relationships between the variables?
#
# **Age and weight have aa weak positive relationship. Exercise time and age have a negative relationship and exercise time and weight have a strongly negative relationship. **
#
# 3.  Which relationships are "real", and which spurious?
#
# **From a conceptual standpoint, it seems as if weight and/or exercise time is a function of age as age is a given variable (can't change age) while the other two are variable. However when looking at the correlation matrix, one sees that the strongest relationship is between exercise time and weight. However, one feels it is necessary to bin the data and to put it into heatmaps to see where the relaationships are strongest as the original scatterplots suggest varying degrees of strength throughout the plot. **
#

# + [markdown] id="_XXg2crAipwP" colab_type="text"
# ## Stretch goals and resources
#
# Following are *optional* things for you to take a look at. Focus on the above assignment first, and make sure to commit and push your changes to GitHub.
#
# - [Spurious Correlations](http://tylervigen.com/spurious-correlations)
# - [NIH on controlling for confounding variables](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4017459/)
#
# Stretch goals:
#
# - Produce your own plot inspierd by the Spurious Correlation visualizations (and consider writing a blog post about it - both the content and how you made it)
# - Pick one of the techniques that NIH highlights for confounding variables - we'll be going into many of them later, but see if you can find which Python modules may help (hint - check scikit-learn)

# + [markdown] id="QitRXbfMhDPR" colab_type="text"
# **Monthly Milk Production vs S&P 500 1962-1975**

# + id="bFOlK3Yk5lb_" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 326} outputId="3ac793b4-fcf6-46bf-e152-0a2bd633e7be"
urlsnp = 'https://raw.githubusercontent.com/aapte11/DS-Sprint-01-Dealing-With-Data/master/S%26P500MonthlyReturns.csv'
urlmilk = 'https://raw.githubusercontent.com/aapte11/DS-Sprint-01-Dealing-With-Data/master/monthly-milk-production-pounds-p.csv'

df_snp = pd.read_csv(urlsnp)
df_milk = pd.read_csv(urlmilk)

print(df_snp.head())

print (df_milk.head())

# + id="6UJF3nAGbjLH" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 50} outputId="02830cac-cea9-4aeb-d637-2448f91f85f6"
df_snp_new = df_snp[['Date', 'Close']]
df_milk_new = df_milk[:-1]

print(len(df_snp_new))
print(len(df_milk_new))

# + id="At-R7DT8dxXs" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 98} outputId="cac16272-60c9-4dfd-9cc1-06dda9670816"
df_final = df_snp_new.join(df_milk_new)

df_final.count()

# + id="3L_rKzkFe_x6" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 191} outputId="938cd95a-6722-49ea-e602-afeeb381fa45"
del df_final['Date']
df_final['SNPClose'] = df_final['Close']
df_final['MMP'] = df_final['Monthly milk production: pounds per cow. Jan 62 ? Dec 75']
del df_final['Close']
del df_final['Monthly milk production: pounds per cow. Jan 62 ? Dec 75']

df_final.head()

# + id="JDKd-6imf-X_" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 105} outputId="9aab2da4-21f0-45da-91eb-a67a0cd61ec2"
df_final.corr()

# + id="5kZa9OkFfS8e" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 393} outputId="0f84b361-ed7f-412e-9b12-3c55469af748"
import matplotlib.pyplot as plt

ax = df_final.plot(x="Month", y="SNPClose", legend=False, color='blue')
ax2 = ax.twinx()
df_final.plot(x="Month", y="MMP", ax=ax2, legend=False, color="lime")
ax.set_ylabel('S&P 500 Closing Price')
ax2.set_ylabel('Monthly Milk Production')
ax.figure.legend()
plt.title("Spurious Correlation - S&P500 vs Milk Production")
plt.show()



# + id="c5-gyR8IkDgG" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered