# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Lesson 1 - Summarizing Data

# +
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import statistics

# %matplotlib inline
# -

# ### Make a blank DataFrame

df = pd.DataFrame()

# ### Populate it with data

df['age'] = [28, 42, 27, 24, 35, 54, 35, 42, 37]

# ## Measures of Central Tendency

# ### Mean (using built-in Python functionality)

mean_py = sum(df['age']) / len(df['age'])

mean_py

# ### Mean (using NumPy)

mean_np = np.mean(df['age'])

mean_np

# ### Median (using built-in Python functionality)

median_py = statistics.median(df['age'])

median_py

# ### Median (using NumPy)

median_np = np.median(df['age'])

median_np

# ### Mode (using built-in Python functionality)

mode_py = statistics.mode(df['age'])

mode_py

# ### Mode (using NumPy)

# Generate a list of unique elements along with how often they occur.
(values, counts) = np.unique(df['age'], return_counts=True)

print(values, counts)

# The location(s) in the values list of the most-frequently-occurring element(s).
ind = [x[0] for x in list(enumerate(counts)) if x[1] == counts[np.argmax(counts)]]

ind

values

# The most frequent element(s).
modes = [values[x] for x in ind]

modes

# ## Measures of Variance

# ### Variance (using NumPy)

df['age']

# change delta degrees of freedom (ddof) to 1 from its default value of 0
var_np = np.var(df['age'], ddof=1)

var_np

# ### Variance (using Pandas)

var_pd = df['age'].var()

var_pd

# ### Standard Deviation (using NumPy)

std_np = np.std(df['age'], ddof=1)

std_np

# ### Standard Deviation (using Pandas)

std_pd = df['age'].std()

std_pd

# ### Standard Error (using NumPy)

se_np = std_np / np.sqrt(len(df['age']))

se_np

# ### Standard Error Examples

# +
# First, create an empty dataframe to store your variables-to-be.
pop = pd.DataFrame()

# Then create two variables with mean = 60, one with a low standard
# deviation (sd=10) and one with a high standard deviation (sd=100).
pop['low_var'] = np.random.normal(60, 10, 10000)
pop['high_var'] = np.random.normal(60, 100, 10000)

# Finally, create histograms of the two variables.
pop.hist(layout=(2, 1), sharex=True)
plt.show()

# Calculate and print the maximum and minimum values for each variable.
print("\nMax of low_var and high_var:\n", pop.max())
print("\nMin of low_var and high_var:\n", pop.min())

# +
# Take a random sample of 100 observations from each variable
# and store it in a new dataframe.
sample = pd.DataFrame()
sample['low_var'] = np.random.choice(pop['low_var'], 100)
sample['high_var'] = np.random.choice(pop['high_var'], 100)

# Again, visualize the data. Note that here we're using a pandas method to 
# create the histogram.
sample.hist()
plt.show()

# Check how well the sample replicates the population.
print("Mean of low_var and high_var:\n", sample.mean())
# -

print("Standard deviation of low_var and high_var:\n", sample.std(ddof=1))

# ## Describing Data with Pandas

# +
# Set up the data
data = pd.DataFrame()
data['gender'] = ['male'] * 100 + ['female'] * 100

# 100 height values for males, 100 height values for females
data['height'] = np.append(np.random.normal(69, 8, 100), np.random.normal(64, 5, 100))

# 100 weight values for males, 100 weight values for females
data['weight'] = np.append(np.random.normal(195, 25, 100), np.random.normal(166, 15, 100))
# -

data.head(10)

data.tail(10)

data['height'].mean()

data['height'].std()

data.describe()

data.groupby('gender').describe()

data['gender'].value_counts()

# # Lesson 2 - Basics of Probability

# ## Perspectives on Probability

# ### _Frequentist_ school of thought
#    - ### Describes how often a particular outcome would occur in an experiment if that experiment were repeated over and over
#    - ### In general, frequentists consider _model parameters to be fixed_ and _data to be random_

# ### _Bayesian_ school of thought
#    - ### Describes how likely an observer expects a particular outcome to be in the future, based on previous experience and expert knowledge
#    - ### Each time an experiment is run, the probability is updated if the new data changes the belief about the likelihood of the outcome
#    - ### The probability based on previous experiences is called the _"prior probability,"_ or the "prior," while the updated probability based on the newest experiment is called the _"posterior probability."_
#    - ### In general, Bayesians consider _model parameters to be random_ and _data to be fixed_
# -------------------------------------------------------------------------------------------------------------------------

# ## Randomness
# ## Sampling
# ## Selection Bias
# -------------------------------------------------------------------------------------------------------------------------

# ## Independence
# ## Dependence
# -------------------------------------------------------------------------------------------------------------------------

# ## Bayes' Rule
#    ## $P(A|B)=\frac{P(B|A)*P(A)}{P(B)}=\frac{P(B|A)*P(A)}{[P(A)*P(B|A)+P(A\sim)*P(B|A\sim)]}$
#
#
# ## Conditional Probability
# -------------------------------------------------------------------------------------------------------------------------

# ## Evaluating Data Sources
#    - ## Bias
#    - ## Quality
#    - ## Exceptional Circumstance
# -------------------------------------------------------------------------------------------------------------------------

# # The Normal Distribution and the Central Limit Theorem

# ## Normality

# ## Deviations from Normality and Descriptive Statistics (skewness)

# ## Other Distributions
#    - ## Bernoulli
#    - ## Binomial
#    - ## Gamma
#    - ## Poisson
#    - ## Conditional Distribution

# ## CLT and Sampling



StarcoderdataJupyterScriptsDedupFiltered