from __future__ import print_function, division

import thinkstats2
import thinkplot

import pandas as pd
import numpy as np

import statsmodels.formula.api as smf

%matplotlib inline

"""
Explanation: Does Trivers-Willard apply to people?
This notebook contains a "one-day paper", my attempt to pose a research question, answer it, and publish the results in one work day.
Copyright 2016 Allen B. Downey
MIT License: https://opensource.org/licenses/MIT
End of explanation
"""


names = ['year', 'mager9', 'restatus', 'mbrace', 'mhisp_r',
        'mar_p', 'dmar', 'meduc', 'fagerrec11', 'fbrace', 'fhisp_r', 'feduc', 
        'lbo_rec', 'previs_rec', 'wic', 'height', 'bmi_r', 'pay_rec', 'sex']
colspecs = [(15, 18),
            (93, 93),
            (138, 138),
            (143, 143),
            (148, 148),
            (152, 152),
            (153, 153),
            (155, 155),
            (186, 187),
            (191, 191),
            (195, 195),
            (197, 197),
            (212, 212),
            (272, 273),
            (281, 281),
            (555, 556),
            (533, 533),
            (413, 413),
            (436, 436),
           ]

colspecs = [(start-1, end) for start, end in colspecs]

df = None

filename = 'Nat2013PublicUS.r20141016.gz'
#df = pd.read_fwf(filename, compression='gzip', header=None, names=names, colspecs=colspecs)
#df.head()

# store the dataframe for faster loading

#store = pd.HDFStore('store.h5')
#store['births2013'] = df
#store.close()

# load the dataframe

store = pd.HDFStore('store.h5')
df = store['births2013']
store.close()

def series_to_ratio(series):
    """Takes a boolean series and computes sex ratio.
    """
    boys = np.mean(series)
    return np.round(100 * boys / (1-boys)).astype(int)

"""
Explanation: Trivers-Willard
According to Wikipedia, the Trivers-Willard hypothesis:

"...suggests that female mammals are able to adjust offspring sex ratio in response to their maternal condition. For example, it may predict greater parental investment in males by parents in 'good conditions' and greater investment in females by parents in 'poor conditions' (relative to parents in good condition)."

For humans, the hypothesis suggests that people with relatively high social status might be more likely to have boys.  Some studies have shown evidence for this hypothesis, but based on my very casual survey, it is not persuasive.
To test whether the T-W hypothesis holds up in humans, I downloaded birth data for the nearly 4 million babies born in the U.S. in 2014.
I selected variables that seemed likely to be related to social status and used logistic regression to identify variables associated with sex ratio.
Summary of results


Running regression with one variable at a time, many of the variables have a statistically significant effect on sex ratio, with the sign of the effect generally in the direction predicted by T-W.


However, many of the variables are also correlated with race.  If we control for either the mother's race or the father's race, or both, most other variables have no additional predictive power.


Contrary to other reports, the age of the parents seems to have no predictive power.


Strangely, the variable that shows the strongest and most consistent relationship with sex ratio is the number of prenatal visits.  Although it seems obvious that prenatal visits are a proxy for quality of health care and general socioeconomic status, the sign of the effect is opposite what T-W predicts; that is, more prenatal visits is a strong predictor of lower sex ratio (more girls).


Following convention, I report sex ratio in terms of boys per 100 girls.  The overall sex ratio at birth is about 105; that is, 105 boys are born for every 100 girls.
Data cleaning
Here's how I loaded the data:
End of explanation
"""


df['boy'] = (df.sex=='M').astype(int)
df.boy.value_counts().sort_index()

"""
Explanation: I have to recode sex as 0 or 1 to make logit happy.
End of explanation
"""


df.year.value_counts().sort_index()

"""
Explanation: All births are from 2014.
End of explanation
"""


df.mager9.value_counts().sort_index()

var = 'mager9'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

df.mager9.isnull().mean()

df['youngm'] = df.mager9<=2
df['oldm'] = df.mager9>=7
df.youngm.mean(), df.oldm.mean()

"""
Explanation: Mother's age:
End of explanation
"""


df.restatus.value_counts().sort_index()

var = 'restatus'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

"""
Explanation: Residence status (1=resident)
End of explanation
"""


df.mbrace.value_counts().sort_index()

var = 'mbrace'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

"""
Explanation: Mother's race (1=White, 2=Black, 3=American Indian or Alaskan Native, 4=Asian or Pacific Islander)
End of explanation
"""


df.mhisp_r.replace([9], np.nan, inplace=True)
df.mhisp_r.value_counts().sort_index()

def copy_null(df, oldvar, newvar):
    df.loc[df[oldvar].isnull(), newvar] = np.nan

df['mhisp'] = df.mhisp_r > 0
copy_null(df, 'mhisp_r', 'mhisp')
df.mhisp.isnull().mean(), df.mhisp.mean()

var = 'mhisp'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

"""
Explanation: Mother's Hispanic origin (0=Non-Hispanic)
End of explanation
"""


df.dmar.value_counts().sort_index()

var = 'dmar'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

"""
Explanation: Marital status (1=Married)
End of explanation
"""


df.mar_p.replace(['U'], np.nan, inplace=True)
df.mar_p.replace(['X'], 'Y', inplace=True)
df.mar_p.value_counts().sort_index()

var = 'mar_p'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

"""
Explanation: Paternity acknowledged, if unmarried (Y=yes, N=no, X=not applicable, U=unknown).
I recode X (not applicable because married) as Y (paternity acknowledged).
End of explanation
"""


df.meduc.replace([9], np.nan, inplace=True)
df.meduc.value_counts().sort_index()

var = 'meduc'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

df['lowed'] = df.meduc <= 2
copy_null(df, 'meduc', 'lowed')
df.lowed.isnull().mean(), df.lowed.mean()

"""
Explanation: Mother's education level
End of explanation
"""


df.fagerrec11.replace([11], np.nan, inplace=True)
df.fagerrec11.value_counts().sort_index()

var = 'fagerrec11'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

df['youngf'] = df.fagerrec11<=2
copy_null(df, 'fagerrec11', 'youngf')
df.youngf.isnull().mean(), df.youngf.mean()

df['oldf'] = df.fagerrec11>=8
copy_null(df, 'fagerrec11', 'oldf')
df.oldf.isnull().mean(), df.oldf.mean()

"""
Explanation: Father's age, in 10 ranges
End of explanation
"""


df.fbrace.replace([9], np.nan, inplace=True)
df.fbrace.value_counts().sort_index()

var = 'fbrace'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

"""
Explanation: Father's race
End of explanation
"""


df.fhisp_r.replace([9], np.nan, inplace=True)
df.fhisp_r.value_counts().sort_index()

df['fhisp'] = df.fhisp_r > 0
copy_null(df, 'fhisp_r', 'fhisp')
df.fhisp.isnull().mean(), df.fhisp.mean()

var = 'fhisp'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

"""
Explanation: Father's Hispanic origin (0=non-hispanic, other values indicate country of origin)
End of explanation
"""


df.feduc.replace([9], np.nan, inplace=True)
df.feduc.value_counts().sort_index()

var = 'feduc'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

"""
Explanation: Father's education level
End of explanation
"""


df.lbo_rec.replace([9], np.nan, inplace=True)
df.lbo_rec.value_counts().sort_index()

var = 'lbo_rec'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

df['highbo'] = df.lbo_rec >= 5
copy_null(df, 'lbo_rec', 'highbo')
df.highbo.isnull().mean(), df.highbo.mean()

"""
Explanation: Live birth order.
End of explanation
"""


df.previs_rec.replace([12], np.nan, inplace=True)
df.previs_rec.value_counts().sort_index()

df.previs_rec.mean()
df['previs'] = df.previs_rec - 7

var = 'previs'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

df['no_previs'] = df.previs_rec <= 1
copy_null(df, 'previs_rec', 'no_previs')
df.no_previs.isnull().mean(), df.no_previs.mean()

"""
Explanation: Number of prenatal visits, in 11 ranges
End of explanation
"""


df.wic.replace(['U'], np.nan, inplace=True)
df.wic.value_counts().sort_index()

var = 'wic'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

"""
Explanation: Whether the mother is eligible for food stamps
End of explanation
"""


df.height.replace([99], np.nan, inplace=True)
df.height.value_counts().sort_index()

df['mshort'] = df.height<60
copy_null(df, 'height', 'mshort')
df.mshort.isnull().mean(), df.mshort.mean()

df['mtall'] = df.height>=70
copy_null(df, 'height', 'mtall')
df.mtall.isnull().mean(), df.mtall.mean()

var = 'mshort'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

var = 'mtall'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

"""
Explanation: Mother's height in inches
End of explanation
"""


df.bmi_r.replace([9], np.nan, inplace=True)
df.bmi_r.value_counts().sort_index()

var = 'bmi_r'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

df['obese'] = df.bmi_r >= 4
copy_null(df, 'bmi_r', 'obese')
df.obese.isnull().mean(), df.obese.mean()

"""
Explanation: Mother's BMI in 6 ranges
End of explanation
"""


df.pay_rec.replace([9], np.nan, inplace=True)
df.pay_rec.value_counts().sort_index()

var = 'pay_rec'
df[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

"""
Explanation: Payment method (1=Medicaid, 2=Private insurance, 3=Self pay, 4=Other)
End of explanation
"""


df.sex.value_counts().sort_index()

"""
Explanation: Sex of baby
End of explanation
"""


def logodds_to_ratio(logodds):
    """Convert from log odds to probability."""
    odds = np.exp(logodds)
    return 100 * odds

def summarize(results):
    """Summarize parameters in terms of birth ratio."""
    inter_or = results.params['Intercept']
    inter_rat = logodds_to_ratio(inter_or)
    
    for value, lor in results.params.iteritems():
        if value=='Intercept':
            continue
        
        rat = logodds_to_ratio(inter_or + lor)
        code = '*' if results.pvalues[value] < 0.05 else ' '
        
        print('%-20s   %0.1f   %0.1f' % (value, inter_rat, rat), code)

"""
Explanation: Regression models
Here are some functions I'll use to interpret the results of logistic regression
End of explanation
"""


model = smf.logit('boy ~ mager9', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Now I'll run models with each variable, one at a time.
Mother's age seems to have no predictive value:
End of explanation
"""


model = smf.logit('boy ~ youngm + oldm', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: The estimated ratios for young mothers is higher, and the ratio for older mothers is lower, but neither is statistically significant.
End of explanation
"""


model = smf.logit('boy ~ C(restatus)', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Neither does residence status
End of explanation
"""


model = smf.logit('boy ~ C(mbrace)', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Mother's race seems to have predictive value.  Relative to whites, black and Native American mothers have more girls; Asians have more boys.
End of explanation
"""


model = smf.logit('boy ~ mhisp', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Hispanic mothers have more girls.
End of explanation
"""


model = smf.logit('boy ~ C(mar_p)', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: If the mother is married or unmarried but paternity is acknowledged, the sex ratio is higher (more boys)
End of explanation
"""


model = smf.logit('boy ~ C(dmar)', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Being unmarried predicts more girls.
End of explanation
"""


model = smf.logit('boy ~ meduc', data=df)    
results = model.fit()
summarize(results)
results.summary()

model = smf.logit('boy ~ lowed', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Each level of mother's education predicts a small increase in the probability of a boy.
End of explanation
"""


model = smf.logit('boy ~ fagerrec11', data=df)    
results = model.fit()
summarize(results)
results.summary()

model = smf.logit('boy ~ youngf + oldf', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Older fathers are slightly more likely to have girls (but this apparent effect could be due to chance).
End of explanation
"""


model = smf.logit('boy ~ C(fbrace)', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Predictions based on father's race are similar to those based on mother's race: more girls for black and Native American fathers; more boys for Asian fathers.
End of explanation
"""


model = smf.logit('boy ~ fhisp', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: If the father is Hispanic, that predicts more girls.
End of explanation
"""


model = smf.logit('boy ~ feduc', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Father's education level might predict more boys, but the apparent effect could be due to chance.
End of explanation
"""


model = smf.logit('boy ~ lbo_rec', data=df)    
results = model.fit()
summarize(results)
results.summary()

model = smf.logit('boy ~ highbo', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Babies with high birth order are slightly more likely to be girls.
End of explanation
"""


model = smf.logit('boy ~ previs', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Strangely, prenatal visits are associated with an increased probability of girls.
End of explanation
"""


model = smf.logit('boy ~ no_previs + previs', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: The effect seems to be non-linear at zero, so I'm adding a boolean for no prenatal visits.
End of explanation
"""


model = smf.logit('boy ~ wic', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: If the mother qualifies for food stamps, she is more likely to have a girl.
End of explanation
"""


model = smf.logit('boy ~ height', data=df)    
results = model.fit()
summarize(results)
results.summary()

model = smf.logit('boy ~ mtall + mshort', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Mother's height seems to have no predictive value.
End of explanation
"""


model = smf.logit('boy ~ bmi_r', data=df)    
results = model.fit()
summarize(results)
results.summary()

model = smf.logit('boy ~ obese', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Mother's with higher BMI are more likely to have girls.
End of explanation
"""


model = smf.logit('boy ~ C(pay_rec)', data=df)    
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: If payment was made by Medicaid, the baby is more likely to be a girl.  Private insurance, self-payment, and other payment method are associated with more boys.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + C(mbrace) + mhisp')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Adding controls
However, none of the previous results should be taken too seriously.  We only tested one variable at a time, and many of these apparent effects disappear when we add control variables.
In particular, if we control for father's race and Hispanic origin, the mother's race has no additional predictive value.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + mar_p')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: In fact, once we control for father's race and Hispanic origin, almost every other variable becomes statistically insignificant, including acknowledged paternity.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + dmar')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Being married still predicts more boys.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + lowed')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: The effect of education disappears.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + highbo')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: The effect of birth order disappears.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + wic')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: WIC is no longer associated with more girls.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + obese')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: The effect of obesity disappears.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + C(pay_rec)')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: The effect of payment method is diminished, but self-payment is still associated with more boys.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + previs')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: But the effect of prenatal visits is still a strong predictor of more girls.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + previs + no_previs')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: And the effect is even stronger if we add a boolean to capture the nonlinearity at 0 visits.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + previs + dmar')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: More controls
Now if we control for father's race and Hispanic origin as well as number of prenatal visits, the effect of marriage disappears.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + previs + C(pay_rec)')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: The effect of payment method disappears.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + previs + no_previs')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Here's a version with the addition of a boolean for no prenatal visits.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + previs + no_previs + mager9')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: Now, surprisingly, the mother's age has a small effect.
End of explanation
"""


formula = ('boy ~ C(fbrace) + fhisp + previs + no_previs + fagerrec11')
model = smf.logit(formula, data=df)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: So does the father's age.  But both age effects are small and borderline significant.
End of explanation
"""


white = df[(df.mbrace==1) & (df.fbrace==1)]
len(white)

"""
Explanation: What's up with prenatal visits?
The predictive power of prenatal visits is still surprising to me.  To make sure we're controlled for race, I'll select cases where both parents are white:
End of explanation
"""


var = 'previs'
white[[var, 'boy']].groupby(var).aggregate(series_to_ratio)

"""
Explanation: And compute sex ratios for each level of previs
End of explanation
"""


formula = ('boy ~ previs + no_previs')
model = smf.logit(formula, data=white)
results = model.fit()
summarize(results)
results.summary()

inter = results.params['Intercept']
slope = results.params['previs']
inter, slope

previs = np.arange(-5, 5)
logodds = inter + slope * previs
odds = np.exp(logodds)
odds * 100

formula = ('boy ~ dmar')
model = smf.logit(formula, data=white)
results = model.fit()
summarize(results)
results.summary()

formula = ('boy ~ lowed')
model = smf.logit(formula, data=white)
results = model.fit()
summarize(results)
results.summary()

formula = ('boy ~ highbo')
model = smf.logit(formula, data=white)
results = model.fit()
summarize(results)
results.summary()

formula = ('boy ~ wic')
model = smf.logit(formula, data=white)
results = model.fit()
summarize(results)
results.summary()

formula = ('boy ~ obese')
model = smf.logit(formula, data=white)
results = model.fit()
summarize(results)
results.summary()

formula = ('boy ~ C(pay_rec)')
model = smf.logit(formula, data=white)
results = model.fit()
summarize(results)
results.summary()

formula = ('boy ~ mager9')
model = smf.logit(formula, data=white)
results = model.fit()
summarize(results)
results.summary()

formula = ('boy ~ youngm + oldm')
model = smf.logit(formula, data=white)
results = model.fit()
summarize(results)
results.summary()

formula = ('boy ~ youngf + oldf')
model = smf.logit(formula, data=white)
results = model.fit()
summarize(results)
results.summary()

"""
Explanation: The effect holds up.  People with fewer than average prenatal visits are substantially more likely to have boys.
End of explanation
"""


