# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # 1. get to the dataset directory

# +
# # cd new_adult_income_data/

# +
# # cd folktables/
# -

# !ls

# # 2. Download the new adult income dataset

from folktables import ACSDataSource, ACSEmployment, ACSIncome
# from . import folktables
from pandas import read_csv
from collections import Counter
import numpy as np
import pandas as pd
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split

data_source = ACSDataSource(survey_year='2018', horizon='1-Year', survey='person')
acs_data = data_source.get_data(states=["CA"], download=True)
features, label, group = ACSIncome.df_to_numpy(acs_data)

# +
# features.shape
# -

df = pd.read_csv('./raw_datasets/new_adult_income.csv')

df.head()

# df.drop('Unnamed: 0',  axis='columns', inplace=True)
df

column_names = ['age', 'workclass', 'education','marital-status', 'occupation', 'native-country', 'relationship', 'hours-per-week','gender','race']

df = pd.DataFrame(features, columns = column_names)

df

df['income'] = label

df['income'] = df['income'].map(lambda item : 0 if item ==False  else 1)
# df = df.replace({'PINCP': {False: 0,  True: 1}})

df.head()

df[]

df.to_csv("../../training_without_unlearning/raw_datasets/new_adult_income.csv")

adult = df

# # Data Visualization

# +
import pandas as pd
from IPython.display import Markdown, display
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc
import numpy as np
from sklearn import metrics



def printmd(string):
    display(Markdown(string))
    


# -

target = label
counter = Counter(target)
for k,v in counter.items():
    per = v / len(target) * 100
    print('Class=%s, Count=%d, Percentage=%.3f%%' % (k, v, per))

printmd('## 2.1. gender vs Income')

# +
gender = round(pd.crosstab(adult.gender, adult.income).div(pd.crosstab(adult.gender, adult.income).apply(sum,1),0),2)
# gender.sort_values(by = '>50K', inplace = True)
ax = gender.plot(kind ='bar', title = 'Proportion distribution across gender levels')
ax.set_xlabel('Gender level')
ax.set_ylabel('Proportion of population')

printmd('We plot a bar graph showing the proportion of income classes across the genders in figure \
        below. From the graph, at an overall view, there exists a wage gap between females and males. \
        Since we do not have the exactly value of the income, we are limited to only observing that the \
        proportion of males earning more than 50k a year is more than double of their female counterparts. \
        ')

# +
printmd('## 2.2. Education vs Income')

education = round(pd.crosstab(adult.education, adult.income).div(pd.crosstab(adult.education, adult.income).apply(sum,1),0),2)
# education = education.reindex(sorted(edu_level, key=edu_level.get, reverse=False))

ax = education.plot(kind ='bar', title = 'Proportion distribution across education levels', figsize = (10,8))
ax.set_xlabel('Education level')
ax.set_ylabel('Proportion of population')

printmd('I plot a bar graph showing the proportion of income classes across education levels \
        in the figure below. As one would expect, we see from the bar graph below that as the \
        education level increase, the proportion of people who earn more than 50k a year also \
        increase. It is interesting to note that only after a master\'s degree, the proportion of \
        people earning more than 50k a year, is a majority.')
        

# +
gender_workclass = round(pd.crosstab(adult.workclass, [adult.income, adult.gender]).div(pd.crosstab(adult.workclass, [adult.income, adult.gender]).apply(sum,1),0),2)
gender_workclass.plot(kind = 'bar', title = 'Proportion distribution across gender for each workclass', figsize = (10,8), rot = 30)
ax.set_xlabel('Gender level')
ax.set_ylabel('Proportion of population')

printmd('Taking a closer look at the disparity in income between men and women, plot the \
        proportion of men and women who earn more than 50k a year, across all the working \
        classes as seen in Fig. 3. We see that men always have a higher proportion earning \
        more than 50k a year than women, except for the \'without.pay\' working class, where \
        they have the same proportion.')
# -

# The class distribution is then summarized, confirming a modest class balance with approximately 59 percent for the majority class (<=50K) and approximately 41 percent for the minority class (>50K).

# create histograms of numeric input variables
from pandas import read_csv
from matplotlib import pyplot
# select columns with numerical data types
num_ix = df.select_dtypes(include=['int64', 'float64']).columns
# select a subset of the dataframe with the chosen columns
subset = df[num_ix]
# create a histogram plot of each numeric variable
subset.hist()
pyplot.show()





# We are plotting a Bar graph using the pandas .plot() function to visualize the relationship between education and income attributes. We achieve it by grouping the values of education using pandas .groupby() function against the mean of income attribute using .mean(). In the same way, we can plot the Bar graphs for all the other independent attributes against income as we see below.

df.groupby('SEX').PINCP.mean().plot(kind='bar')

df.groupby('AGEP').PINCP.mean().plot(kind='bar')

df.groupby('SCHL').PINCP.mean().plot(kind='bar')

df.groupby('RAC1P').PINCP.mean().plot(kind='bar')

# !pip3 install mlxtend

df = pd.read_csv('./raw_datasets/new_adult_income.csv')

df.drop('Unnamed: 0',  axis='columns', inplace=True)
df

column_names = ['age', 'workclass', 'education','marital-status', 'occupation', 'native-country', 'relationship', 'hours-per-week','gender','race','income']

df.columns = column_names

df.head()

gender = df['gender']
gender

df['male'] = gender
df['female'] = gender

df = df.replace({'male': {2.0: 0, 1.0: 1}})
df = df.replace({'female': {2.0: 1, 1.0: 0}})
# df.drop('gender')

df.

for i in range(df['gender'].shape[0]):
    if i in male_index[0]:
        print(i)
        df['male'] = '1'
        df['female'] = '0'
    else:
        df['male'] = '0' 
        df['female'] = '1'

df['female']



StarcoderdataJupyterScriptsDedupFiltered