# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Income classification

# Dataset description:
#
# * age: continuous.
# * workclass: Private, Self-emp-not-inc, Self-emp-inc, Federal-gov, Local-gov, State-gov, Without-pay, Never-worked.
# * fnlwgt: continuous.
# * education: Bachelors, Some-college, 11th, HS-grad, Prof-school, Assoc-acdm, Assoc-voc, 9th, 7th-8th, 12th, Masters, 1st-4th, 10th, Doctorate, 5th-6th, Preschool.
# * education-num: continuous.
# * marital-status: Married-civ-spouse, Divorced, Never-married, Separated, Widowed, Married-spouse-absent, Married-AF-spouse.
# * occupation: Tech-support, Craft-repair, Other-service, Sales, Exec-managerial, Prof-specialty, Handlers-cleaners, Machine-op-inspct, Adm-clerical, Farming-fishing, Transport-moving, Priv-house-serv, Protective-serv, Armed-Forces.
# * relationship: Wife, Own-child, Husband, Not-in-family, Other-relative, Unmarried.
# * race: White, Asian-Pac-Islander, Amer-Indian-Eskimo, Other, Black.
# * sex: Female, Male.
# * capital-gain: continuous.
# * capital-loss: continuous.
# * hours-per-week: continuous.
# * native-country: United-States, Cambodia, England, Puerto-Rico, Canada, Germany, Outlying-US(Guam-USVI-etc), India, Japan, Greece, South, China, Cuba, Iran, Honduras, Philippines, Italy, Poland, Jamaica, Vietnam, Mexico, Portugal, Ireland, France, Dominican-Republic, Laos, Ecuador, Taiwan, Haiti, Columbia, Hungary, Guatemala, Nicaragua, Scotland, Thailand, Yugoslavia, El-Salvador, Trinadad&Tobago, Peru, Hong, Holand-Netherlands.
#

# +
from IPython.core.interactiveshell import InteractiveShell

InteractiveShell.ast_node_interactivity = "all"

# +
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import pyplot as plt

# %matplotlib inline
# -

df = pd.read_csv("dataset/income_evaluation.csv")
df.head(4)

df.columns = [col.strip() for col in df.columns]
df.columns

df.info()

# +
nominal_features = [
    "workclass",
    "marital-status",
    "occupation",
    "relationship",
    "race",
    "sex",
    "native-country",
]

ordinal_features = ["education-num"]
redundant_features = ["education"]
numerical_features = ["age", "fnlwgt", "capital-gain", "capital-loss", "hours-per-week"]

target_column = "income"
# -

# Looks like education and education-num provide the same information
# but in different forms. Consider education-num as oridinal encoding of education
df.loc[:, "education-num"].nunique(), df.loc[:, "education"].nunique()

df.groupby(by=["education-num", "education"])["age"].count().sort_index()

df.drop(columns=redundant_features, inplace=True)

# +
for col in nominal_features:
    df.loc[:, col] = df[col].astype("category")

for col in ordinal_features:
    df.loc[:, col] = df[col].astype("category")

df.loc[:, target_column] = df.loc[:, target_column].astype("category")

df.info()

# +
# split training and test set
from sklearn.model_selection import train_test_split

X = df.iloc[:, :-1]
y = df.iloc[:, -1]
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.1, stratify=y, random_state=42
)

X_train.shape, X_test.shape

# +
from sklearn.preprocessing import LabelEncoder

target_encoder = LabelEncoder()
y_train = target_encoder.fit_transform(y_train)
# -

df_train = X_train.copy()
df_train.describe()

classes, counts = np.unique(y_train, return_counts=True)
pd.Series(counts, index=classes) / y_train.shape[0]

for col in df_train.columns:
    if str(df_train[col].dtype) == "category":
        df_train[col].value_counts()

df_train.replace(to_replace="?", value="other", inplace=True)

df_train.columns

# using pearson correlation coefficient to see if numerical features are correlated
pd.concat((df_train.loc[:, numerical_features], y_train), axis=1).corr()

df_train.loc[:, numerical_features].hist(figsize=(20, 10))

# +
from sklearn.feature_selection import SelectKBest, f_classif

print(f"Total numerical features: {len(numerical_features)}")
feature_selector = SelectKBest(score_func=f_classif, k=4)
feature_selector.fit(df_train.loc[:, numerical_features], y_train)
feature_selector.get_support()

# +
for col, retain in zip(numerical_features, feature_selector.get_support()):
    if not retain:
        df_train.drop(columns=col, inplace=True)

df_train.columns

# +
from sklearn.feature_selection import chi2
from sklearn.preprocessing import OrdinalEncoder

categories = []
for idx, col in enumerate(nominal_features):
    categories.append(df_train.loc[:, col].unique())

cat_encoder = OrdinalEncoder(categories=categories, dtype=np.int8)
ordinal_encoded = cat_encoder.fit_transform(df_train.loc[:, nominal_features])

scores, pvalues = chi2(ordinal_encoded, y_train)

list(zip(scores, nominal_features))

# +
from sklearn.feature_selection import mutual_info_classif

mutual_info_classif(ordinal_encoded, y_train)
# -

chi2_selector = SelectKBest(chi2, k=5)
chi2_selector.fit(ordinal_encoded, y_train)
chi2_selector.get_support()

mut_info_selector = SelectKBest(mutual_info_classif, k=5)
mut_info_selector.fit(ordinal_encoded, y_train)
mut_info_selector.get_support()

# +
# as of now I will go with mutual_info_classif
for col, retain in zip(nominal_features, mut_info_selector.get_support()):
    if not retain:
        df_train.drop(columns=col, inplace=True)

df_train.columns

# +
from sklearn.preprocessing import OneHotEncoder

nominal_encoder = OneHotEncoder(sparse=False, dtype=np.int8)
x_train_enc = nominal_encoder.fit_transform(
    df_train.loc[
        :, ["workclass", "education-num", "marital-status", "occupation", "relationship", "sex"]
    ]
)
x_train_enc.shape
# -

df_train_enc = pd.DataFrame(
    x_train_enc, columns=np.array(list(chain(*nominal_encoder.categories_)), dtype=object)
)
df_train_enc.shape

# +
df_temp = df_train.loc[:, ["age", "capital-gain", "capital-loss", "hours-per-week"]]
df_temp.shape
df_temp.values.shape, df_train_enc.values.shape

cols = list(df_temp.columns) + list(df_train_enc.columns)
df_train_trans = pd.DataFrame(np.c_[df_temp, df_train_enc], columns=cols)

df_train_trans.shape

# +
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import RepeatedStratifiedKFold, cross_val_score

rf_clf = RandomForestClassifier(n_estimators=100, random_state=42)
scores = cross_val_score(rf_clf, df_train_trans, y_train, cv=10, n_jobs=-1, scoring="accuracy")
scores.mean(), scores.std()

StarcoderdataJupyterScriptsDedupFiltered