# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# ### Load the data

df_train_feature = pd.read_csv('../data.nosync/raw/train_features.csv')
df_train_target = pd.read_csv('../data.nosync/raw/train_salaries.csv')
df_test_feature = pd.read_csv('../data.nosync/raw/test_features.csv')

# ### Familiarize with data and sanity check it
#
# Get to know about the data and make sure the type of the data in each column is matching the description and column name. 

# +
num_train_samples, num_features = df_test_feature.shape
df_train = df_train_feature.merge(df_train_target, on='jobId')

print(f'There are {num_train_samples} train samples in training data with {num_features} features')
display(df_train_feature.info())
display(df_train_feature.head(3))
# -

df_train_target.info()
df_train_target.head(3)


# ### Check for missing info
#
# check if what is the percentage of missing values (anything that is null, None, etc)

def replace_missing_data_with(df_list, missing_data_list=None, with_this=[None]):
    if missing_data_list is None or with_this is None:
        return
    
    if type(df_list) is not list:
        raise TypeError(f'df_list should be a pandas list. {type(df_list)} is not accepted here. ')
    
    for df in df_list:
        if type(df) is not pd.core.frame.DataFrame:
            raise TypeError(f'df_list should should contain only pandas pandas data-frame. list of {type(df)} is not accepted here. ')
    
    if type(missing_data_list) is not list:
        missing_data_list = list(missing_data_list)

    for df in df_list:
        df.replace(missing_data_list, with_this, inplace=False)


df_test_feature.replace(['NONE'], [None], inplace=True)
df_test_feature.head(10)
for col in df_test_feature:
    print(f"percentage of missing data in {col} column is equal to {100.0*df_test_feature[col].isnull().sum()/num_test_samples:0.2f}")

df_train_feature.replace(['NONE'], [None], inplace=True)
df_train_feature.head(10)
for col in df_train_feature:
    print(f"percentage of missing train data in {col} column is equal to {100.0*df_train_feature[col].isnull().sum()/num_train_samples:0.2f}")

# This shows that a significant portion of "degree" and "major" is missing data and could potentially be dropped. But at the same time they are intuitively important features. 
# For now we are going to keep them. So reverting back the None values to MISSING so that they get their own categroies. 

df_test_feature.replace([None], ['MISSING'], inplace=True)
df_train_feature.replace([None], ['MISSING'], inplace=True)

# sanity check there is no NaN, or None data
df_train.salary.isnull().any()

# ### Should we normalized numerical data for GD based optimizers?
#
# In this part we wanna check what is the range of the diferent numerical variables. If they ar ein the same order, then we there is no need to normalized or standardize them. Otherwise, we need to take care of scaling them inot the same range

print(f"range of yearsExperience varable is {df_train_feature['yearsExperience'].max()-df_train_feature['yearsExperience'].min()}")
print(f"range of milesFromMetropolis varable is {df_train_feature['milesFromMetropolis'].max()-df_train_feature['milesFromMetropolis'].min()}")

# **There is no need for scaling**, based on this investigation. It should be noted that any new feature should fall into this range, as well, if we are going to use any GD based optimization model. 

# ### Summerize Categorical and Numerical features
#
# Just get a hang of what is the statistics of both numerical and categorical values. 

df_train.describe(include = ['O'])

# jobId is unique and therefore does not carry any information. Hence, dropping it here onward. 

df_train.drop(columns='jobId', axis=1, inplace=True)

## get the list of categorical features
cat_feat_list = list(df_train.describe(include = ['O']).columns)
print(cat_feat_list)

# Apart from ```degree``` and ```major``` features, all others seems to have a uniform distribution of data among difernet categories. 

_, axes = plt.subplots(2, 1)
for i, feature in enumerate(['degree', 'major']):
    sns.countplot(x=feature, data=df_train, ax=axes[i])

df_train.describe(include = [np.number])

num_feat_list = df_train.describe(include = [np.number]).columns
print(num_feat_list)


# ### Visualize and probe salary
#
# Hist plot all salaries and research it for potential feature eng efforts. Get rif of poterntial outliers. 

def get_iqr_bounds(df, feature, alpha = 1.5):
    
    if not isinstance(feature, str):
        raise TypeError('feature should be string, indicating the name of the feature')
    
    stats = df[feature].describe()
    iqr = stats['75%'] - stats['25%']
    lower, upper = stats['25%'] - alpha * iqr, stats['75%'] + alpha * iqr
    return lower, upper


salary_lower_bound, salary_upper_bound = get_iqr_bounds(df_train, 'salary')
df_train[df_train.salary < salary_lower_bound]
df_train[df_train.salary > salary_upper_bound]["jobType"].unique()

df_train.jobType.unique()

df_train[(df_train.salary > salary_upper_bound) & (df_train.jobLevel == 'MANAGER')]

sns.distplot(df_train.salary, bins=25, rug=True)

sns.jointplot(data=df_train, x="salary", y="milesFromMetropolis", kind="kde")

sns.jointplot(data=df_train, x="salary", y="yearsExperience")

sns.pairplot(df_train, hue="industry")

# from these plots and investigation, looks like we have lower bound outliers, so we'll get rid of them.

df_train = df_train[df_train.salary > salary_lower_bound]
df_train.shape

# **look into categorical features**
#
# try to see if there is any correlation between any of the categories and the target variable.

df_train.groupby("degree")["salary"].describe()

df_train.groupby("degree")["salary"].mean().sort_values()

df_train["degree"].unique()

df_train.groupby(["degree", "major"])["salary"].mean().sort_values()

# from this, it is apparent that if we treat degree as an ordinal value, there is a high correlation between degree and salary. will consider this as an ordinal value, with HIGH_SCHOOL being the lowest and DOCTORAL being the highest.

df_train.groupby("major")["salary"].mean().sort_values()

df_train.groupby("industry")["salary"].mean().sort_values()


def seperate_features_into_categorical_numerical_time(df):
    '''returns three seperate dataframes. the first one is the categorical portion and the second one is the numerical part and third one is timeData portion of the data'''
    return df.select_dtypes(include=['category', 'object']), df.select_dtypes(include='number'), df.select_dtypes(include=['datetime', 'datetime64', 'timedelta64', 'timedelta'])



df_train_cat_feat, df_train_num_feat, _ = seperate_features_into_categorical_numerical_time(df_train)


# **Correlation between features**
#
# look into the correlation between all pairs of features. adding new ordinal features for features "degree, industry, and major" based on above mappings, to investigate any correlation between these features. 

def encode_ordinal(df, features_list, target, replace=False, verbose=False):
    
    for feature in features_list:
        
        if feature not in df.columns:
            print(f"{feature} is not among the feature. skipping ... ")
            continue
        
        map_series = df.groupby(feature)[target].mean().sort_values()
        
        if verbose:
            print(map_series)
        
        map_dict = map_series.to_dict();
        
        if replace:
            df[feature] = df[feature].replace(map_dict)
        else:
            df[feature + "_ordinal"] = df[feature].replace(map_dict)
    
    return df


df_train_new = encode_ordinal(df_train, ["industry", "degree", "major"], "salary")

df_train_new[['industry_ordinal', 'salary']].head()

df_train_new.corr()

# +

sns.heatmap(df_train_new.corr(), cmap='Blues', annot=True)
# -

# There doesn't seem to be any significant correlation between features. 

# ### Base-Line model for feature importance

# In this section we will build two simple models using RandomForest and XGBoost, in order to identify the most important features. This info can be used to create new categorical features based on the combination of these important features. Hopefully, these features will improve the results of SVM, Linear Regression, and KNN meahtods, as these methods are not capable of capturing the relationship between features. 
#
# It should be noted that sklearn.ensemble.RandomForestRegressor internally will convert all data into np.float32. That means that all the categorical features should be encodede into numerical flaot features. 
#
# Usually the best way of encoding categorical features is  1-hot-encoding which does not assume any relation between different categories. One down side of this method is that it will result in highly sparse matrix after encoding especially when the categorical feature is of high cardinality. In our case, one of the seeminglt important vategorical feature, companyId, is hight cardinal and will suffer from this drawback. Also since this is just a baseline method, and in general DT based methods can handle categorical features by nature at this point we will just Ordinally encode the categorical features. 
#
# Ordinal encoding, suggests that there is hierarchical relation between different categories. Since we are predicting salary, it seems to be fair to assume that the order of categories in each feature is proportional to the mean of salary for that category. In other words, we will be using ```encode_ordinal()``` method introduced above to take care of tis encoding. 

from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split

# +
## Some minor clean-up, enconding and the train/test split

# first get rid of unique feature as it doesn't carry any info - jobId
try:
    df_train.drop(['jobId'], axis=1, inplace=True)
except KeyError as kr:
    print(KeyError)

# ordinal encoding of categorical features
df_train_encoded = encode_ordinal(df_train, cat_feat_list, 'salary', replace=True, verbose=False)

# some more clean-up 
X_target  = df_train_encoded['salary']
X_feature = df_train_encoded.drop(['salary'], axis=1)

display(X_feature.head(3))

# train/test split
X_train, X_test, y_train, y_test = train_test_split(X_feature, X_target, test_size=0.20)
# -

# fit a baseline random forest to extract the important features
rf_model = RandomForestRegressor(n_estimators=100, max_depth=5, n_jobs=4, random_state=13)
rf_model.fit(X_train, y_train)
# print(rf_model.feature_importances_)

# print the feature importance
display(dict(zip(X_train.columns, rf_model.feature_importances_)))

# **NOTE** We will use the result of this method in order generate new features. First thing in mind is that we can generate new categorical features based on combination of top 2~4 features. 

# +
## PCA models?
# -

# #### PlayGround - PlayGround - PlayGround - PlayGround

for col in ['degree']:
    ax = sns.barplot(x=col, y='salary', data=df_train)

sns.catplot(x='degree', y='salary', hue='major', data=df_train)

## looks like we have a candidate for categorical cobinations
0, 1, 2
sns.countplot(x='jobType', data=df_train, hue='degree')

StarcoderdataJupyterScriptsDedupFiltered