# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ## Data Challenge #1
#
# **Author:** <NAME>
#
# **Date:** February 18, 2018

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
# %matplotlib inline

data = pd.read_csv('employee_retention_data.csv')
print(data.shape)
data.head()

data.dtypes

#not an imbalanced dataset
data['quit_date'].isna().sum()

#check nans, no rows need to be eliminated
data.isna().sum()

#convert the dates to the appropriate types 
data['join_date']=data['join_date'].astype('M')
data['quit_date']=data['quit_date'].astype('M')
data.dtypes

#check that nans were maintained
data['quit_date'].isna().sum()

#add a quit column 
data['quit']=data['quit_date'].notna()


# +
#calculate months at company
def monthsAtCompany(start,end):
    if type(end) == type(pd.NaT):#very tricky comparison to make
        period=(pd.to_datetime('2015-12-13', format='%Y-%m-%d')
                -start)/np.timedelta64(1, 'M')
    else:
        period=(end-start)/np.timedelta64(1, 'M')
    return period

#np.vectorize(monthsAtCompany)(data['join_date'], data['quit_date'])


# -

#check that calculation works
monthsAtCompany(data['join_date'][2], data['quit_date'][2])

#define tenure duration column
for i in range(len(data)):
    data.loc[i,'tenure']=monthsAtCompany(data['join_date'][i], data['quit_date'][i])

#check if tenure seems correct
data.head()

print(data['tenure'].describe())
data['tenure'].hist(bins=20,figsize=(8, 4))

#people in the region below a month at the company
data[data['tenure']<1]['tenure'].hist()

#people in the region near december 2015
data[data['quit_date']>pd.to_datetime('2015-12-01', format='%Y-%m-%d')
    ]['quit_date'].hist(figsize=(8, 3))

# * what's tricky about this problem is that we're being asked to predict when an employer is going to quit. 
# * This seems like a regression problem, but in fact mixes two predictions, whether an employee is going to quit (classification problem) and when (regression)
# * might be simpler to **first attempt a binary classification** on retention and then try to predict the tenure length
# * afterwards a regressor can be trained, but only by using data on employees that quit. I'm not sure how to then combine both predictions. 
# * **Note:** the relevant dates (join, quit) are in the period between 2011 and 2015 which I think does not overlap with any major economical downturn which could be very important for the analysis (indeed, I confirmed this suspicion below). Also, from the description of the problem **I assume that employees that left, did so by quitting and from being fired**. 

#sanity checks:
print(data['seniority'].describe())
data['seniority'].value_counts().sort_index(ascending=False)

#two values are incorrect, so erase them:
data[data['seniority']> 30]

data.drop([24700,24701],inplace=True)
data.shape


def convert4prediction(series,inverse=False):
    '''
    converts a datetime64 pandas data type to a numerical value, by 
    using unix time (number of nanoseconds since 1970-01-01) as a 
    reference point:
    https://stackoverflow.com/questions/50979873/linear-fit-to-pandas-datetime64-values
    The function returns the number of years passed since 2011-01-01
    (date from problem statement). 
    Function can apply the inverse function too by seeting the second
    parameter to True.
    '''
    if inverse:
        return pd.to_datetime((series+41)*(1e9 *60*60*24*365))
    else:
        return pd.to_numeric(series)/(1e9 *60*60*24*365) -41


#remove employee_id from variables
variables=data.columns.tolist()
print(variables)
del variables[0]
variables

# #copy data to new dataframe and convert timestamp colums to numerical
df=data[variables].copy()
df['join_date']=convert4prediction(data['join_date'])
#nan positions preserved because of matching index:
df['quit_date']=convert4prediction(data[data['quit']]['quit_date'])
df.head()

#encode categoricals
df['dept'].value_counts()

data['company_id'].value_counts()

df = pd.get_dummies(df, columns=['dept'])
df = pd.get_dummies(df, columns=['company_id'])
df['quit']=df['quit'].map({True:1,False:0})
df.head()

#separate into train, cv, test:
from sklearn import model_selection #for StratifiedKFold, cross_val_score 
from sklearn.linear_model import LogisticRegression

seed=106637
train,test = model_selection.train_test_split(df,test_size=0.1,
                                        random_state=seed,stratify=df['quit'])


# +
#some EDA
def distributionPlot(df,col,units,grouping='quit'):
    '''conveniently making EDA plots'''
    plt.figure(figsize=(9,6))
    sns.distplot(df.groupby(grouping)[col].get_group(0),kde=False,rug=False, label=grouping.capitalize()+'=0');
    sns.distplot(df.groupby(grouping)[col].get_group(1),kde=False,rug=False, label=grouping.capitalize()+'=1');
    plt.xlabel(col.capitalize()+units, fontsize=14)
    plt.title(col.capitalize()+' distribution segregated by \''+grouping+'\' status', fontsize=16);
    plt.legend();
    
distributionPlot(train,'tenure',' (months)')
# -

distributionPlot(train,'salary',' (dollars)')

distributionPlot(train,'seniority',' (years)')

distributionPlot(train,'join_date',' (years after 2011)')
#this is a nice plot :)

# * data distributions segregated by quitting status show that `salary` and `seniority` do not seem to play an obviously important role
# * however it seems that **`join_date` is important** for predicting wheter an employee quits or not. Seems that the closest to the 'great recession' the employee joined the company, the longer the he or she stayed at the company. As the economy improved employees were more secure that they will be able to find jobs elsewhere
# * department and company variables might show good predictive power
# * I will use the original data to show these two variables (`dept` and `company_id`)

plt.figure(figsize=(9,6))
sns.barplot(x='dept',y='quit',data=data)
plt.xlabel('\'company_id\'', fontsize=14)
plt.ylabel('P(quit = True|\'dept\')', fontsize=14);
#there seems to be some statistically significant differences between departments (i.e engineer and sales or design)

plt.figure(figsize=(9,6))
sns.barplot(x='company_id',y='quit',data=data)
plt.xlabel('\'company_id\'', fontsize=14)
plt.ylabel('P(quit = True|\'company_id\')', fontsize=14);
#the confidence bar sizes are obvious given the sizes of companies 11 and 12
#seems that company 2 has a bit of difference with respect to other companies

# ### Let's build a predictive model...
# (logistic regression has already been loaded)

target='quit'
features = df.columns.tolist()
features

features.remove('quit') #the target
features.remove('quit_date')#will keep tenure which is obtained from join_date and quit_date
features

model1=LogisticRegression()

# **scaling the data...**

from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
X_train = scaler.fit_transform(train[features])
X_test = scaler.transform(test[features])

model1.fit(X_train, train['quit'])
print('Accuracy of Logistic regression classifier on training set: {:.2f}'
     .format(model1.score(X_train, train['quit'])))
print('Accuracy of Logistic regression classifier on test set: {:.2f}'
     .format(model1.score(X_test, test['quit'])))
#maybe overfitting?

# * indeed **`join_date` is the most important feature**. Note that salary is the least important one!!

feat_importance = pd.DataFrame(data={'feature': features, 'coefficient': model1.coef_[0]})
feat_importance

# ### Predicting tenure:

reduced=df[df['quit']==1]
reduced.head()

Rtrain,Rtest = model_selection.train_test_split(reduced,test_size=0.1,random_state=seed)

from sklearn.linear_model import LinearRegression
model2= LinearRegression()

Rtarget='quit_date'
Rfeatures = reduced.columns.tolist()
Rfeatures

Rfeatures.remove(Rtarget) #the target
Rfeatures.remove('quit')  
Rfeatures.remove('tenure') 
Rfeatures

R_train = scaler.fit_transform(Rtrain[Rfeatures])
R_test = scaler.transform(Rtest[Rfeatures])

model2.fit(R_train, Rtrain[Rtarget])
print('R^2 of Linear regression on training set: {:.2f}'
     .format(model2.score(R_train, Rtrain[Rtarget])))
print('R^2 of Linear regression on test set: {:.2f}'
     .format(model2.score(R_test, Rtest[Rtarget])))
#very hard to estimate quit_date

Rfeat_importance = pd.DataFrame(data={'feature': Rfeatures, 'coefficient': model2.coef_})
Rfeat_importance



StarcoderdataJupyterScriptsDedupFiltered