# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""

#### The dataset, contain information on 36,000 insurance policies . We have to predict the ultimate insurance amount claimed by a person. In this, notebook, I am trying to compare the working of KNN, GLM and Random Forest for linear regression. Both KNN and GLM were carried with and without feature selection
<br>
"""
"""
#### The missing values accounted to about 120 datapoint, so they were removed.
"""
# importing required 

import matplotlib.pyplot as plt
import seaborn as sns
import sklearn.preprocessing as pre
import sklearn.neighbors as NN
import sklearn.linear_model as lm
from sklearn import model_selection as ms
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
"""
## 1. Data loading
"""
#loading the training data
# since the first row of the original file was unordered, it was made coherent at first
data = pd.read_csv("../input/training-data/train_SJC.csv")
data.head()
"""
### data summary <br>
#### Data fields
* ClaimNumber: Unique policy identifier
* DateTimeOfAccident: Date and time of accident
* DateReported: Date that accident was reported
* Age: Age of worker
* Gender: Gender of worker
* MaritalStatus: Martial status of worker. (M)arried, (S)ingle, (U)unknown.
* DependentChildren: The number of dependent children
* DependentsOther: The number of dependants excluding children
* WeeklyWages: Total weekly wage
* PartTimeFullTime: Binary (P) or (F)
* HoursWorkedPerWeek: Total hours worked per week
* DaysWorkedPerWeek: Number of days worked per week
* ClaimDescription: Free text description of the claim
* InitialIncurredClaimCost: Initial estimate by the insurer of the claim cost
* UltimateIncurredClaimCost: Total claims payments by the insurance company. This is the field you are asked to predict in the test set.
"""
data.info() 
data.describe()
data.describe(include='object') 
data.shape   # to check the shape of the dataset
"""
Thus the training dataset has 36176 examples with 15 features.
"""
"""
## 2. Data preprocessing
"""
"""
### checking whether the variables of object datatype have "unknown" category 
"""
data['Gender'].value_counts()
# removing the gender other than male or female, since there is only one such value
data = data[data['Gender']!='U']
data['Gender'].value_counts()
data['MaritalStatus'].value_counts()
(3542/36175)*100
# leaving marital status as such since other category constitute about 10 % of the total data
data['PartTimeFullTime'].value_counts()
data.isnull().sum()    # checking for missing values
"""
MaritalStatus, WeeklyWages, HoursWorkedPerWeek are the three columns with missig values. <br>

"""
"""
## 3. Data transformation
"""
# copying the dataset so that original dataset remains in tact and we can do the transformations on the new dataset

data_new = data
# comparing the data shape before and after dropping the missing values

data_new.shape , data_new.dropna().shape

# so, if we drop missing values, 127 rows are dropped. Since the dataset is large, we can proceed with this step

df= data_new.dropna()
df.head()
df.columns
"""
## 4. EDA
"""
df['Age'].min() , df['Age'].max()
# creating a new columns in which age is grouped

df['Age_group']=pd.cut(df['Age'],bins=[13,18,41,59,80], labels=['Too young','Young','Middle aged','Old'])
df['Age_group'].value_counts()
df['WeeklyWages'].min() , df['WeeklyWages'].max()
# creating a new columns in which Weekly wage is grouped

df['Wage_category'] = pd.cut(df['WeeklyWages'],bins=[0,50,250,900,2500,8000], labels=['very low','low','moderate','high','very high'])
df['Wage_category'].value_counts()
# copying the dataset for data encoding and scaling
df_non_miss = df
# finding out the range of the column 'UltimateIncurredClaimCost'
df['UltimateIncurredClaimCost'].min() , df['UltimateIncurredClaimCost'].max()
# binning the result column for ease of EDA
df['UIC_bin'] = pd.cut(df['UltimateIncurredClaimCost'],bins=[100,500,1000,5000,10000,5000000], labels=['too low','level2','level3','level4','too high'])
df['UIC_bin'].value_counts()
# checking the shape of the dataset
df.shape
# outlier analysis

df.plot.box(figsize=(25,6))  # only continuous variables are plotted
sns.distplot(df['UltimateIncurredClaimCost']) # plot to find the distribution of a continuous variable.
"""
The data is highly positively skewed
"""
"""
Now, since,'UltimateIncurredClaimCost' is the final outcome, we are trying to plot it on the basis of several categorical variables(either provided in the dataset or created by us).
"""
sns.catplot(data=df,x='UltimateIncurredClaimCost',kind ='box', col = 'Gender')
sns.catplot(data=df,x='UltimateIncurredClaimCost',kind ='box', col = 'Age_group')
sns.catplot(data=df,x='UltimateIncurredClaimCost',kind ='box', col = 'Wage_category')
"""
**From the above graphs, we can infer that there is one person who got a claimed of a large sum. <br>
<br>
It is a middle aged women with a salary ranging between 250 and 900 units.**
"""
"""
Since the datapoint is an outlier, we can remove the point from the dataset for ease of analysis. <br>
Otherwise, it can affect our prediction.
"""
# removing outlier

df[df['UltimateIncurredClaimCost']>1000000]  # this is the outlier value
# data without outlier

df = df[df['UltimateIncurredClaimCost']<1000000] #outlier removed
df.plot.box(figsize=(25,6))  #since there are no outliers, the plot is more clear now.
df.columns
cat_df = df.select_dtypes(exclude=[float,int])
cat_df.head()
num_df = df.select_dtypes(include=[float,int])
num_df.head()
list_df = [  'Gender','MaritalStatus', 'DependentChildren', 'DependentsOther', 
       'PartTimeFullTime', 'HoursWorkedPerWeek', 'DaysWorkedPerWeek',
        'InitialIncurredCalimsCost', 'Age_group', 'Wage_category', 'UIC_bin']
sns.pairplot(df[list_df],hue='Gender',diag_kind="kde")
sns.countplot(data=df , x= 'Gender' , hue = 'UIC_bin')
sns.countplot(data=df , x= 'Age_group' , hue = 'UIC_bin')
sns.countplot(data=df , x= 'Wage_category' , hue = 'UIC_bin')
sns.countplot(data=df , x= 'PartTimeFullTime' , hue = 'UIC_bin')
sns.countplot(data=df , x= 'DaysWorkedPerWeek', hue = 'UIC_bin')
sns.countplot(data=df , x= 'DependentChildren' , hue = 'UIC_bin')
sns.countplot(data=df , x='DependentsOther' , hue = 'UIC_bin')
"""
****

#### EDA CONCLUSION <br> 

1. Most of the claimers were men.
2. Most of them claimed an amount between 1000 and 5000 units.
3. Most of the claimers were young or middle aged.
4. Their weekly wages ranged between 50 and 900 units.
5. Most of the claimers worked for 5 days a week.
6. Mostly there were no dependent people.



****
"""
"""
## 4. Data Encoding and Scaling
"""
le = pre.LabelEncoder()
df.columns
# for data transformation purpose, all the variables should be either float or string.
# taking the required columns (with float value)
list_df_new=['Age', 'Gender','MaritalStatus', 'DependentChildren', 'DependentsOther', 'WeeklyWages',
       'PartTimeFullTime', 'HoursWorkedPerWeek', 'DaysWorkedPerWeek',
        'InitialIncurredCalimsCost','UltimateIncurredClaimCost']  
#label encoding all the values
for x in list_df_new:
    df[x] = le.fit_transform(df[x])
# old dataset modified with selected columns to create a new dataset
# all the columns are encoded
df_ml = df[list_df_new] 

df_ml.head()
df_ml.info()
# for data normalisation
# each value in the data is scaled between 0 and 1

min_max=pre.minmax_scale
df_test=min_max(df_ml.values)
"""
#### All the values are scaled between the range 0 and 1 scale
"""
# normalised data to a dataframe
df_norm = pd.DataFrame(data=df_test,columns = df_ml.columns.to_list())
df_norm.head()
# the dataset is divided into two.
# first part consists of all other values other than the output variable

features=df_norm.drop('UltimateIncurredClaimCost',axis=1)
Y=df_norm['UltimateIncurredClaimCost']
# splitting the data into train and test data.

x_train,x_test,y_train,y_test=ms.train_test_split(features,Y,test_size=0.3,random_state=1234566)
#checking the shape of the data

x_train.shape,x_test.shape,y_train.shape,y_test.shape
"""
## 5. Machine learning model
### KNN and GLM (before feature selction)
* Here, I have to build a regression model. At first, i am going to create a KNN regressor model, then a linear regressor model
"""
"""
KNN regressor: for different values of n_neighbours, the train and test score are calculated.
"""
#using KNN and specifying the parameters.

KNN=NN.KNeighborsRegressor(n_neighbors=25)
# fitting the data to the model

KNN.fit(x_train,y_train)
KNN.predict(x_test)
KNN.score(x_test,y_test)
KNN.score(x_train,y_train)
"""
**K  ....................  train score  ..............    test score** <br>
 <br>
 5 ..............0.8463563754247978 .........  0.7609426674886294 <br>
 10 .......... 0.824918474501286 .........  0.7785861677346249 <br>
 15 .......... 0.8165400362017157 ...........  0.7831364931774443 <br>
 25 .......... 0.8070922295398893..........  0.7835218825541196 <br>
 30...........0.8036581846568207..........0.7828346411572328
"""
"""
As the n-neighbour value(k value) increases, the test score is achieving better value. But, after a certain point, test score is declining. So, we can choose k=25 as the preferred value
"""
"""
 

Now, let us consider the Generalised linear model
"""
# splitting the dataset into other variables and outcome variable 
# similar to what we did earlier.

features=df_norm.drop('UltimateIncurredClaimCost',axis=1)
y=df_norm['UltimateIncurredClaimCost']
features.head()
y.head()
# splitting the data into train and test set

x_train,x_test,y_train,y_test=ms.train_test_split(features,y,test_size=0.3,random_state=1234566)
x_train.shape ,x_test.shape ,y_train.shape ,y_test.shape
# using glm for linear reagression
glm = lm.LinearRegression()
glm.fit(x_train,y_train)
glm.predict(x_test)
glm.score(x_test,y_test)
glm.score(x_train,y_train)
"""
### Feature importance selection
"""

model = DecisionTreeRegressor()
# fit the model
model.fit(x_train,y_train)
# get importance
importance = model.feature_importances_
# summarize feature importance
print('Feature \t Score: ')
for i,v in enumerate(importance):
    
    print(i,"\t",v)
# plot feature importance
plt.bar([x for x in range(len(importance))], importance)
plt.show()
"""
## 6. Feature importance
"""
# function to calculate glm test and train score

def f_train_test_score(data,drop_col):
    #normalising the data
    df_test=min_max(df_ml.values)  
    
    #converting normalised data to dataframe
    df_norm = pd.DataFrame(data=df_test,columns = df_ml.columns.to_list()) 
    
    
    # dividing features and outcomes into two files and then splitting it into train ans test sets
    features= df_norm.drop(drop_col,axis=1) 
    y= df_norm['UltimateIncurredClaimCost']
    x_train,x_test,y_train,y_test=ms.train_test_split(features,y,test_size=0.3,random_state=1234566)
    
    #using glm, fitting the data into glm and printing the train and test scores.
    glm = lm.LinearRegression()
    glm.fit(x_train,y_train)
    print(" The test score is ",glm.score(x_test,y_test))
    print(" The train score is ",glm.score(x_train,y_train))
   
"""
trying to delete columns which are less important on the basis of test score.
"""
f_train_test_score(df_ml,['UltimateIncurredClaimCost'])
"""
the test score is increasing, therefore the variable we dropped is of less importance
"""
"""
test score decreased, so don't drop the variable. similarly, continue till we get maximum test score.
"""
f_train_test_score(df_ml,['DependentsOther','DependentChildren','UltimateIncurredClaimCost'])
f_train_test_score(df_ml,['DependentsOther','PartTimeFullTime','UltimateIncurredClaimCost'])
"""
Here, by dropping the column 'PartTimeFullTime', the model performance is slightly increasing. So we can drop that column.
"""
f_train_test_score(df_ml,['DependentsOther','PartTimeFullTime','Gender','UltimateIncurredClaimCost'])
f_train_test_score(df_ml,['DependentsOther','PartTimeFullTime','MaritalStatus','UltimateIncurredClaimCost'])
f_train_test_score(df_ml,['DependentsOther','HoursWorkedPerWeek','PartTimeFullTime','UltimateIncurredClaimCost'])
f_train_test_score(df_ml,['DependentsOther','PartTimeFullTime','Age','UltimateIncurredClaimCost'])
f_train_test_score(df_ml,['DependentsOther','PartTimeFullTime','WeeklyWages','UltimateIncurredClaimCost'])
f_train_test_score(df_ml,['DependentsOther','PartTimeFullTime','InitialIncurredCalimsCost','UltimateIncurredClaimCost'])
"""
['DependentsOther','PartTimeFullTime','UltimateIncurredClaimCost'] are the columns to be dropped
"""
"""
If we drop the column 'InitialIncurredCalimsCost', the train and test scores are decreasing rapidly. So, don't drop the column and it is the most important feature in prediction of ultimate claim incurred.
"""
"""
## KNN after feature selection
"""
list_df_new1=['Age', 'Gender','MaritalStatus', 'DependentChildren', 'WeeklyWages',
             'HoursWorkedPerWeek', 'DaysWorkedPerWeek',
        'InitialIncurredCalimsCost','UltimateIncurredClaimCost']  
#label encoding all the values
for x in list_df_new1:
    df[x] = le.fit_transform(df[x])
df_ml1 = df[list_df_new1]
df_test=min_max(df_ml1.values)
# normalised data to a dataframe
df_norm = pd.DataFrame(data=df_test,columns = df_ml1.columns.to_list())
df_norm.head()
# splitting the dataset into other variables and outcome variable 
# similar to what we did earlier.

features=df_norm.drop('UltimateIncurredClaimCost',axis=1)
y=df_norm['UltimateIncurredClaimCost']
# splitting the data into train and test data.

x_train,x_test,y_train,y_test=ms.train_test_split(features,Y,test_size=0.3,random_state=1234566)
KNN=NN.KNeighborsRegressor(n_neighbors=20)
KNN.fit(x_train,y_train)
KNN.predict(x_test)
KNN.score(x_test,y_test)
KNN.score(x_train,y_train)
"""
## 7. Prediction
"""
# final test and train score is:
f_train_test_score(df_ml,['DependentsOther','PartTimeFullTime','UltimateIncurredClaimCost'])
# importing the test data

new_test = pd.read_csv("../input/machine-learning-24-hrs-hackathon/Test_SJC.csv")
new_test.head()
#checking for null values

new_test.isnull().sum()
# replacing null values from test data

new_test['MaritalStatus']=new_test['MaritalStatus'].fillna(new_test['MaritalStatus'].mode()[0])
new_test.isnull().sum()
new_test
new_list = ['Age','Gender', 'MaritalStatus', 'DependentChildren',
        'WeeklyWages', 'HoursWorkedPerWeek', 'DaysWorkedPerWeek', 'InitialIncurredCalimsCost']
test =new_test[new_list]
test.shape

for x in new_list:
    test[x]=le.fit_transform(test[x])
# normalising test data
final_test=min_max(test.values)
# importing the sample submission csv
sample = pd.read_csv('../input/machine-learning-24-hrs-hackathon/sample_submission.csv')
sample.shape
sample1 = sample
#predicted value of knn
predicted_knn = KNN.predict(test)
# adding new column to sample csv
sample['UltimateIncurredClaimCost']= predicted_knn
sample.head()  # dataframe of prediction using KNN
sample.to_csv("Sample Submission new.csv", index = False)  #saving the modified dataframe as a csv file
#new_sample = pd.read_csv('../input/machine-learning-24-hrs-hackathon/sample_submission.csv')
# prediction using glm

#predicted_glm = glm.predict(final_test)
#predicted_glm.shape
# adding new column to the sample csv file
#new_sample['UltimateIncurredClaimCost']= predict_glm
#new_sample['UltimateIncurredClaimCost']= predict_glm
#new_sample.to_csv("NewSampleSubmission.csv", index = False)
#new_sample.head()  # dataframe of prediction using GLM
"""
## 8. Model Used: Random Forest
"""
# Instantiate model with 3000 decision trees
RF = RandomForestRegressor(n_estimators = 1000, random_state = 50)


# Train the model on training data
RF.fit(x_train, y_train);
# Use the forest's predict method on the test data

predictions = RF.predict(x_test)

# Calculate the absolute errors
errors = abs(predictions - y_test)
errors
predictions = RF.predict(final_test)
# adding new column to sample csv
sample1['UltimateIncurredClaimCost']= predictions
sample1.head()
sample1.to_csv("SampleSubmission.csv", index = False)  #saving the modified dataframe as a csv file
"""
## 9. Reason for selecting the model

* KNN was a primitive model and it required lesser time.
* The other two models were tried out to find out if there was any change in the final rsme score.
"""
"""
## 10. Conclusion<br><br>
Out of all the models used, the rsme score for KNN was the least.
"""