# libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
plt.rcParams.update({'figure.max_open_warning': 0})
import seaborn as sns
import sklearn.preprocessing as pre
import sklearn.model_selection as ms

import warnings
warnings.filterwarnings(action="ignore", category=DeprecationWarning)
warnings.simplefilter(action='ignore', category=FutureWarning)

from sklearn.model_selection import train_test_split
"""
### 1.1 Read data
"""
# read training data
train=pd.read_csv('../input/machine-learning-24-hrs-hackathon/train_SJC.csv')
test=pd.read_csv('../input/machine-learning-24-hrs-hackathon/Test_SJC.csv')
# display first rows (transposed)
df=train
df.head(6).T
train.shape , test.shape
# basis statistics: nominal features 
df.describe(include=['object']).T
df = train
df=df.rename(columns={"Unnamed: 0":"ClaimNumber","Unnamed: 1":"DateTimeOfAccident","Unnamed: 3":"Age","Unnamed: 4":"Gender","Unnamed: 5":"MaritalStatus","Unnamed: 6":"DependentChildren","Unnamed: 8":"WeeklyWages","Unnamed: 9":"PartTimeFullTime","Unnamed: 10":"HoursWorkedPerWeek","Unnamed: 12":"ClaimDescription","Unnamed: 13":"InitialIncurredCalimsCost","Unnamed: 14":'UltimateIncurredClaimCost'},inplace=False)
df=df.drop([0,1])
df.head()
df.info()
"""
### 1.2 Count and replace missing values
"""
df['HoursWorkedPerWeek'].fillna(df['HoursWorkedPerWeek'].median(),inplace=True)
df['InitialIncurredCalimsCost'] = pd.to_numeric(df['InitialIncurredCalimsCost'],errors = 'coerce')
df['UltimateIncurredClaimCost'] = pd.to_numeric(df['UltimateIncurredClaimCost'],errors = 'coerce')
df['HoursWorkedPerWeek'] = pd.to_numeric(df['HoursWorkedPerWeek'],errors = 'coerce')
df['Age'] = pd.to_numeric(df['Age'],errors = 'coerce')
df['WeeklyWages'] = pd.to_numeric(df['WeeklyWages'],errors = 'coerce')
df['DaysWorkedPerWeek'] = pd.to_numeric(df['DaysWorkedPerWeek'],errors = 'coerce',downcast='integer')
df['DependentsOther'] = pd.to_numeric(df['DependentsOther'],errors = 'coerce',downcast='integer')

df['DependentChildren'] = pd.to_numeric(df['DependentChildren'],errors = 'coerce')
df.info()
"""
# missing value treatment
"""
df['WeeklyWages'].fillna(df['WeeklyWages'].mean(),inplace=True)
df['MaritalStatus']=df['MaritalStatus'].fillna("s")
df['HoursWorkedPerWeek'].fillna(df['HoursWorkedPerWeek'].median(),inplace=True)
"""
### 1.3 Basic feature engineering
"""

# Some date features
df['YearOfAccident']  = pd.DatetimeIndex(df['DateTimeOfAccident']).year
df['MonthOfAccident']  = pd.DatetimeIndex(df['DateTimeOfAccident']).month
df['DayOfAccident']  = pd.DatetimeIndex(df['DateTimeOfAccident']).day
df['WeekdayOfAccident']  = pd.DatetimeIndex(df['DateTimeOfAccident']).day_name()
df['HourOfAccident']  = pd.DatetimeIndex(df['DateTimeOfAccident']).hour
df['YearReported']  = pd.DatetimeIndex(df['DateReported']).year

# Reporting delay in weeks 
df['DaysReportDelay'] = pd.DatetimeIndex(df['DateReported']).date - pd.DatetimeIndex(df['DateTimeOfAccident']).date
df['DaysReportDelay'] = (df['DaysReportDelay']  / np.timedelta64(1, 'D')).astype(int)
df['WeeksReportDelay'] = np.floor(df['DaysReportDelay'] / 7.).astype(int)
df['WeeksReportDelay'] = np.clip(df['WeeksReportDelay'], a_max=55, a_min=None)

# drop unneccessary columns
df.drop(['ClaimNumber','DateTimeOfAccident','DaysReportDelay','DateReported','WeekdayOfAccident'],axis=1,inplace=True)
df.shape
"""
### 1.4 Skewness and claim costs (transform, analyze)
"""
# Skewness of the numerical feature distributions
print(df.skew())
# The log1p function applies log(1+x) to all elements of the column
df["LogUltimateIncurredClaimCost"] = np.log1p(df["UltimateIncurredClaimCost"])
df["LogInitialIncurredCalimsCost"] = np.log1p(df["InitialIncurredCalimsCost"])

# plot distribution: claim costs (log)
plt.subplots(figsize=(10, 6))
sns.distplot(df.LogUltimateIncurredClaimCost, kde=False, label='Ultimate',bins=100)
sns.distplot(df.LogInitialIncurredCalimsCost, kde=False, label='Initial', bins=100)
plt.xlabel('claim costs (log)')
plt.legend()
plt.show()
# search for frequent initial costs and calculate some statistics for ultimate costs (mean)
df['UltimateIncurredClaimCost'].groupby(df['InitialIncurredCalimsCost']).agg(['mean','median','min','count']).query('count >= 2000')
# Average cost factor Ultimate / Initial
df.UltimateIncurredClaimCost.sum() / df.InitialIncurredCalimsCost.sum()
"""
The average ultimate costs of frequent low initial costs (500,1000,1500,3500) are by a factor of 2 to 3 higher and thus remarkably higher than average (1.40). 

Let's see if there is a pattern in the scatter plot inital vs. ultimate costs:
"""
# Scatter plot: claim costs (log)
plt.subplots(figsize=(10, 7))
sns.scatterplot(data=df, x="LogInitialIncurredCalimsCost",y="LogUltimateIncurredClaimCost")
plt.show()
# Scatter plot: zoom into claim costs (log)
plt.subplots(figsize=(10, 7))
sns.scatterplot(data=df.query('LogInitialIncurredCalimsCost > 6 and LogInitialIncurredCalimsCost < 12 and LogUltimateIncurredClaimCost < 12'), x="LogInitialIncurredCalimsCost",y="LogUltimateIncurredClaimCost")
plt.show()
"""
### 1.5 Visualize numerical features
"""
# Generate a list of numerical variables, remove claim cost variables
num_list = [c for c in df.columns if((df[c].dtype != np.object) and not "Cost" in c)] 
# plot histograms
for name in num_list:
    f, ax = plt.subplots(figsize=(10, 5))
    nbins = min(df[name].value_counts().count(),70)
    plt.hist(data=df, x=name, bins=nbins)
    plt.xlabel(name)
    plt.show()
# List of features with to many different values
num_list_bins =['Age','InitialIncurredCalimsCost','DaysWorkedPerWeek']

# plot binned plot boxplots for 'LogUltimateIncurredClaimCost'
for name in num_list_bins:
    f, ax = plt.subplots(figsize=(14, 5))
    df['bin_'] = pd.cut(df[name], 8)
    sns.boxplot(x='bin_', y='LogUltimateIncurredClaimCost', data=df)
    plt.xlabel(name)
    plt.show()

df.drop(['bin_'],axis=1,inplace=True)
"""
### 1.6 Correlations
"""
# calcuate correlation matrix
corrmat = df.corr()

# Draw the heatmap 
f, ax = plt.subplots(figsize=(12, 9))
sns.heatmap(corrmat, annot=True, square=True, cmap='RdYlGn')
plt.show()
"""
### 1.7 Visualize categorical features
"""
"""
# gender
"""
# Generate a list of categorical variables and remove those with too many different values (e.g. 'ClaimDescription')
# plot distributi
sns.countplot(x="Gender",data=df)
plt.show()
#  plot boxplots for 'LogUltimateIncurredClaimCost'
sns.boxplot(x="Gender", y="LogUltimateIncurredClaimCost", data=df)
plt.show()
"""
# partimefulltime
"""
sns.countplot(x="PartTimeFullTime",data=df)
plt.show()
sns.boxplot(x="PartTimeFullTime", y="LogUltimateIncurredClaimCost", data=df)
plt.show()
le=pre.LabelEncoder()
list_df=['Gender','MaritalStatus','PartTimeFullTime']
for x in list_df:
  df[x]=le.fit_transform(df[x].astype(str))

df.info()
"""
### 1.8 Very basic text processing

Is there predictive power in claim descriptions? Let's get a first impression.
"""
# dispay claim description of a) the most severe claims ...
vars = ['UltimateIncurredClaimCost','ClaimDescription']
df[vars].sort_values(by='UltimateIncurredClaimCost', ascending=False).head(8)
# ... and b) the least severe claims
df[vars].sort_values(by='UltimateIncurredClaimCost', ascending=True).head(8)
# search for some words and create new features
text = ['NECK','BACK','KNEE','FINGER','EYE','STRUCK','HAMMER','LADDER','STAIR','FELT','TRAUMA']
for name in text:
    df['CD_' + name] = np.where( (df['ClaimDescription'].str.find(name) < 0), 0, 1)

# some two or tree word features
df['CD_FOREIGN_BODY'] = np.where( (df['ClaimDescription'].str.find('FOREIGN BODY') < 0), 0, 1)
df['CD_BACK_STRAIN']  = np.where( (df['ClaimDescription'].str.find('BACK STRAIN') < 0), 0, 1)
df['CD_SOFT_TISSUE_'] = np.where( (df['ClaimDescription'].str.find('SOFT TISSUE INJURY') < 0), 0, 1)
df['CD_WORKPLACE_STRESS'] = np.where( (df['ClaimDescription'].str.find('WORKPLACE STRESS') < 0), 0, 1)
df['CD_LOWER_BACK_STRAIN'] = np.where( (df['ClaimDescription'].str.find('LOWER BACK STRAIN') < 0), 0, 1)

# body side, lacerated/laceration:
df['CD_LEFT_RIGHT'] = np.where( ((df['ClaimDescription'].str.find('LEFT') < 0) & (df['ClaimDescription'].str.find('RIGHT') < 0)), 0, 1)
df['CD_LACERAT_'] = np.where( (df['ClaimDescription'].str.find('LACERAT') < 0), 0, 1)
df['UltimateIncurredClaimCost'].groupby(df['CD_LACERAT_']).agg(['count','median','mean'])
"""
The claim descriptions seem to be predictive and should be continued using modern natural language processing techniques (NLP)and PCA.
"""
"""
### 1.9 Treatment of extreme outliers
"""
# Print the most expensive claim amounts
vars = ['UltimateIncurredClaimCost']
print(df[vars].sort_values(by='UltimateIncurredClaimCost', ascending=False).head(8))

df['UltimateIncurredClaimCost'] = np.where(df['UltimateIncurredClaimCost'] > 1000000, 1000000., df['UltimateIncurredClaimCost']) * 1.000
df['UltimateIncurredClaimCost'].mean() 
"""
### 1.10 Define target: UltimateIncurredClaimCost
"""
# To avoid confusion (e.g. log or untransformed), we give the target a new name: loss
df['loss'] = df["UltimateIncurredClaimCost"]
# drop unneccessary columns
df['UltimateIncurredClaimCost'].mean() 

df.info()
le=pre.LabelEncoder()
list_df=['Gender','MaritalStatus','PartTimeFullTime']
for x in list_df:
  df[x]=le.fit_transform(df[x].astype(str))
df.info()
df.shape
"""
<a id="ch2"></a>
## 2.  Data preperation for modeling
"""
"""
### Feature engineering test data


"""
# read test data
test=pd.read_csv('../input/machine-learning-24-hrs-hackathon/Test_SJC.csv')
print('Number of rows and columns:', test.shape)
test.head()
test.isnull().sum()
test['InitialIncurredCalimsCost'] = pd.to_numeric(test['InitialIncurredCalimsCost'],errors = 'coerce')
test['HoursWorkedPerWeek'] = pd.to_numeric(test['HoursWorkedPerWeek'],errors = 'coerce',downcast='integer')
test['WeeklyWages'] = pd.to_numeric(test['WeeklyWages'],errors = 'coerce',downcast='integer')
test['Age'] = pd.to_numeric(test['Age'],errors = 'coerce')
test['DaysWorkedPerWeek'] = pd.to_numeric(test['DaysWorkedPerWeek'],errors = 'coerce',downcast='integer')
test['DependentChildren'] = pd.to_numeric(test['DependentChildren'],errors = 'coerce')
test['DependentsOther'] = pd.to_numeric(test['DependentsOther'],errors = 'coerce',downcast='integer')
le=pre.LabelEncoder()
list_df=['Gender','MaritalStatus','PartTimeFullTime']
for x in list_df:
  df[x]=le.fit_transform(df[x].astype(str))

test['MaritalStatus']=test['MaritalStatus'].fillna("s")
test.info()
le=pre.LabelEncoder()
list_df=['Gender','MaritalStatus','PartTimeFullTime']
for x in list_df:
  test[x]=le.fit_transform(test[x].astype(str))

# Some date features
test['YearOfAccident']  = pd.DatetimeIndex(test['DateTimeOfAccident']).year
test['MonthOfAccident']  = pd.DatetimeIndex(test['DateTimeOfAccident']).month
test['DayOfAccident']  = pd.DatetimeIndex(test['DateTimeOfAccident']).day
test['WeekdayOfAccident']  = pd.DatetimeIndex(test['DateTimeOfAccident']).day_name()
test['HourOfAccident']  = pd.DatetimeIndex(test['DateTimeOfAccident']).hour
test['YearReported']  = pd.DatetimeIndex(test['DateReported']).year

# Reporting delay in weeks 
test['DaysReportDelay'] = pd.DatetimeIndex(test['DateReported']).date - pd.DatetimeIndex(test['DateTimeOfAccident']).date
test['DaysReportDelay'] = (test['DaysReportDelay']  / np.timedelta64(1, 'D')).astype(int)
test['WeeksReportDelay'] = np.floor(test['DaysReportDelay'] / 7.).astype(int)
test['WeeksReportDelay'] = np.clip(test['WeeksReportDelay'], a_max=55, a_min=None)

# drop unneccessary columns
test.drop(['ClaimNumber','DateTimeOfAccident','DaysReportDelay','DateReported','WeekdayOfAccident'],axis=1,inplace=True)
test.shape
# Very basis text processing: claim description features

# create new features for some "cheap" or "expensive" words 
for name in text:
    test['CD_' + name] = np.where( (test['ClaimDescription'].str.find(name) < 0), 0, 1)

# some two or tree word features
test['CD_FOREIGN_BODY'] = np.where( (test['ClaimDescription'].str.find('FOREIGN BODY') < 0), 0, 1)
test['CD_BACK_STRAIN']  = np.where( (test['ClaimDescription'].str.find('BACK STRAIN') < 0), 0, 1)
test['CD_SOFT_TISSUE_'] = np.where( (test['ClaimDescription'].str.find('SOFT TISSUE INJURY') < 0), 0, 1)
test['CD_WORKPLACE_STRESS'] = np.where( (test['ClaimDescription'].str.find('WORKPLACE STRESS') < 0), 0, 1)
test['CD_LOWER_BACK_STRAIN'] = np.where( (test['ClaimDescription'].str.find('LOWER BACK STRAIN') < 0), 0, 1)

# body side, lacerated/laceration:
test['CD_LEFT_RIGHT'] = np.where( ((test['ClaimDescription'].str.find('LEFT') < 0) & (test['ClaimDescription'].str.find('RIGHT') < 0)), 0, 1)
test['CD_LACERAT_'] = np.where( (test['ClaimDescription'].str.find('LACERAT') < 0), 0, 1)
test.info()
le=pre.LabelEncoder()
list_df=['Gender','MaritalStatus','PartTimeFullTime']
for x in list_df:
  test[x]=le.fit_transform(test[x].astype(str))

"""
### 2.2 One-hot-Encoding (nominal features to dummies) 
"""
# combine, drop claim description and get dummies
df_all = pd.concat([df.assign(role="train"), test.assign(role="test")])
df_all.drop(['ClaimDescription'],axis=1,inplace=True)
df_all.drop(['UltimateIncurredClaimCost'],axis=1,inplace=True)
df_all.drop(['LogUltimateIncurredClaimCost'],axis=1,inplace=True)

df_all = pd.get_dummies(df_all)

# seperate
test_dummies, df_dummies = df_all[df_all["role_test"].eq(1)], df_all[df_all["role_train"].eq(1)]
df_dummies.drop(['role_test','role_train'],axis=1,inplace=True)
test_dummies.drop(['role_test','role_train','loss'],axis=1,inplace=True)
df_dummies.shape
"""
### 2.3  Create modeling data sets
"""
seed = 1234

# split data in feature matrix X and label y for training and validation
X = df_dummies.drop(['loss'], axis=1)
y = df_dummies['loss']
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.25, random_state=seed)
X_test = test_dummies
print(X_train.shape)
print(X_val.shape)
print(y_train.shape)
print(y_val.shape)
pip  install xgboost
"""
<a id="ch3"></a>
## 3. Boosting Models, Validation, Scoring

We restrict ourselves here to gradient boosted regression tree models. They do not require scaling and are known to be well suited for contests with tabular data. When treating 'ClaimDescription' with NLP methods, this may change towards artificial neural networks.
"""
!pip install lightgbm
!pip install catboost
# libraries
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import RandomizedSearchCV
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor
import time
#import shap 
"""
### 3.1 CatBoost (with default settings)
"""
# CatBoostRegressor (Default values)
tic = time.time()
CGB = CatBoostRegressor(logging_level='Silent')    
CGB.fit(X_train, y_train)
print("time (sec):" + "%6.0f" % (time.time() - tic))

# Validation MSE
result = np.sqrt(mean_squared_error(y_val, CGB.predict(X_val)))
#result = mean_squared_error(np.expm1(y_val), np.expm1(CGB.predict(X_val)))
print("MSE:" + "%6.2f" % result)
# CatBoost: Plot feature importance
(pd.Series(CGB.feature_importances_, index=X.columns).nlargest(20).plot(kind='barh'))  
plt.show()
"""
### 3.2 lightGBM: Hyperparametertuning
"""
# LGBMRegressor: Hyperparameter tuning with RandomizedSearchCV
tic = time.time()
param_grid ={'learning_rate': [0.02,0.025], 'n_estimators': [500], 'num_leaves': [30,40,50],'feature_fraction': [0.7]} 
LGB_random_search = RandomizedSearchCV(LGBMRegressor(),param_grid, scoring='neg_mean_squared_error', cv=4,  n_iter=5, random_state=seed)
LGB_random_search.fit(X_train,y_train)
print("Best parameters:",LGB_random_search.best_params_)
LGB = LGBMRegressor(**LGB_random_search.best_params_)    
LGB.fit(X_train, y_train)
print("time (sec):" + "%6.0f" % (time.time() - tic))

# Validation MSE
result = np.sqrt(mean_squared_error(y_val, LGB.predict(X_val)))
#result = mean_squared_error(np.expm1(y_val), np.expm1(LGB.predict(X_val)))
print("MSE:" + "%6.2f" % result)
# lightGBM: Plot feature importance
(pd.Series(LGB.feature_importances_, index=X.columns).nlargest(20).plot(kind='barh'))  
plt.show()
"""
### 3.3 XGBoost: Hyperparametertuning
"""
# XGBRegressor: Hyperparameter tuning with RandomizedSearchCV
tic = time.time()
param_grid ={'learning_rate': [0.02,0.025], 'max_depth': [3,4,5],'n_estimators': [100],'colsample_bytree': [0.5], 'subsample': [0.4], 'tree_method': ["hist"] } 
XGB_random_search = RandomizedSearchCV(XGBRegressor(),param_grid, scoring='neg_mean_squared_error', cv=4,  n_iter=5, random_state=seed)
XGB_random_search.fit(X_train,y_train)
print("Best parameters:",XGB_random_search.best_params_)
XGB = XGBRegressor(**XGB_random_search.best_params_)    
XGB.fit(X_train, y_train)
print("time (sec):" + "%6.0f" % (time.time() - tic))

# Validation MSE
result = np.sqrt(mean_squared_error(y_val, XGB.predict(X_val)))
#result = mean_squared_error(np.expm1(y_val), np.expm1(XGB.predict(X_val)))
print("RMSE:" + "%6.2f" % result)
# XGBoost: Plot feature importance
(pd.Series(XGB.feature_importances_, index=X.columns).nlargest(20).plot(kind='barh'))  
plt.show()
from sklearn.linear_model import Ridge
ridge = Ridge()
ridge.fit(X_train,y_train)
print("Train data :",ridge.score(X_train,y_train))
print("Test data :",ridge.score(X_val,y_val))
pred = ridge.predict(X_val)
print(np.sqrt(mean_squared_error(y_val, pred)))
"""
### 3.5 Model blend and scoring
"""
# Validation MSE (Blend of LightGBM and XGBoost)
predictions_val = 0.5*(LGB.predict(X_val)+XGB.predict(X_val))
result = np.sqrt(mean_squared_error(y_val, predictions_val))
print("MSE:" + "%6.2f" % result)
# Scoring: Make predictions on test data and write submission file: LGB + XGB
predictions = XGB.predict(X_test)
df_test_pred = pd.DataFrame({'UltimateIncurredClaimCost':predictions})
df_test_pred.to_csv('submission.csv',index=False)
df_test_pred.head()
sub=pd.read_csv('../input/machine-learning-24-hrs-hackathon/sample_submission.csv')
sub['UltimateIncurredClaimCost'] = 0.5*(LGB.predict(X_test)+XGB.predict(X_test))
sub.to_csv('submission_cgb.csv', index = False)
sub.head(5)
print(np.mean(sub['UltimateIncurredClaimCost']))