# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory
from sklearn.pipeline import Pipeline
from sklearn.metrics import accuracy_score
from sklearn.ensemble import IsolationForest
import seaborn as sns

from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.impute import SimpleImputer
from sklearn.impute import KNNImputer
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Lasso, Ridge, ElasticNet
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import RidgeClassifier
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.preprocessing import PolynomialFeatures
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import confusion_matrix
from sklearn.metrics import plot_confusion_matrix
from sklearn.metrics import f1_score
from sklearn.ensemble import VotingClassifier
from imblearn.under_sampling import RandomUnderSampler
from imblearn.over_sampling import RandomOverSampler
from sklearn.feature_selection import SelectKBest, chi2
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import RFE
from sklearn.svm import SVR
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import roc_auc_score



import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
train = pd.read_csv('../input/iba-ml1-mid-project/train.csv')
test = pd.read_csv('../input/iba-ml1-mid-project/test.csv')
df = train.replace(',','.', regex=True).astype(float) #changed the type to float as data type one of the columns was object
iso=IsolationForest(contamination=0.10)
imp=SimpleImputer(strategy='mean')
X=df[['age','number_dependent_family_members', 'monthly_income', 'number_of_credit_lines', 'real_estate_loans', 'ratio_debt_payment_to_income','credit_line_utilization','number_of_previous_late_payments_up_to_59_days','number_of_previous_late_payments_up_to_89_days','number_of_previous_late_payments_90_days_or_more']]
y=df[['defaulted_on_loan']]

rus=RandomUnderSampler(random_state=42)
ros=RandomOverSampler(random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8) 

"""
Any train_size ratio yielded same results.
"""
X_rus_train, y_rus_train = rus.fit_resample(X_train, y_train)

"""
Decided to undersample the data since oversampling would take much more time to calculate.
"""
sns.scatterplot(data=X_rus_train, x='age', y='monthly_income') #taking on pair plot to compair it to after outlier removal 

outlier_predXtr = iso.fit_predict(imp.fit_transform(X_rus_train)) #checking outliers in X_train
mask=outlier_predXtr != -1
X_train2, y_train2 = X_rus_train.iloc[mask, :], y_rus_train.iloc[mask] #updating training dataset
sns.scatterplot(data=X_train2, x='age', y='monthly_income') 

"""
We can see that number of outliers decreased previous operation
"""
estimator=SVR(kernel='linear')
numerics = ['age','number_dependent_family_members', 'monthly_income', 'number_of_credit_lines', 'real_estate_loans', 'ratio_debt_payment_to_income','credit_line_utilization','number_of_previous_late_payments_up_to_59_days','number_of_previous_late_payments_up_to_89_days','number_of_previous_late_payments_90_days_or_more']
numeric_transformer = Pipeline(steps=[
    ('impute', KNNImputer()),
    ('scaler', MinMaxScaler())
    ])

column_preprocessing = ColumnTransformer(transformers=[
    ('numeric', numeric_transformer, numerics)
    
    ])

model = Pipeline(steps=[
    ('preprocessing', column_preprocessing),
    ('feat_select', RFE(estimator)),
    ('classification', VotingClassifier(estimators=[
        ('dt', DecisionTreeClassifier()),
        ('knn', KNeighborsClassifier()),
        ('randomforest', RandomForestClassifier())
    ], voting='soft'))
    ])

  
param_space = {
    'preprocessing__numeric__impute__n_neighbors':[3,9],
    'classification__knn__n_neighbors':[3,9],
    'classification__dt__max_depth':[5,7],
    'feat_select__n_features_to_select':[4,7]
    }
gs= GridSearchCV(model, param_space, cv= 15, scoring='roc_auc', refit='precision_score')

gs.fit(X_train2, y_train2.values.ravel() )

y_predict=gs.predict(X_test) #find target values on test dataset
y_predict
"""
Initially I was calculating roc_auc_score by using gs.predict which would show 75% score. Later I realized that the task is to find probabilities of positive and negative cases, so I used gs.predict_proba which showed better results of ~82%
"""
y_predict_probs = gs.predict_proba(X_test) #find probabilities of target values on test dataset
y_predict_probs
plot_confusion_matrix(gs, X_test, y_test) 
positive_probs = y_predict_probs[:, 1]

roc_auc_score(y_test, positive_probs)

newtest=test.drop(['Id'], axis=1).replace(',','.', regex=True).astype(float) #preparing test data
resultsprob=gs.predict_proba(newtest) # .replace(',','.', regex=True).astype(float))
resultsprob.shape #checking the size 
resultsdfprob=pd.DataFrame(resultsprob[:,1], columns = ['Predicted'])
resultsdfprob.index.name = 'Id'
resultsdfprob.index += 1 
resultsdfprob.to_csv('submission proba Aghamir Aghazada.csv')

from sklearn import metrics
from sklearn.metrics import roc_curve
from matplotlib import pyplot
fpr, tpr, thresholds = roc_curve(y_test, positive_probs)

pyplot.plot(fpr, tpr, marker='.', label='model')
pyplot.xlabel('False Positive Rate')
pyplot.ylabel('True Positive Rate')
pyplot.plot([0, 1], [0, 1], linestyle='--', label='No Skill')
pyplot.legend()
pyplot.show()