"""
## <center>30 Days of ML competition</center>
### <center>Classify anomalies of target</center>

<center><img src='https://storage.googleapis.com/kaggle-media/Images/30_Days_ML_Hero.png' width='320' height="320" ></center>

This is the final competition of [30 Days of ML program](https://www.kaggle.com/thirty-days-of-ml).

#### Objective:
The main idea is to divide target into 2 groups: anomaly values and normal values. I attempt to pull out something useful to boost regression model and use it later in regression model as feature. Feature engineering is important aspects in Kaggle competitions. 
Then build classification model that predict class in test data and we can use this as feature.

#### What is classification?
Classification is the process of identifying and grouping predetermined categories. In machine learning classification is also the process of predicting the class of given data points.

#### Dataset:
The dataset is used for this competition is synthetic (and generated using a CTGAN), but based on a real dataset. The original dataset deals with predicting the amount of an insurance claim. 
* 'cat0' - 'cat9' categorical features
* 'cont0' - 'cont13' continuous features
* 'target' - continous target

Thanks [@FilterJoe](https://www.kaggle.com/filterjoe/30dtoml-eda-part-2-target-analysis?scriptVersionId=73461234) for great insights from EDA
"""
"""
### Import libraries
"""
import os
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt # plotting
import seaborn as sns # plotting
from sklearn.metrics import mean_squared_error # MSE metric
from sklearn.model_selection import train_test_split
from catboost import CatBoostClassifier
from catboost import Pool
import shap as shap # feature importance plotting
from imblearn.over_sampling import SMOTENC # for Imbalanced Classification
# Metrics
from sklearn.metrics import accuracy_score
from sklearn.metrics import roc_auc_score
from sklearn.metrics import recall_score
from sklearn.metrics import precision_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import roc_curve

pd.set_option('display.max_columns', 100)

SEED = 91 # random seed
"""
# 1. Load data and first look
"""
PATH = '/kaggle/input/30-days-of-ml/' # you can use your own local path

print(f"Files in directory {PATH.split('/')[-2]}:")
for _, _, filenames in os.walk(PATH):
    for filename in filenames:
        print('  '+os.path.join(filename))
"""
Load data
"""
try:
    df_train = pd.read_csv(PATH+'train.csv', index_col=0)
    df_test = pd.read_csv(PATH+'test.csv', index_col=0)
    submission = pd.read_csv(PATH+'sample_submission.csv', index_col=0)
    print('All data has been loaded successfully!')
except Exception as err:
    print(repr(err))
full_lenght_data = len(df_train) + len(df_test)
print(f"train: {len(df_train)} ({100*len(df_train)/full_lenght_data}%)")
print(f"test:  {len(df_test)} ({100*len(df_test)/full_lenght_data}%)")
df_train.info()
df_test.info()
"""
No missing values in dataset
"""
df_train.head()
"""
# 2. Exploratory Data Analysis (EDA)
"""
"""
Use only meaningful features  
(some features are deleted as useless because of feature importance of built regression model)  
See here: https://www.kaggle.com/sergeyzemskov/catboost-feature-importance-with-shap
"""
CAT_FEATURES = ['cat1', 'cat6', 'cat7', 'cat8', 'cat9']
NUM_FEATURES = ['cont0', 'cont1', 'cont2', 'cont3', 'cont4', 'cont5', 'cont6', 'cont7', 'cont8',
                'cont9', 'cont10', 'cont11', 'cont12', 'cont13']
ALL_FEATURES = CAT_FEATURES+NUM_FEATURES
"""
### Categorical features

Compare distribution of features in train and test data
"""
"""
First join train and test data
"""
df_full = pd.concat([df_test[CAT_FEATURES], df_train[CAT_FEATURES]]).sort_index()
df_full['test'] = 'train'
df_full.loc[df_test.index,'test'] = 'test'
df_full.head()
plt.figure(figsize=(20,8))
plt.subplots_adjust(hspace=0.5, wspace=0.3)
#sns.set_palette("Spectral")
for i, col in enumerate(CAT_FEATURES):
    plt.subplot(2, 5, i+1)
    (df_full[col]
     .groupby(df_full['test'])
     .value_counts(normalize=True)
     .rename('proportion %')
     .reset_index()
     .pipe((sns.barplot, "data"), x=col, y='proportion %', hue='test'))
    plt.title(col)
plt.show()
"""
Variables of categorical features on test and train data distributed very similar
"""
"""
### Continuous features
"""
df_train[NUM_FEATURES].describe()
sns.pairplot(df_train[[
    *NUM_FEATURES[:7],
    'target'
]])
sns.pairplot(df_train[[
    *NUM_FEATURES[7:],
    'target'
]])
"""
### Target
"""
df_train['target'].describe()
"""
Quantile 25% is 7.74 but min is 0.14. We have outliers at the bottom.

"""
f,ax=plt.subplots(3,1,figsize=(14,10))
#plt.title('Target values', fontsize=16)
sns.histplot(df_train['target'], ax=ax[0])
sns.boxplot(x=df_train['target'], color='lightblue', saturation=0.8, ax=ax[1])
ax[1].axvline(np.percentile(df_train['target'],.1), label='.1%', c='orange', linestyle=':', linewidth=3)
ax[1].axvline(np.percentile(df_train['target'],.5), label='.5%', c='darkblue', linestyle=':', linewidth=3)
ax[1].legend()
sns.ecdfplot(df_train['target'], ax=ax[2])
ax[0].set_xticks(np.arange(0,10.8,.5))
ax[1].set_xticks(np.arange(0,10.8,.5))
ax[2].set_xticks(np.arange(0,10.8,.5))
plt.show()
print(f"Skew of target {df_train['target'].skew():.3f}")
"""
We observe some peaks and boundaries of target values distribution. Target has a litle right skewed distribution, but we have about some amount of otliers on the left tail (from 0.746555 to 6.9)
"""
"""
Let's look more closely at target without left tail
"""
fig, ax = plt.subplots()
fig.set_size_inches(21, 5)
sns.histplot(df_train[df_train['target'] > 6.9]['target'], bins=500)
ax.set_xticks(np.arange(6.9, max(df_train['target']), .05))
plt.xticks(rotation=90, ha='right')
plt.show()
"""
Target has some anomaly "peaks" and there is some pattern of target distribution
"""
peaks = [8.1184, 8.5241, 8.8411, 9.1089, 9.3723, 9.6729, 10.0079]
step = 0.005
"""
#### Add new features as target at anomaly peak or not
1 - anomaly  
0 - normal
"""
df_train['target_anomaly'] = df_train['target'].apply(lambda x: 1 if (
    [True for p in peaks if (x>(p-step) and x<(p+step))]
)else 0)
df_train[['target', 'target_anomaly']].groupby('target_anomaly').agg(['count', 'mean'])
print(f"Peaks contain {df_train['target_anomaly'].mean()*100:.2f}% of values")
"""
Let's look at values of continuous features if target if in anomalu peak or not
"""
df_train[[*NUM_FEATURES, 'target_anomaly']].groupby('target_anomaly').agg('mean')
df_train[[*NUM_FEATURES, 'target_anomaly']].groupby('target_anomaly').agg('median')
"""
There is some difference of continous features cont0, cont4, cont5, cont7, cont8, cont10, cont11, cont12 if target is in peaks or not
"""
"""
What about pairwise relationships if target is in peaks
"""
df_anomaly = df_train[df_train['target_anomaly'] == 1]
sns.pairplot(df_anomaly[[
    *NUM_FEATURES[:7],
    'target'
]])
sns.pairplot(df_anomaly[[
    *NUM_FEATURES[7:],
    'target'
]])
"""
## Correlation matrix
"""
corrMatrix = df_train.corr(method='pearson', min_periods=1)
plt.figure(figsize=(15,10))
# Generate a mask for the upper triangle
mask = np.triu(np.ones_like(corrMatrix, dtype=bool))
ax = sns.heatmap(corrMatrix, annot=True, mask=mask, cbar_kws={"shrink": .5}, cmap='coolwarm')
plt.show()
"""
There's no significance linear correlation.
"""
"""
# 3. Train Model

Apply upsampling technique using SMOTE library.  
Generate minor class from 5% up to 70% of major class.

_* SMOTE (Synthetic Minority Over-sampling Technique) is a type of over-sampling procedure that is used to correct the imbalances in the groups._
"""
X = df_train[ALL_FEATURES]
y = df_train['target_anomaly']

X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.25, random_state=SEED)

print(np.bincount(y_train.values))
oversample = SMOTENC(categorical_features=[i for i in range(len(CAT_FEATURES))], sampling_strategy=0.7, random_state=SEED)
X_train, y_train = oversample.fit_resample(X_train, y_train)
print(np.bincount(y_train.values))
"""
Additionally we use weights for classes as 1:1.5
"""
catboost_classifier_params = {
    'class_weights': [1,1.5],
    'cat_features': CAT_FEATURES,
    'iterations': 12000,
    'learning_rate': 0.03,
    'early_stopping_rounds': 50,
    'verbose': 2000,
    'random_seed': SEED,
}
model_classifier = CatBoostClassifier(**catboost_classifier_params)
model_classifier.fit(X_train, y_train,
          eval_set=(X_val, y_val))

predicted = model_classifier.predict(X_val)
"""
# 4. Model evaluate
"""
print(f"Accuracy: {accuracy_score(y_val, predicted)}")
print(f"Recall: {recall_score(y_val, predicted)}")
print(f"Precision: {precision_score(y_val, predicted)}")
print(f"ROC-AUC: {roc_auc_score(y_val, predicted)}")
df_cm = pd.DataFrame(confusion_matrix(y_val, predicted), columns=np.unique(y_val), index = np.unique(y_val))
df_cm.index.name = 'Actual'
df_cm.columns.name = 'Predicted'
sns.heatmap(df_cm, cmap="Blues", annot=True, fmt='g')
# Plot ROC curve
def plot_roc_curve(fpr=None, tpr=None):
    """Plot custom histogram"""
    plt.figure(figsize=(5,5))
    plt.title('ROC-curve', fontsize=16)
    plt.xlabel('False Positive Rate', fontsize=14)
    plt.ylabel('True Positive Rate', fontsize=14)
    
    plt.plot(fpr, tpr)

    # ROC-curve of random model
    plt.plot([0, 1], [0, 1], linestyle='--')
    
    plt.ylim([0.0, 1.0])
    plt.xlim([0.0, 1.0])
    plt.grid(True)
    
    plt.show()
    

probabilities_valid = model_classifier.predict_proba(X_val)
fpr, tpr, thresholds = roc_curve(y_val, probabilities_valid[:,1])
plot_roc_curve(fpr, tpr)
"""
<img src=https://static.wikia.nocookie.net/disney/images/a/ad/Profile_-_Sadness.png width="200" height="200" align="left">  


"""
"""
**Bad result! ROC-AUC is near 0.5**  
"""
feature_importance_df = pd.DataFrame(model_classifier.feature_importances_, index=X_train.columns)
feature_importance_df.sort_values(by=0, ascending=False)
train_data = Pool(data=X_train,
                  label=y_train,
                  cat_features=CAT_FEATURES
                 )
                 
explainer = shap.TreeExplainer(model_classifier)
shap_values = explainer.shap_values(train_data)
shap.summary_plot(shap_values, X_train, feature_names=ALL_FEATURES)
"""
#### Feature cont1 stands out among others, it's the most important feature
"""
"""
What is special in 'cont1' feature?
"""
fig, ax = plt.subplots()
fig.set_size_inches(21, 5)
sns.histplot(df_train['cont1'], bins=300)
ax.set_xticks(np.arange(min(df_train['cont1']), max(df_train['cont1']), .01))
plt.xticks(rotation=90, ha='right')
plt.show()
fig, ax = plt.subplots()
fig.set_size_inches(21, 5)
sns.histplot(df_test['cont1'], bins=300)
ax.set_xticks(np.arange(min(df_train['cont1']), max(df_train['cont1']), .01))
plt.xticks(rotation=90, ha='right')
plt.show()
"""
# 5. Conclusion
We can't predict target class after dividing target into 2 anomaly or not.   
But maybe we can get something useful from cont1? Try it here:  
https://www.kaggle.com/sergeyzemskov/xgb-catboost-xgb-lgb-add-cont1-cat
"""
"""
#### _If you find it useful please upvote_
"""