# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import seaborn as sn
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import classification_report, accuracy_score, confusion_matrix
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score, GridSearchCV
from sklearn.preprocessing import MinMaxScaler
from sklearn.ensemble import RandomForestClassifier
import xgboost as xgb


# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
train_data = pd.read_csv('../input/tabular-playground-series-dec-2021/train.csv', index_col='Id')
test_data = pd.read_csv('../input/tabular-playground-series-dec-2021/test.csv', index_col='Id')
train_data.head()
"""
> EDA from https://www.kaggle.com/mhslearner/tps-dec-eda-resnet/notebook
"""
from sklearn.model_selection import train_test_split
train_data = train_data[train_data['Cover_Type'] != 5] # because we have only a single sample labeled 5
y_train = train_data['Cover_Type']
X_train = train_data.drop(['Cover_Type'], axis=1)

# corrMatrix = X_train.corr(method='pearson', min_periods=1)
# corrMatrix.style.background_gradient(axis=None)
cor_targ = train_data.corrwith(train_data["Cover_Type"]).reset_index()
cor_targ.columns =['feature', 'CorrelatioWithTarget']
cor_targ.sort_values('CorrelatioWithTarget',ascending = False).T
X_train.drop(['Soil_Type15', 'Soil_Type7'], axis=1, inplace=True)
ax = plt.figure(figsize=(12, 6))
cover_type= train_data['Cover_Type'].value_counts().sort_index()
print(cover_type)
sn.barplot(x=cover_type.index, y=cover_type,palette="BuPu_r")
plt.show()
plt.figure(figsize=(15,8))
features = train_data.columns.values[0:54]
sn.histplot(train_data[features].mean(axis=1),color="red", kde=True,bins=120, label='train')
sn.histplot(test_data[features].mean(axis=1),color="darkblue", kde=True,bins=120, label='test')
plt.title("Distribution of mean values per row in the train and test data")
plt.legend()
plt.show()
plt.figure(figsize=(15,8))
features = train_data.columns.values[0:54]
sn.distplot(train_data[features].mean(axis=0),color="red", kde=True,bins=120, label='train')
sn.distplot(test_data[features].mean(axis=0),color="darkblue", kde=True,bins=120, label='test')
plt.title("Distribution of mean values per row in the train and test data")
plt.legend()
plt.show()
# num_cols = [
#     "Elevation",
#     "Aspect",
#     "Slope",
#     "Horizontal_Distance_To_Hydrology",
#     "Vertical_Distance_To_Hydrology",
#      "Horizontal_Distance_To_Roadways",
#     "Hillshade_9am",
#     "Hillshade_Noon",
#     "Hillshade_3pm",
#     "Horizontal_Distance_To_Fire_Points",
# ]
# i = 1
# plt.figure()
# fig, ax = plt.subplots(figsize=(18, 15))
# for col in num_cols:
#     plt.subplot(5,3,i)
#     sn.histplot(train_data[col],color="yellow", kde=True,bins=100, label='train')
#     sn.histplot(test_data[col],color="Darkblue", kde=True,bins=100, label='test')
#     i += 1
# plt.legend()
# plt.title(" numirical features Distribution in both train and test data")  
# plt.show()
test_data.drop(['Soil_Type15', 'Soil_Type7'], axis=1, inplace=True)
label_encoder = LabelEncoder()
# X_train = X_train.loc[train.Cover_Type != 5]

X = X_train.astype(np.float32).to_numpy()
y = label_encoder.fit_transform(y_train)
feat_names = X_train.columns
num_class = len(label_encoder.classes_)
X_test = test_data.astype(np.float32).to_numpy()
del test_data

oof_proba = np.zeros((len(y), num_class), dtype=np.float32)
test_proba = np.zeros((len(X_test), num_class), dtype=np.float32)
from sklearn.model_selection import StratifiedKFold
scaler = MinMaxScaler() 
# lm = LogisticRegression(multi_class='ovr', solver='liblinear')
# lm =  RandomForestClassifier(n_estimators=i,random_state=0)
params = {
    'num_class': num_class,
    'objective': 'multi:softprob',
    'tree_method': 'gpu_hist',
    'predictor': 'gpu_predictor',
    'eval_metric': ['merror', 'mlogloss'],
    'learning_rate': .09,
    'max_depth': 0,
    'subsample': .15,
    'sampling_method': 'gradient_based',
    'seed': 64,
    'grow_policy': 'lossguide',
    'max_leaves': 255,
    'lambda': 100,
}
model = xgb.XGBClassifier(params=params, num_boost_round=600, early_stopping_rounds=30,
        verbose_eval=100)
my_pipeline = Pipeline(steps=[('preprocessor', scaler),
                ('model', model)
                ])
param_grid = {
    "model__n_estimators": [20, 50, 100, 200],
    "model__learning_rate": [0.01, 0.1, 0.05],
}



fit_params = {"model__early_stopping_rounds": 10, 
              "model__verbose": False} 

# skf = StratifiedKFold(n_splits=2, random_state=0, shuffle=True)
X_train, X_val, y_train, y_val = train_test_split(X, y, shuffle=True, test_size=0.33, stratify=y)
evalset = [(X_val,y_val)]
model.fit(X_train, y_train, eval_set=evalset, verbose=False)
    
yhat = model.predict(X_val)
score = accuracy_score(y_val, yhat)
print('Accuracy: %.3f' % score)

results = model.evals_result()
# plot learning curves
plt.plot(results['validation_0']['mlogloss'], label='validation')
# show the legend
plt.legend()
# show the plot
plt.show()

## Stratified K-fold - took too long to train so I omitted it.
# for train_index, val_index in skf.split(X, y):
# #     dval = xgboost.DMatrix(X[val_idx], label=y[val_idx])
#     X_train, X_val = X[train_index], X[val_index]
#     y_train, y_val = y[train_index], y[val_index]
    
#     evalset = [(X_val,y_val)]
#     model.fit(X_train, y_train, eval_set=evalset, verbose=False)
    
#     yhat = model.predict(X_val)
#     score = accuracy_score(y_val, yhat)
#     print('Accuracy: %.3f' % score)
#     # retrieve performance metrics
#     results = model.evals_result()
#     # plot learning curves
#     plt.plot(results['validation_0']['mlogloss'], label='train')
#     plt.plot(results['validation_1']['mlogloss'], label='test')
#     # show the legend
#     plt.legend()
# # show the plot
#     plt.show()
    
    

# searchCV = GridSearchCV(my_pipeline, cv=5, param_grid=param_grid,scoring='accuracy')
# searchCV.fit(X_train, y_train)  
# cross_val_scores = cross_val_score(my_pipeline, X_train, y_train,
#                               cv=5,
#                               scoring='accuracy')
# lm.fit(X_train, y_train)
# y_pred = lm.predict(X_valid)
# print(cross_val_scores)
y_pred = model.predict(X_test)
results
df_sub = pd.read_csv('../input/tabular-playground-series-dec-2021/sample_submission.csv', usecols=['Id'])
df_sub['Cover_type'] = pd.Series(label_encoder.inverse_transform(y_pred))
df_sub.to_csv('submission.csv', index=False)

df_sub.tail()
! kaggle competitions submit -c tabular-playground-series-dec-2021 -f ./submission.csv
!kaggle competitions submit -c tabular-playground-series-dec-2021 -f ./submission.csv
!ls -alh ~/.kaggle