"""
## Note:
1. This is just a Keras Starter For This Competition.
2. This kernel is made for getting hands on over ideas of K fold Cross Validation With Neural Networks (Dense Networks in this case)
3. It is risky to use Neural Networks With small datasets , Still this is  just an experiment
4. The code for Neural Network is Taken from " Deep Learning With Python " by Francois Chollet.
5. <b> Please do point out any correction and mistakes if it exists as I am a beginner in Machine Learning I am very much open to learn by Constructive Criticism </b>
"""
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import pandas as pd
import numpy as np
from sklearn.svm import SVC
#from sklearn.multioutput import ClassifierChain
from sklearn.naive_bayes import GaussianNB
#from modlamp.sequences import MixedLibrary
from sklearn.model_selection import train_test_split
from skmultilearn.problem_transform import BinaryRelevance, LabelPowerset, ClassifierChain
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, GradientBoostingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import f1_score
import xgboost as xgb
from sklearn.metrics import log_loss
#from mlxtend.classifier import StackingClassifier
from sklearn.datasets import make_multilabel_classification
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from lightgbm import LGBMClassifier as lgb
# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
train_feat = pd.read_csv("../input/lish-moa/train_features.csv")
train_targ = pd.read_csv("../input/lish-moa/train_targets_scored.csv")
test_feat = pd.read_csv("../input/lish-moa/test_features.csv")
subm = pd.read_csv("../input/lish-moa/sample_submission.csv")
X = np.array(train_feat.loc[:,'g-0':'c-99'])
y = np.array(train_targ.loc[:,"5-alpha_reductase_inhibitor":"wnt_inhibitor"])
X_train , X_test , y_train , y_test = train_test_split(X , y , test_size = 0.20 , random_state = np.random.randint(2))
print((X_train.shape , y_train.shape))
print((X_test.shape , y_test.shape))
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X_train  = scaler.fit_transform(X_train)
X_test = scaler.fit_transform(X_test)
"""
### Model Architecture
1. Three Hidden Layers
2. Activation Units 1024 , 512 and 256 in each Dense Layer
3. Rms Prop Optimizer Used for Finding Params
4. Dropout of 0.2 added between each layers
"""
from keras import models
from keras import layers
from keras.layers import Dropout

def build_model():
    model = models.Sequential()
    model.add(layers.Dense(1024, activation = 'relu' , input_shape = (X_train.shape[1],)))
    model.add(Dropout(0.2))
    model.add(layers.Dense(512 , activation ='relu'))
    model.add(Dropout(0.2))
    model.add(layers.Dense(256 , activation ='relu'))
    model.add(Dropout(0.2))
    model.add(layers.Dense(206 , activation = 'sigmoid'))
    model.compile(optimizer = 'rmsprop' , loss = 'binary_crossentropy')
    return model
import numpy as np
k= 10
num_val_samples = len(X_train) // k 
num_epochs  = 30
all_scores = []

for i in range(k):
    print("Processing Fold #",i)
    val_data =  X_train[i* num_val_samples : (i+1) * num_val_samples]
    val_targets = y_train[i* num_val_samples : (i+1) * num_val_samples]
    
    partial_train_data  = np.concatenate([X_train[:i * num_val_samples] , X_train[(i+1) * num_val_samples :]] , axis = 0)
    partial_train_targets  = np.concatenate([y_train[:i * num_val_samples] , y_train[(i+1) * num_val_samples :]] , axis = 0)
    
    model = build_model()
    
    history = model.fit(partial_train_data , partial_train_targets ,  epochs = num_epochs , batch_size = 64 , verbose = 2)
    y_prob = model.predict_proba(val_data)
    score  = log_loss(val_targets , y_prob)
    
    #score = np.array(score)
    #val_score = 
    all_scores.append(score)
    

print("Scores on All 10 Folds")
print(all_scores)
# Building Final Model
model = build_model()
model.fit(X_train , y_train)
## Evaluation on Holdout
probs = model.predict_proba(X_test)
probs
## Evaluation of Log loss
log_loss(y_test , probs)
probs.shape
# Building Entire Model with full Train Data
X = np.array(train_feat.loc[: , 'g-0':'c-99'])
y = np.array(train_targ.loc[:, "5-alpha_reductase_inhibitor":"wnt_inhibitor"])
model = build_model()
model.fit(X , y ,  epochs = 30 , batch_size = 128)
X_test =  np.array(test_feat.loc[:,'g-0':'c-99'])
X.shape
cols  = [c for c in subm.columns if c not in 'sig_id']
#X_test = test_feat[cols]
X_test = scaler.fit_transform(X_test)
y_prob = model.predict_proba(X_test)
#cols  = [c for c in subm.columns if c not in 'sig_id']
#cols
df_prob = pd.DataFrame(y_prob , columns =cols )
ids = subm['sig_id']
ids_df  = pd.DataFrame(data =ids)
df_final = pd.concat([ids_df ,df_prob],axis =1 )
print(df_final.head())
print("Submission File Shape",subm.shape)
print("Our File Shape" , df_final.shape)
df_final.to_csv("submission.csv", index = False)
