"""
## This is preprocess for what stacking model based on 8 model about Gradient Boost and Linear Regression

### You can use the ouput files submission_{name}.csv or training_{name}.csv
### submission_{name}.csv ・・・ A submission file predicted using the {name} algorithm.
### training_{name}.csv ・・・ A training file predicted using the {name} algorithm.

Stacking code is https://www.kaggle.com/ngo1013/svr-stacking/
"""
import sklearn
print(sklearn.__version__) # 0.17.1

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)|
loading = pd.read_csv('/kaggle/input/trends-assessment-prediction/loading.csv') # タンパク質の濃度マップ
fnc = pd.read_csv("/kaggle/input/trends-assessment-prediction/fnc.csv") # functional network connectivity (脳の接続性の指標らしい)
score = pd.read_csv("/kaggle/input/trends-assessment-prediction/train_scores.csv")
score
fnc_features, loading_features = list(fnc.columns[1:]), list(loading.columns[1:])# コラムを抽出 (idを除外)
df = fnc.merge(loading, on="Id")
score["is_train"] = True
df = df.merge(score, on="Id", how="left")
test_df = df[df["is_train"] != True].copy()
df = df[df["is_train"] == True].copy()
FNC_SCALE = 1/400

df[fnc_features] *= FNC_SCALE
test_df[fnc_features] *= FNC_SCALE
features = loading_features + fnc_features
from sklearn.model_selection import KFold

def metric(y_true, y_pred):
    return np.mean(np.sum(np.abs(y_true - y_pred), axis=0)/np.sum(y_true, axis=0))
"""
# LightGBM
"""
import lightgbm
from lightgbm import LGBMRegressor
lgb_params = {
        'max_depth': -1,
        'num_leaves': 3,
        'min_data_in_leaf': 13,
    }
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)

overal_score = 0
for target, c, w in [("age", 100, 0.3), ("domain1_var1", 10, 0.175), ("domain1_var2", 10, 0.175), ("domain2_var1", 10, 0.175), ("domain2_var2", 10, 0.175)]:    
    y_oof = np.zeros(df.shape[0])
    y_test = np.zeros((test_df.shape[0], NUM_FOLDS))
    
    for f, (train_ind, val_ind) in enumerate(kf.split(df, df)):
        train_df, val_df = df.iloc[train_ind], df.iloc[val_ind] # train, val split
        train_df = train_df[train_df[target].notnull()] # null排除

        model = LGBMRegressor(**lgb_params) # create lightGBM model
        model.fit(train_df[features], train_df[target]) #dfからfeature, targetを指定してmodelをtrain

        y_oof[val_ind] = model.predict(val_df[features]) #Out of fold, クロスバリデーションの外のデータを集める
        y_test[:, f] = model.predict(test_df[features])
                
    df["pred_{}".format(target)] = y_oof
    test_df[target] = y_test.mean(axis=1)
    score = metric(df[df[target].notnull()][target].values, df[df[target].notnull()]["pred_{}".format(target)].values)
    overal_score += w*score
    print(target, np.round(score, 4))
    print()
    
print("Overal score:", np.round(overal_score, 4))
t_df = df[["Id", "pred_age", "pred_domain1_var1", "pred_domain1_var2", "pred_domain2_var1", "pred_domain2_var2"]]
t_df = t_df.sort_values("Id")
t_df.head(1)
t_df.to_csv("training_lightgbm.csv", index=False)
sub_df = test_df[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]]
sub_df = sub_df.sort_values("Id")
sub_df.to_csv("submission_lightgbm.csv", index=False)
"""
# XGBoost
"""
import xgboost
from xgboost import XGBRegressor
xgb_params = {
        'objective': 'reg:squarederror',
        "learning_rate": 0.1,
        'max_depth': 3,
        'n_estimators': 200,
        'subsample': 0.5,
        'colsample_bytree': 0.5
    }
%%time
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)

overal_score = 0
for target, c, w in [("age", 100, 0.3), ("domain1_var1", 10, 0.175), ("domain1_var2", 10, 0.175), ("domain2_var1", 10, 0.175), ("domain2_var2", 10, 0.175)]:    
    y_oof = np.zeros(df.shape[0])
    y_test = np.zeros((test_df.shape[0], NUM_FOLDS))
    
    for f, (train_ind, val_ind) in enumerate(kf.split(df, df)):
        train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
        train_df = train_df[train_df[target].notnull()]

        model = XGBRegressor(**xgb_params)
        model.fit(train_df[features], train_df[target])

        y_oof[val_ind] = model.predict(val_df[features])
        y_test[:, f] = model.predict(test_df[features])
        
    df["pred_{}".format(target)] = y_oof
    test_df[target] = y_test.mean(axis=1)
    score = metric(df[df[target].notnull()][target].values, df[df[target].notnull()]["pred_{}".format(target)].values)
    overal_score += w*score
    print(target, np.round(score, 4))
    print()
    
print("Overal score:", np.round(overal_score, 4))
t_df = df[["Id", "pred_age", "pred_domain1_var1", "pred_domain1_var2", "pred_domain2_var1", "pred_domain2_var2"]]
t_df = t_df.sort_values("Id")
t_df.to_csv("training_xgboost.csv", index=False)
sub_df = test_df[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]]
sub_df = sub_df.sort_values("Id")
sub_df.to_csv("submission_xgboost.csv", index=False)
"""
# Lasso
"""
from sklearn.linear_model import Lasso
%%time
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)

overal_score = 0
for target, c, w in [("age", 100, 0.3), ("domain1_var1", 10, 0.175), ("domain1_var2", 10, 0.175), ("domain2_var1", 10, 0.175), ("domain2_var2", 10, 0.175)]:    
    y_oof = np.zeros(df.shape[0])
    y_test = np.zeros((test_df.shape[0], NUM_FOLDS))
    
    for f, (train_ind, val_ind) in enumerate(kf.split(df, df)):
        train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
        train_df = train_df[train_df[target].notnull()]

        model = Lasso(alpha=0.1)
        model.fit(train_df[features], train_df[target])

        y_oof[val_ind] = model.predict(val_df[features])
        y_test[:, f] = model.predict(test_df[features])
        
    df["pred_{}".format(target)] = y_oof
    test_df[target] = y_test.mean(axis=1)
    score = metric(df[df[target].notnull()][target].values, df[df[target].notnull()]["pred_{}".format(target)].values)
    overal_score += w*score
    print(target, np.round(score, 4))
    print()
    
print("Overal score:", np.round(overal_score, 4))
t_df = df[["Id", "pred_age", "pred_domain1_var1", "pred_domain1_var2", "pred_domain2_var1", "pred_domain2_var2"]]
t_df = t_df.sort_values("Id")
t_df.to_csv("training_lasso.csv", index=False)
sub_df = test_df[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]]
sub_df = sub_df.sort_values("Id")
sub_df.to_csv("submission_lasso.csv", index=False)
"""
# Ridge
"""
from sklearn.linear_model import BayesianRidge
%%time
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)

overal_score = 0
for target, c, w in [("age", 100, 0.3), ("domain1_var1", 10, 0.175), ("domain1_var2", 10, 0.175), ("domain2_var1", 10, 0.175), ("domain2_var2", 10, 0.175)]:    
    y_oof = np.zeros(df.shape[0])
    y_test = np.zeros((test_df.shape[0], NUM_FOLDS))
    
    for f, (train_ind, val_ind) in enumerate(kf.split(df, df)):
        train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
        train_df = train_df[train_df[target].notnull()]

        model = BayesianRidge()
        model.fit(train_df[features], train_df[target])

        y_oof[val_ind] = model.predict(val_df[features])
        y_test[:, f] = model.predict(test_df[features])
        
    df["pred_{}".format(target)] = y_oof
    test_df[target] = y_test.mean(axis=1)
    score = metric(df[df[target].notnull()][target].values, df[df[target].notnull()]["pred_{}".format(target)].values)
    overal_score += w*score
    print(target, np.round(score, 4))
    print()
    
print("Overal score:", np.round(overal_score, 4))
t_df = df[["Id", "pred_age", "pred_domain1_var1", "pred_domain1_var2", "pred_domain2_var1", "pred_domain2_var2"]]
t_df = t_df.sort_values("Id")
t_df.to_csv("training_ridge.csv", index=False)
sub_df = test_df[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]]
sub_df = sub_df.sort_values("Id")
sub_df.to_csv("submission_ridge.csv", index=False)
"""
# ElasticNet
"""
from sklearn.linear_model import ElasticNet
%%time
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)

overal_score = 0
for target, c, w in [("age", 100, 0.3), ("domain1_var1", 10, 0.175), ("domain1_var2", 10, 0.175), ("domain2_var1", 10, 0.175), ("domain2_var2", 10, 0.175)]:    
    y_oof = np.zeros(df.shape[0])
    y_test = np.zeros((test_df.shape[0], NUM_FOLDS))
    
    for f, (train_ind, val_ind) in enumerate(kf.split(df, df)):
        train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
        train_df = train_df[train_df[target].notnull()]

        model = ElasticNet()
        model.fit(train_df[features], train_df[target])

        y_oof[val_ind] = model.predict(val_df[features])
        y_test[:, f] = model.predict(test_df[features])
        
    df["pred_{}".format(target)] = y_oof
    test_df[target] = y_test.mean(axis=1)
    score = metric(df[df[target].notnull()][target].values, df[df[target].notnull()]["pred_{}".format(target)].values)
    overal_score += w*score
    print(target, np.round(score, 4))
    print()
    
print("Overal score:", np.round(overal_score, 4))
t_df = df[["Id", "pred_age", "pred_domain1_var1", "pred_domain1_var2", "pred_domain2_var1", "pred_domain2_var2"]]
t_df = t_df.sort_values("Id")
t_df.to_csv("training_elastic.csv", index=False)
sub_df = test_df[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]]
sub_df = sub_df.sort_values("Id")
sub_df.to_csv("submission_elastic.csv", index=False)
"""
# TweedieRegressor
"""
from sklearn.linear_model import TweedieRegressor
%%time
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)

overal_score = 0
for target, c, w in [("age", 100, 0.3), ("domain1_var1", 10, 0.175), ("domain1_var2", 10, 0.175), ("domain2_var1", 10, 0.175), ("domain2_var2", 10, 0.175)]:    
    y_oof = np.zeros(df.shape[0])
    y_test = np.zeros((test_df.shape[0], NUM_FOLDS))
    
    for f, (train_ind, val_ind) in enumerate(kf.split(df, df)):
        train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
        train_df = train_df[train_df[target].notnull()]

        model = TweedieRegressor()
        model.fit(train_df[features], train_df[target])

        y_oof[val_ind] = model.predict(val_df[features])
        y_test[:, f] = model.predict(test_df[features])
        
    df["pred_{}".format(target)] = y_oof
    test_df[target] = y_test.mean(axis=1)
    score = metric(df[df[target].notnull()][target].values, df[df[target].notnull()]["pred_{}".format(target)].values)
    overal_score += w*score
    print(target, np.round(score, 4))
    print()
    
print("Overal score:", np.round(overal_score, 4))
t_df = df[["Id", "pred_age", "pred_domain1_var1", "pred_domain1_var2", "pred_domain2_var1", "pred_domain2_var2"]]
t_df = t_df.sort_values("Id")
t_df.to_csv("training_tweedie.csv", index=False)
sub_df = test_df[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]]
sub_df = sub_df.sort_values("Id")
sub_df.to_csv("submission_tweedie.csv", index=False)
"""
# Linear Regression
"""
from sklearn.linear_model import LinearRegression
%%time
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)

overal_score = 0
for target, c, w in [("age", 100, 0.3), ("domain1_var1", 10, 0.175), ("domain1_var2", 10, 0.175), ("domain2_var1", 10, 0.175), ("domain2_var2", 10, 0.175)]:    
    y_oof = np.zeros(df.shape[0])
    y_test = np.zeros((test_df.shape[0], NUM_FOLDS))
    
    for f, (train_ind, val_ind) in enumerate(kf.split(df, df)):
        train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
        train_df = train_df[train_df[target].notnull()]

        model = LinearRegression()
        model.fit(train_df[features], train_df[target])

        y_oof[val_ind] = model.predict(val_df[features])
        y_test[:, f] = model.predict(test_df[features])
        
    df["pred_{}".format(target)] = y_oof
    test_df[target] = y_test.mean(axis=1)
    score = metric(df[df[target].notnull()][target].values, df[df[target].notnull()]["pred_{}".format(target)].values)
    overal_score += w*score
    print(target, np.round(score, 4))
    print()
    
print("Overal score:", np.round(overal_score, 4))
t_df = df[["Id", "pred_age", "pred_domain1_var1", "pred_domain1_var2", "pred_domain2_var1", "pred_domain2_var2"]]
t_df = t_df.sort_values("Id")
t_df.to_csv("training_linear.csv", index=False)
sub_df = test_df[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]]
sub_df = sub_df.sort_values("Id")
sub_df.to_csv("submission_linear.csv", index=False)
"""
# SVM
"""
from sklearn.svm import SVR
NUM_FOLDS = 4
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)

overal_score = 0
for target, c, w in [("age", 100, 0.3), ("domain1_var1", 10, 0.175), ("domain1_var2", 10, 0.175), ("domain2_var1", 10, 0.175), ("domain2_var2", 10, 0.175)]:    
    y_oof = np.zeros(df.shape[0])
    y_test = np.zeros((test_df.shape[0], NUM_FOLDS))
    
    for f, (train_ind, val_ind) in enumerate(kf.split(df, df)):
        train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
        train_df = train_df[train_df[target].notnull()]

        model = SVR(C=c, cache_size=3000.0)
        model.fit(train_df[features], train_df[target])

        y_oof[val_ind] = model.predict(val_df[features])
        y_test[:, f] = model.predict(test_df[features])
        
    df["pred_{}".format(target)] = y_oof
    test_df[target] = y_test.mean(axis=1)
    
    score = metric(df[df[target].notnull()][target].values, df[df[target].notnull()]["pred_{}".format(target)].values)
    overal_score += w*score
    print(target, np.round(score, 4))
    print()
    
print("Overal score:", np.round(overal_score, 4))
t_df = df[["Id", "pred_age", "pred_domain1_var1", "pred_domain1_var2", "pred_domain2_var1", "pred_domain2_var2"]]
t_df = t_df.sort_values("Id")
t_df.to_csv("training_svr.csv", index=False)
sub_df = test_df[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]]
sub_df = sub_df.sort_values("Id")
sub_df.to_csv("submission_svr.csv", index=False)