# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# %matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from tqdm import tnrange, tqdm_notebook
import gc
import operator

sns.set_context('talk')

pd.set_option('display.max_columns', 500)

import warnings
warnings.filterwarnings('ignore', message='Changing the shape of non-C contiguous array')

# # Read the data

from os import listdir
from os.path import isfile, join
stacking_dir = 'stacking'
results_dir = 'results'
stacking_files = [f for f in listdir(stacking_dir) if isfile(join(stacking_dir, f))]

stacking_files

dfXtrain = pd.concat([pd.read_csv(join(stacking_dir, f), index_col='ID', sep=';') for f in stacking_files], axis=1)
dfXtest = pd.concat([pd.read_csv(join(results_dir, f), index_col='ID', sep=';') for f in stacking_files], axis=1)
dfXtest.columns = dfXtrain.columns
dfYtrain = pd.read_csv('preprocessed_csv/y_train_tree.csv', header=None, names=['ID', 'COTIS'], sep=';')

dfYtrain = dfYtrain.set_index('ID')

department = pd.read_csv('preprocessed_csv/train_tree.csv', index_col='id', sep=';')['department']

# # Preprocessing

y_train = np.array(dfYtrain)
x_train = np.array(dfXtrain)
x_test = np.array(dfXtest)

# # Save routines

model_name = 'COTIS'
dfYtest = pd.DataFrame({'ID': dfXtest.index, model_name: np.zeros(dfXtest.shape[0])})
dfYtest = dfYtest[['ID', model_name]]
dfYtest.head()


def save_to_file(y, file_name):
    dfYtest[model_name] = y
    dfYtest.to_csv('results/{}'.format(file_name), index=False, sep=';')


# # Train XGB

from sklearn.cross_validation import LabelKFold


def plot_quality(grid_searcher, param_name):
    means = []
    stds = []
    for elem in grid_searcher.grid_scores_:
        means.append(np.mean(elem.cv_validation_scores))
        stds.append(np.sqrt(np.var(elem.cv_validation_scores)))
    means = np.array(means)
    stds = np.array(stds)
    
    params = grid_searcher.param_grid
    
    plt.figure(figsize=(10, 6))
    plt.plot(params[param_name], means)
    plt.fill_between(params[param_name], \
                 means + stds, means - stds, alpha = 0.3, facecolor='blue')
    plt.xlabel(param_name)
    plt.ylabel('MAPE')


def mape(y_true, y_pred): 
    return -np.mean(np.abs((y_true - y_pred) / y_true)) * 100


def mape_scorer(est, X, y):
    gc.collect()
    return mape(y, est.predict(X))


class MyGS():
    class Element():
        def __init__(self):
            self.cv_validation_scores = []
        
        
        def add(self, score):
            self.cv_validation_scores.append(score)
    
    
    def __init__(self, param_grid, name, n_folds):
        self.param_grid = {name: param_grid}
        self.grid_scores_ = [MyGS.Element() for item in param_grid]
        
    
    def add(self, score, param_num):
        self.grid_scores_[param_num].add(score)
        

def amse(preds, dtrain, alpha=1, beta=1, gamma=1, divider=1):
    labels = dtrain.get_label()
    labels2 = np.power(labels, 2)
    grad = np.where(preds > labels, 
                    alpha * (preds - labels) / labels2 - gamma, 
                    alpha * (preds - labels) / (labels2 * divider) + gamma)
    second_grad = beta * np.where(preds > labels, 1, 1 / divider) / labels2
    return grad, second_grad


def amse_fixed(alpha, beta, gamma, divider):
    def wrapper(*args, **kwargs):
        return amse(*args, alpha=alpha, beta=beta, gamma=gamma, divider=divider, **kwargs)
    return wrapper


# Подберём веса

# +
# %%time

n_folds = 10
label_kfold = LabelKFold(np.array(department), n_folds=n_folds)
weight_list = [(alpha, beta, 1 - alpha - beta) 
               for alpha in np.arange(0.025, 1, 0.025)
               for beta in np.arange(0.025, 1 - alpha, 0.025) 
               ]
scores = [-100 for i in range(n_folds)]
train_scores = [-100 for i in range(n_folds)]
weights = [0 for i in range(n_folds)]

for fold_index, (train_index, test_index) in enumerate(label_kfold):
    X = x_train[train_index]
    y = y_train[train_index][:, 0]
    X_validation = x_train[test_index]
    y_validation = y_train[test_index][:, 0]
    good_score = -np.Inf
    good_weight = None
    for weight in weight_list:
        wX = X / weight
        y_pred = X.flatten()[np.argmin(wX, axis=1) + np.arange(X.shape[0]) * X.shape[1]]
        #y_pred = np.sum(X * weight, axis=1)
        score = mape(y, y_pred)
        if score > good_score:
            good_score = score
            good_weight = weight
    wX = X_validation / good_weight
    y_pred = X_validation.flatten()[np.argmin(wX, axis=1) + np.arange(X_validation.shape[0]) * X_validation.shape[1]]
    #y_pred = np.sum(X_validation * good_weight, axis=1)
    scores[fold_index] = mape(y_validation, y_pred)
    train_scores[fold_index] = good_score
    weights[fold_index] = good_weight   
# -

plt.plot(train_scores)
mean = np.mean(train_scores)
plt.plot([mean for i in train_scores])

plt.plot(scores)
mean = np.mean(scores)
plt.plot([mean for i in scores])

weights

# С учётом того, что при разной величине шага веса получаются чуть ли не абсолютно разными, а скор не меняется, можно сделать вывод, что с точки зрения метрии нет особой разницы между ответами

# Вот график хорошести таргета с точки зрения метрики. 

y_dist = dfYtrain.groupby(dfYtrain.COTIS // 10 * 10).size()
y_dist = y_dist / y_dist.index
y_dist.plot()

# А вот и самое хорошее нулевое приближение

y_dist.argmax()

y_zero = 260

# Добавим четвёртый столбец с нулевым приближеним

dfXtrain['y_zero'] = y_zero
dfXtest['y_zero'] = y_zero

y_train = np.array(dfYtrain)
x_train = np.array(dfXtrain)
x_test = np.array(dfXtest)

# +
# %%time

n_folds = 10
label_kfold = LabelKFold(np.array(department), n_folds=n_folds)
weight_list = [(alpha, beta, gamma, 1 - alpha - beta - gamma) 
               for alpha in np.arange(0.05, 1, 0.05)
               for beta in np.arange(0.05, 1 - alpha, 0.05)
               for gamma in np.arange(0.05, 1 - alpha - beta, 0.05)
               ]
scores = [-100 for i in range(n_folds)]
train_scores = [-100 for i in range(n_folds)]
weights = [0 for i in range(n_folds)]

for fold_index, (train_index, test_index) in enumerate(label_kfold):
    X = x_train[train_index]
    y = y_train[train_index][:, 0]
    X_validation = x_train[test_index]
    y_validation = y_train[test_index][:, 0]
    good_score = -np.Inf
    good_weight = None
    for weight in weight_list:
        wX = X / weight
        y_pred = X.flatten()[np.argmin(wX, axis=1) + np.arange(X.shape[0]) * X.shape[1]]
        #y_pred = np.sum(X * weight, axis=1)
        score = mape(y, y_pred)
        if score > good_score:
            good_score = score
            good_weight = weight
    wX = X_validation / good_weight
    y_pred = X_validation.flatten()[np.argmin(wX, axis=1) + np.arange(X_validation.shape[0]) * X_validation.shape[1]]
    #y_pred = np.sum(X_validation * good_weight, axis=1)
    scores[fold_index] = mape(y_validation, y_pred)
    train_scores[fold_index] = good_score
    weights[fold_index] = good_weight   
# -

plt.plot(train_scores)
mean = np.mean(train_scores)
plt.plot([mean for i in train_scores])

plt.plot(scores)
mean = np.mean(scores)
plt.plot([mean for i in scores])

weights

# Херня в общем

dfXtrain.drop('y_zero', axis=1, inplace=True)
dfXtest.drop('y_zero', axis=1, inplace=True)

y_train = np.array(dfYtrain)
x_train = np.array(dfXtrain)
x_test = np.array(dfXtest)

# Попробуем подобрать, на который стоит домжножать, чтобы скостить скор

weight = (0.37500000000000006, 0.22500000000000001, 0.40000000000000002)

wX = x_train / weight
one_x = x_train.flatten()[np.argmin(wX, axis=1) + np.arange(x_train.shape[0]) * x_train.shape[1]]

onex = one_x.reshape((one_x.shape[0], 1))

# +
# %%time

n_folds = 10
label_kfold = LabelKFold(np.array(department), n_folds=n_folds)
weight_list = [w for w in np.arange(0.8, 1.2, 0.005)
               ]
scores = [-100 for i in range(n_folds)]
train_scores = [-100 for i in range(n_folds)]
weights = [0 for i in range(n_folds)]

for fold_index, (train_index, test_index) in enumerate(label_kfold):
    X = onex[train_index]
    y = y_train[train_index][:, 0]
    X_validation = onex[test_index]
    y_validation = y_train[test_index][:, 0]
    good_score = -np.Inf
    good_weight = None
    for weight in weight_list:
        y_pred = X[:, 0] * weight
        score = mape(y, y_pred)
        if score > good_score:
            good_score = score
            good_weight = weight
    y_pred = X_validation[:, 0] * good_weight
    scores[fold_index] = mape(y_validation, y_pred)
    train_scores[fold_index] = good_score
    weights[fold_index] = good_weight   
# -

plt.plot(train_scores)
mean = np.mean(train_scores)
plt.plot([mean for i in train_scores])

plt.plot(scores)
mean = np.mean(scores)
plt.plot([mean for i in scores])

weights

# Лажа какая-то, эта штука считает, что надо увеличить предсказания, а не уменьшить. То есть при оптимизации квадратичного MAPE мы слишком занизили результаты, что очень странно

# Бля, лидерборд это подтверждает. Взвешенный алгоритм с умножением на 1.01 даёт лучшие результаты, чем просто взвешенный

# Найдём хороший вес для x_train[:, 1]

onex = x_train[:, 1].reshape((x_train.shape[0], 1))

# +
# %%time

n_folds = 10
label_kfold = LabelKFold(np.array(department), n_folds=n_folds)
weight_list = [w for w in np.arange(0.8, 1.2, 0.005)
               ]
scores = [-100 for i in range(n_folds)]
train_scores = [-100 for i in range(n_folds)]
weights = [0 for i in range(n_folds)]

for fold_index, (train_index, test_index) in enumerate(label_kfold):
    X = onex[train_index]
    y = y_train[train_index][:, 0]
    X_validation = onex[test_index]
    y_validation = y_train[test_index][:, 0]
    good_score = -np.Inf
    good_weight = None
    for weight in weight_list:
        y_pred = X[:, 0] * weight
        score = mape(y, y_pred)
        if score > good_score:
            good_score = score
            good_weight = weight
    y_pred = X_validation[:, 0] * good_weight
    scores[fold_index] = mape(y_validation, y_pred)
    train_scores[fold_index] = good_score
    weights[fold_index] = good_weight   
# -

plt.plot(train_scores)
mean = np.mean(train_scores)
plt.plot([mean for i in train_scores])

plt.plot(scores)
mean = np.mean(scores)
plt.plot([mean for i in scores])

weights

# # Save

weight = (0.37500000000000006, 0.22500000000000001, 0.40000000000000002)

wX = x_test / weight
y_pred = 1.01 * x_test.flatten()[np.argmin(wX, axis=1) + np.arange(x_test.shape[0]) * x_test.shape[1]]

save_to_file(y_pred, 'weight_stacking.csv')

wX = x_test / weight
y_pred = x_test.flatten()[np.argmin(wX, axis=1) + np.arange(x_test.shape[0]) * x_test.shape[1]]

save_to_file(y_pred, 'weight_stacking_without_101.csv')

y_pred = x_test[:, 1] * 1.01

save_to_file(y_pred, 'xbg_tune_eta015_num300_with_101.csv')







StarcoderdataJupyterScriptsDedupFiltered