# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import numpy as np
import pandas as pd
from sklearn.metrics import balanced_accuracy_score as bas
import gc
gc.enable()

from sklearn.linear_model import LogisticRegression, SGDClassifier

from os import listdir
from os.path import isfile, join
# -

data = pd.read_csv('../data-reduced-800-v3-shuffled.csv', index_col = 0)
testfile = pd.read_csv('../test.csv')
catcode = pd.read_csv('../data-simplified-1-catcode.csv', header = None, names = ['category'])['category']

sp = int(len(data) * 0.8) # Split Point
sp2 = len(data)
y_train = data.category.values[:sp]
y_test = data.category.values[sp:sp2]
rel = data.label_quality.values[sp:sp2]

# ## Val

DIR = '../ensemb3/'

from os import listdir
from os.path import isfile, join
val = {f[4:-4] : f for f in listdir(DIR) if ((isfile(join(DIR, f))) and f[0] == 'v')}

val

# %%time
for k, v in val.items():
    print(k)
    val[k] = pd.read_csv(DIR+v, dtype = np.float32, header = None, index_col = False)

classes = pd.Series(range(1588))
out = classes.loc[~classes.isin(pd.Series(y_test[rel == 0]))]
outsiders = pd.Series(y_test).isin(out)

num_rels = (rel == 0).sum()
num_rels, len(rel)

dfs = sorted(val.keys() - {})

X = np.zeros((len(y_test) * 1588, len(dfs)), dtype = np.float32)
y = np.zeros((len(X)), dtype = np.int32)

val_len = len(y_test)

for k, v in val.items():
    print(k, v.shape)

# +
# # %%time
mask = rel == 0
for i in range(1588):
    for d, df in enumerate(dfs):
        X[i * val_len : (i + 1) * val_len, d] = val[df].values[:, i]
        y[i * val_len : (i + 1) * val_len] = 1 * (y_test == i)
        
#         X[i * val_len : (i + 1) * val_len, d] = val[df].values[mask, i]
#         y[i * val_len : (i + 1) * val_len] = 1 * (y_test[mask] == i)
# -

ix = np.random.randint(0, len(X), len(X) // 100) # better than shuffle arange ??? how?

# %%time
est = SGDClassifier(loss = 'modified_huber', max_iter=10, tol =  1e-5, penalty='l2', alpha = 1e-4)
est.fit(X[ix], y[ix]) # consider sample weight?

# %%time
# 0.916317 (10)
new = sum(val[df].values[mask] * coef for df, coef in zip(dfs, est.coef_.flatten())) / est.coef_.sum()
val_pred = new.argmax(axis = 1)
print(bas(y_test[rel == 0], val_pred))

# 0.9170534864774611
for df, coef in zip(dfs, est.coef_.flatten()):
    print(f"'{df}' : {coef},")



# These values were obtained in the cell above
coefval = {
'lstm_char_unproc' : 0.15214980219793464,
'lstm_word' : 0.5057028723619695,
'mnb' : 0.45090371936039003,
'sgd_char-v7' : 0.32103481728540106,
'sgd_char-v7_45' : 0.35221132620470513,
'sgd_word-v7' : 0.39905860580347985,
}



# %%time
ensemble_val = sum(val[df] * coef for df, coef in coefval.items()) / sum(coefval.values())

# %%time
ensemble_val.to_csv('../ensens/val_ensemble11.csv', index = False, header = False)

# ## Test

DIR = '../ensemb3/'

test = {f[5:-4] : f for f in listdir(DIR) if ((isfile(join(DIR, f))) and f[0] == 't')}

test, len(test)

# %%time
for k, v in test.items():
    print(k)
    test[k] = pd.read_csv(DIR+v, header = None, index_col = False)

# These values were obtained in the section above
coeftest = {
'lstm_char_unproc' : 0.15214980219793464,
'lstm_word' : 0.5057028723619695,
'mnb' : 0.45090371936039003,
'sgd_char-v7' : 0.32103481728540106,
'sgd_char-v7_45' : 0.35221132620470513,
'sgd_word-v7' : 0.39905860580347985,
}



# %%time
ensemble = sum(test[df] * coef for df, coef in coeftest.items()) / sum(coeftest.values())

sample = pd.read_csv('../sample_submission.csv')

# %%time
sample['category'] = np.vectorize(catcode.get)(ensemble.values.argmax(axis=1))

sample.head()

sample.to_csv('../submission_ensemble11.csv', index = False)
sample.shape

pd.read_csv('../submission_ensemble11.csv').head()

# %%time
ensemble.to_csv('../ensens/test_ensemble11.csv', index = False, header = False)

StarcoderdataJupyterScriptsDedupFiltered