# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + colab={"base_uri": "https://localhost:8080/", "height": 121} colab_type="code" executionInfo={"elapsed": 28730, "status": "ok", "timestamp": 1577159871829, "user": {"displayName": "\u0410\u043b\u0435\u043a\u0441\u0435\u0439 \u041f\u043e\u0434\u0447\u0435\u0437\u0435\u0440\u0446\u0435\u0432", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mBbUw_CVz8K3RIunXwSoj-hNZ6f2buYD0JYAcB_=s64", "userId": "04087359208169148337"}, "user_tz": -180} id="t4A90cOO7c1V" outputId="0cf2380e-f2c6-4c0e-ce29-ce71ec35536f"
from google.colab import drive
drive.mount('/content/drive')

# + colab={} colab_type="code" id="z853A2LX7czQ"
import os
os.chdir('/content/drive/Shared drives/Тяжелые проекты/ИАД/intro-to-dl-seminars/hw_5_text')

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" executionInfo={"elapsed": 2001, "status": "ok", "timestamp": 1577159872625, "user": {"displayName": "\u0410\u043b\u0435\u043a\u0441\u0435\u0439 \u041f\u043e\u0434\u0447\u0435\u0437\u0435\u0440\u0446\u0435\u0432", "photoUrl": "<KEY>", "userId": "04087359208169148337"}, "user_tz": -180} id="saTsr2pV7hRm" outputId="07ddef08-3f4f-4a58-ee9d-5f3a06a67533"
# %tensorflow_version 2.x

# + _cell_guid="2f9b7a76-8625-443d-811f-8f49781aef81" _uuid="598f965bc881cfe6605d92903b758778d400fa8b" colab={} colab_type="code" id="CFuFMvTB7W5X"
import sys, os, re, csv, codecs, numpy as np, pandas as pd

import tensorflow as tf
import tensorflow.keras as keras
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Bidirectional, GlobalMaxPool1D, GlobalAveragePooling1D, concatenate, Conv1D, LeakyReLU,BatchNormalization, MaxPooling1D, Flatten
from tensorflow.keras.models import Model
from tensorflow.keras import initializers, regularizers, constraints, optimizers, layers
from tensorflow.keras.optimizers import Adamax, Adam
from tqdm import tqdm_notebook

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" executionInfo={"elapsed": 8432, "status": "ok", "timestamp": 1577159881430, "user": {"displayName": "\u0410\u043b\u0435\u043a\u0441\u0435\u0439 \u041f\u043e\u0434\u0447\u0435\u0437\u0435\u0440\u0446\u0435\u0432", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mBbUw_CV<KEY>_=s64", "userId": "04087359208169148337"}, "user_tz": -180} id="Uxm_X04M7hH0" outputId="39cd78e5-4eaa-4f18-a005-e50f4a2220e2"
tf.test.gpu_device_name()

# + [markdown] colab_type="text" id="wGuk08efe0Wm"
# По результатам экспериментов, лучше себя показывает эмбединг, обученный на твиттере

# + _cell_guid="66a6b5fd-93f0-4f95-ad62-3253815059ba" _uuid="729b0f0c2a02c678631b8c072d62ff46146a82ef" colab={} colab_type="code" id="LRlgQIsL7W5l"
EMBEDDING_FILE='glove/glove.twitter.27B.100d.txt'
TRAIN_DATA_FILE='x_train.txt'
TEST_DATA_FILE='x_test.txt'
TRAIN_LABELS='y_train.csv'

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" executionInfo={"elapsed": 20041, "status": "ok", "timestamp": 1577159895516, "user": {"displayName": "\u0410\u043b\u0435\u043a\u0441\u0435\u0439 \u041f\u043e\u0434\u0447\u0435\u0437\u0435\u0440\u0446\u0435\u0432", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mBbUw_CVz8K3RIunXwSoj-hNZ6f2buYD0JYAcB_=s64", "userId": "04087359208169148337"}, "user_tz": -180} id="krM4bAlmJcly" outputId="17d961ba-10db-4484-9ddf-716d9acddaf2"
# !cat {EMBEDDING_FILE} | wc -l

# + _cell_guid="2807a0a5-2220-4af6-92d6-4a7100307de2" _uuid="d365d5f8d9292bb9bf57d21d6186f8b619cbe8c3" colab={} colab_type="code" id="IlWp_zxx7W5s"
embed_size = 100 # Размерность эмбединга
max_features = 50000 # Максимальное количество фич
maxlen = 150 # Максимальная длина фразы в токенах

# + _cell_guid="ac2e165b-1f6e-4e69-8acf-5ad7674fafc3" _uuid="8ab6dad952c65e9afcf16e43c4043179ef288780" colab={"base_uri": "https://localhost:8080/", "height": 104} colab_type="code" executionInfo={"elapsed": 53549, "status": "ok", "timestamp": 1577159931371, "user": {"displayName": "\u0410\u043b\u0435\u043a\u0441\u0435\u0439 \u041f\u043e\u0434\u0447\u0435\u0437\u0435\u0440\u0446\u0435\u0432", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mBbUw_CVz8K3RIunXwSoj-hNZ6f2buYD0JYAcB_=s64", "userId": "04087359208169148337"}, "user_tz": -180} id="9SkUItC47W51" outputId="a1c4678d-0a0a-4409-8094-417a1bf3f7c6"
train = pd.read_csv(TRAIN_DATA_FILE, header=None, names=['text'], sep="ЯЯЯЯЯЯЯЯ")
test = pd.read_csv(TEST_DATA_FILE, header=None, names=['text'], sep="ЯЯЯЯЯЯЯЯ")
y = pd.read_csv(TRAIN_LABELS)['Probability'].values

# + colab={} colab_type="code" id="z-UlywlsTu0l"
list_sentences_train = train["text"].fillna("_na_").values
list_sentences_test = test["text"].fillna("_na_").values

# + _cell_guid="79afc0e9-b5f0-42a2-9257-a72458e91dbb" _uuid="c292c2830522bfe59d281ecac19f3a9415c07155" colab={"base_uri": "https://localhost:8080/", "height": 50} colab_type="code" executionInfo={"elapsed": 173431, "status": "ok", "timestamp": 1577160402716, "user": {"displayName": "\u0410\u043b\u0435\u043a\u0441\u0435\u0439 \u041f\u043e\u0434\u0447\u0435\u0437\u0435\u0440\u0446\u0435\u0432", "photoUrl": "https://lh3.googleusercontent.<KEY>", "userId": "04087359208169148337"}, "user_tz": -180} id="ZRpqTr_n7W59" outputId="0368e46f-008e-4c3e-9ed3-5b368db7bc33"
# %%time
tokenizer = Tokenizer(num_words=max_features)
tokenizer.fit_on_texts(list(list_sentences_train))
list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)
list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
X_train = pad_sequences(list_tokenized_train, maxlen=maxlen)
X_test = pad_sequences(list_tokenized_test, maxlen=maxlen)


# + [markdown] colab_type="text" id="LlhjA2NAJ9B0"
# Как можно заметить, данные в векторах не соответствуют тому формату, который был подготовлен

# + _cell_guid="7d19392b-7750-4a1b-ac30-ed75b8a62d52" _uuid="e9e3b4fa7c4658e0f22dd48cb1a289d9deb745fc" colab={"base_uri": "https://localhost:8080/", "height": 66, "referenced_widgets": ["d8e6b4d9f03d4015ba53e5f196e6e80c", "552b2b34199e4f419a2495f4da49f8e0", "d40c2c2dd3bb4d83b3eaf8b03ecd0ec8", "c5a94fdaa722433caa341d2f51d1c991", "654ebf0991664179a397a4b8bf448c75", "204be243ffcb4f0698659eb86ac54212", "92379b34508442dd9d4ff376e39c44bf", "6eec39b2280a4e1ca99b590857a196d1"]} colab_type="code" executionInfo={"elapsed": 35878, "status": "ok", "timestamp": 1577160438583, "user": {"displayName": "\u0410\u043b\u0435\u043a\u0441\u0435\u0439 \u041f\u043e\u0434\u0447\u0435\u0437\u0435\u0440\u0446\u0435\u0432", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mBbUw_CVz8K3RIunXwSoj-hNZ6f2buYD0JYAcB_=s64", "userId": "04087359208169148337"}, "user_tz": -180} id="OFerHr1y7W6F" outputId="466d8b2d-eedf-439b-f6da-6e250af690f9"
def get_coefs(word,*arr): 
    return word, np.asarray(arr, dtype='float32')
with open(EMBEDDING_FILE) as f:
    embeddings_index = dict(get_coefs(*o.strip().split()) for o in tqdm_notebook(f))

# + colab={} colab_type="code" id="_SefmWhF_ETr"
sh = embeddings_index['a'].shape

# + colab={"base_uri": "https://localhost:8080/", "height": 34} colab_type="code" executionInfo={"elapsed": 279, "status": "ok", "timestamp": 1577160438855, "user": {"displayName": "\u0410\u043b\u0435\u043a\u0441\u0435\u0439 \u041f\u043e\u0434\u0447\u0435\u0437\u0435\u0440\u0446\u0435\u0432", "photoUrl": "<KEY>", "userId": "04087359208169148337"}, "user_tz": -180} id="HcVOwSEM9980" outputId="0cdb2752-309c-4a9d-e3f4-c89e5f768a00"
for k, v in embeddings_index.items():
    if v.shape != sh:
        print(k)
        

# + colab={} colab_type="code" id="unvEYixa-ME4"
del embeddings_index['-0.32053']

# + _cell_guid="4d29d827-377d-4d2f-8582-4a92f9569719" _uuid="96fc33012e7f07a2169a150c61574858d49a561b" colab={"base_uri": "https://localhost:8080/", "height": 87} colab_type="code" executionInfo={"elapsed": 2000, "status": "ok", "timestamp": 1577160440846, "user": {"displayName": "\u0410\u043b\u0435\u043a\u0441\u0435\u0439 \u041f\u043e\u0434\u0447\u0435\u0437\u0435\u0440\u0446\u0435\u0432", "photoUrl": "<KEY>", "userId": "04087359208169148337"}, "user_tz": -180} id="BVr_bz917W6L" outputId="daf6b977-739d-4a0b-c9e3-637798aba142"
all_embs = np.stack(embeddings_index.values())
emb_mean,emb_std = all_embs.mean(), all_embs.std()
emb_mean,emb_std

# + _cell_guid="62acac54-0495-4a26-ab63-2520d05b3e19" _uuid="574c91e270add444a7bc8175440274bdd83b7173" colab={} colab_type="code" id="woInL1nt7W6T"
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))
for word, i in word_index.items():
    if i >= max_features: 
        continue
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None: 
        embedding_matrix[i] = embedding_vector


# + [markdown] colab_type="text" id="_yHP3enynKHs"
# Сводная таблица результата обучения простой CNN на 10% данных (эмбединг, Conv1D, GlobalMaxPooling, Dense).
#
# Чем больше фильтров, тем лучше модель, но размер кернела лучше взять средним.
#
# | val_AUC |   32   |   64   |   128  |   256  |
# |:-------:|:------:|:------:|:------:|:------:|
# |    3    | 0.9678 | 0.9712 | 0.9732 | 0.9754 |
# |    5    | 0.9704 | 0.9720 | 0.9741 | 0.9758 |
# |    6    | 0.9694 | 0.9713 | 0.9738 | 0.9758 |
#

# + [markdown] colab_type="text" id="UAC8ezCytDdT"
# Далее возьмем 2 разноприродные модели, которые показали наилучший результат и объединим их.
#
# Так же была попытка добавить еще 1 вход, основанный на метаданных текста (количество слов, символов, уникальных слов и т.д.), но такая модель показывала неудовлетворительное качество

# + _cell_guid="0d4cb718-7f9a-4eab-acda-8f55b4712439" _uuid="dc51af0bd046e1eccc29111a8e2d77bdf7c60d28" colab={} colab_type="code" id="BcF-TBkO7W6c"
def cnn_layers(inp):
    x = Embedding(max_features, embed_size, input_length=maxlen, weights=[embedding_matrix], trainable = True)(inp)

    x = Conv1D(filters=64, kernel_size=5)(x)
    x = LeakyReLU(0.1)(x)
    x = Conv1D(filters=64, kernel_size=5)(x)
    x = BatchNormalization()(x)
    x = LeakyReLU(0.1)(x)  
    x = MaxPooling1D(padding='same')(x)
    x = Dropout(0.2)(x)

    x = Conv1D(filters=128, kernel_size=5)(x)
    x = LeakyReLU(0.1)(x)
    x = Conv1D(filters=128, kernel_size=5)(x)
    x = BatchNormalization()(x)
    x = LeakyReLU(0.1)(x)  
    x = MaxPooling1D(padding='same')(x)
    x = Dropout(0.2)(x)

    x = Conv1D(filters=256, kernel_size=5)(x)
    x = LeakyReLU(0.1)(x)
    x = Conv1D(filters=256, kernel_size=5)(x)
    x = BatchNormalization()(x)
    x = LeakyReLU(0.1)(x)  
    x = MaxPooling1D(padding='same')(x)
    x = Dropout(0.2)(x)

    x = Flatten()(x)
    x = Dense(512)(x)
    x = BatchNormalization()(x)
    x = LeakyReLU(0.1)(x)  
    return x


# + colab={} colab_type="code" id="9vRBoNL99P3h"
def lstm_layers(inp):
    x = Embedding(max_features, embed_size, input_length=maxlen, weights=[embedding_matrix], trainable = True)(inp)
    x = Bidirectional(LSTM(80, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)
    gmx = GlobalMaxPool1D()(x)
    gax = GlobalAveragePooling1D()(x)
    concatenated = concatenate([gmx, gax])
    x = Dense(80, activation="relu")(concatenated)
    return x


# + colab={} colab_type="code" id="4VqEN9Ku9ONX"
input_text = Input(shape=(maxlen,))

cnn = cnn_layers(input_text)
lstm = lstm_layers(input_text)
concatenated = concatenate([cnn, lstm])

x = Dropout(0.1)(concatenated)
x = Dense(1, activation="sigmoid")(x)
model = Model(inputs=input_text, outputs=x)
model.compile(loss='binary_crossentropy', optimizer=Adam(clipvalue=2, clipnorm=2), metrics=['accuracy' ,'AUC'])

# + colab={} colab_type="code" id="PMzlbCxtDi5i"
model=keras.models.load_model()

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} colab_type="code" executionInfo={"elapsed": 8598, "status": "ok", "timestamp": 1577134285731, "user": {"displayName": "\u0410\u043b\u0435\u043a\u0441\u0435\u0439 \u041f\u043e\u0434\u0447\u0435\u0437\u0435\u0440\u0446\u0435\u0432", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mBbUw_CVz8K3RIunXwSoj-hNZ6f2buYD0JYAcB_=s64", "userId": "04087359208169148337"}, "user_tz": -180} id="bvuOfJZdA878" outputId="12ad7682-31cd-445e-af3e-110b1a15ff0c"
model.summary()


# + colab={} colab_type="code" id="rglNGLdxBSER"
class ModelSaveCallback(keras.callbacks.Callback):

    def __init__(self, file_name):
        super(ModelSaveCallback, self).__init__()
        self.file_name = file_name

    def on_epoch_end(self, epoch, logs=None):
        filename = self.file_name.format(epoch)
        keras.models.save_model(self.model, filename)
class ModelPredictCallback(keras.callbacks.Callback):

    def __init__(self, file_name):
        super(ModelPredictCallback, self).__init__()
        self.file_name = file_name

    def on_epoch_end(self, epoch, logs=None):
        filename = self.file_name.format(epoch)
        result = self.model.predict(X_test, batch_size=batch_size*16, verbose=0)
        df = pd.DataFrame()
        df['Id'] = range(1, len(X_test) + 1)
        df['Probability'] = result
        df.to_csv(filename, index=None)


# + colab={} colab_type="code" id="lHX0rUk3Bj8g"
TAKE=28 # Номер попытки
batch_size=512
NAME='kernel_conv_lstm_nofeatures_glove.twitter.27B.100d'
model_filename = '{0:02d}_{1:s}_{{0:02d}}.hdf5'.format(TAKE, NAME)
predict_filename = '{0:02d}_{1:s}_{{0:02d}}.csv'.format(TAKE, NAME)

# + _cell_guid="333626f1-a838-4fea-af99-0c78f1ef5f5c" _uuid="c1558c6b2802fc632edc4510c074555a590efbd8" colab={} colab_type="code" id="L0mZIwSL7W6j"
model.fit(X_train, y, batch_size=batch_size, epochs=1, validation_split=0.1,verbose=1,
          initial_epoch=0,
        callbacks = [
            ModelSaveCallback(model_filename),
            ModelPredictCallback(predict_filename),
        ],
        )

# + colab={"base_uri": "https://localhost:8080/", "height": 75} colab_type="code" executionInfo={"elapsed": 1829, "status": "ok", "timestamp": 1577138131739, "user": {"displayName": "\u0410\u043b\u0435\u043a\u0441\u0435\u0439 \u041f\u043e\u0434\u0447\u0435\u0437\u0435\u0440\u0446\u0435\u0432", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mBbUw_CVz8K3RIunXwSoj-hNZ6f2buYD0JYAcB_=s64", "userId": "04087359208169148337"}, "user_tz": -180} id="ihaURq2aEJTj" outputId="356560e9-a3d2-4602-f68e-5b93071128a7"
from IPython.display import Audio
Audio('First_epoch.mp3', autoplay=True) 

# + colab={"base_uri": "https://localhost:8080/", "height": 522} colab_type="code" executionInfo={"elapsed": 7212994, "status": "error", "timestamp": 1577146185060, "user": {"displayName": "\u0410\u043b\u0435\u043a\u0441\u0435\u0439 \u041f\u043e\u0434\u0447\u0435\u0437\u0435\u0440\u0446\u0435\u0432", "photoUrl": "https://lh3.googleusercontent.com/a-/AA<KEY>", "userId": "04087359208169148337"}, "user_tz": -180} id="NOIOuOAaA3dF" outputId="2110d997-1e3a-4e2c-8da1-e56659cd23d2"
model.fit(X_train, y, batch_size=batch_size, epochs=7, validation_split=0.1,verbose=2,
          initial_epoch=1,
        callbacks = [
            ModelSaveCallback(model_filename),
            ModelPredictCallback(predict_filename),
        ],
        )

# + [markdown] colab_type="text" id="7DMNHB9ATgP_"
# Наша модель полностью обучилась, добавим элемента неожиданности

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} colab_type="code" executionInfo={"elapsed": 7362944, "status": "ok", "timestamp": 1577154314824, "user": {"displayName": "\u0410\u043b\u0435\u043a\u0441\u0435\u0439 \u041f\u043e\u0434\u0447\u0435\u0437\u0435\u0440\u0446\u0435\u0432", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mBbUw_CVz8K3RIunXwSoj-hNZ6f2buYD0JYAcB_=s64", "userId": "04087359208169148337"}, "user_tz": -180} id="_rsYODImetYC" outputId="8b9e1d19-1754-4ded-9277-5d10c724a564"
for i in range(20):
    print(i)
    idx = np.random.randint(X_train.shape[0], size=int(X_train.shape[0]/20))
    X_train1 = X_train[idx, :]
    y1 = y[idx]
    model.fit(X_train1, y1, batch_size=int(batch_size/2), epochs=i+4,verbose=1, initial_epoch=i+3)

# + colab={} colab_type="code" id="7Uz7PLkJQZyL"
result = model.predict(X_test, batch_size=batch_size*16, verbose=0)
df = pd.DataFrame()
df['Id'] = range(1, len(X_test) + 1)
df['Probability'] = result
df.to_csv('night_sgd.csv', index=None)

# + colab={"base_uri": "https://localhost:8080/", "height": 67} colab_type="code" id="Jy_u39mnQc14" outputId="f802a56e-0b26-4f14-f468-ca9f722a1cec"
model.fit(X_train, y, batch_size=batch_size, epochs=3, validation_split=0.1,verbose=1,
          initial_epoch=0,
        callbacks = [
            ModelSaveCallback(model_filename),
            ModelPredictCallback(predict_filename),
        ],
        )

# + colab={} colab_type="code" id="9ShXTSvOSyXK"


StarcoderdataJupyterScriptsDedupFiltered