# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: tf
#     language: python
#     name: tf
# ---

# %load_ext autoreload
# %autoreload 2

# %matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_context('paper')
sns.set_style('white')

import configs

lstm_gen = configs.config_generator('CharLSTM')

lstm_confs = configs.get_x_configs('CharLSTM', 100)

import pandas as pd
df = pd.DataFrame(lstm_confs)

df.optimizer.value_counts()

df.lstm_layer.value_counts()

df.batch_size.value_counts()

cnn_confs = configs.get_x_configs('CharCNN', 100)

import pandas as pd
cdf = pd.DataFrame(cnn_confs)

cdf.optimizer.value_counts()

cdf.lstm_layer.value_counts()

cdf.batch_size.value_counts()

import os

# + active=""
# os.mkdir('hp_search/models')

# + active=""
# os.mkdir('hp_search')
# os.mkdir('hp_search/conf')

# + active=""
# os.mkdir('hp_search/conf_json')

# +
import json

max_digits = len(str(len(lstm_confs)))
full_confs = []
for i, conf in enumerate(lstm_confs):
    fc = configs.create_conf_file(f'hp_search/conf/morph_charlstm.{str(i).zfill(max_digits)}.conf', 
                             f'hp_search/models/morph_charlstm.{str(i).zfill(max_digits)}.model', 
                             'gold_morpheme', 
                             conf, 
                             'alt_tok_yap_ft_sg')
    full_confs.append(fc)

for i, fc in enumerate(full_confs):
    with open( f'hp_search/conf_json/morph_charlstm.{str(i).zfill(max_digits)}.json', 'w') as of:
        of.write(json.dumps(fc))

# +
max_digits = len(str(len(cnn_confs)))
full_confs = []
for i, conf in enumerate(cnn_confs):
    fc = configs.create_conf_file(f'hp_search/conf/morph_charcnn.{str(i).zfill(max_digits)}.conf', 
                             f'hp_search/models/morph_charcnn.{str(i).zfill(max_digits)}.model', 
                             'gold_morpheme', 
                             conf, 
                             'alt_tok_yap_ft_sg')
    full_confs.append(fc)

for i, fc in enumerate(full_confs):
    with open( f'hp_search/conf_json/morph_charcnn.{str(i).zfill(max_digits)}.json', 'w') as of:
        of.write(json.dumps(fc))
# -

# ## Token

# +
max_digits = len(str(len(lstm_confs)))
full_confs = []
for i, conf in enumerate(lstm_confs):
    fc = configs.create_conf_file(f'hp_search/conf/token_charlstm.{str(i).zfill(max_digits)}.conf', 
                             f'hp_search/models/token_charlstm.{str(i).zfill(max_digits)}.model', 
                             'gold_token_bioes', 
                             conf, 
                             'alt_tok_tokenized_ft_sg')
    full_confs.append(fc)

for i, fc in enumerate(full_confs):
    with open( f'hp_search/conf_json/token_charlstm.{str(i).zfill(max_digits)}.json', 'w') as of:
        of.write(json.dumps(fc))

# +
max_digits = len(str(len(cnn_confs)))
full_confs = []
for i, conf in enumerate(cnn_confs):
    fc = configs.create_conf_file(f'hp_search/conf/token_charcnn.{str(i).zfill(max_digits)}.conf', 
                             f'hp_search/models/token_charcnn.{str(i).zfill(max_digits)}.model', 
                             'gold_token_bioes', 
                             conf, 
                             'alt_tok_tokenized_ft_sg')
    full_confs.append(fc)

for i, fc in enumerate(full_confs):
    with open( f'hp_search/conf_json/token_charcnn.{str(i).zfill(max_digits)}.json', 'w') as of:
        of.write(json.dumps(fc))
# -





# ## Check results

len(cnn_confs[0]), len(lstm_confs[0])

lstm_confs[0]

# +
import re
DEV_RES_LINE = re.compile('Dev: .*; acc: (?P<acc>[^,]+), p: (?P<p>[^,]+), r: (?P<r>[^,]+), f: (?P<f>[-\d\.]+)')


res = []
for f in os.scandir('hp_search/logs'):
    if f.name.startswith('.ipy'):
        continue
    arch = f.name.split('.')[0]
    unit = arch.split('_')[0]
    conf_num = f.name.split('.')[1]
    matching_conf = cnn_confs[int(conf_num)] if 'cnn' in arch else lstm_confs[int(conf_num)]
    params = { 'arch': arch, 'unit': unit, 'conf_num': conf_num}
    params.update(matching_conf)
    with open(f.path, 'r') as fp:
        i= 0
        for line in fp:
            m = DEV_RES_LINE.match(line)
            if m:
                r = m.groupdict().copy()
                for k, v in r.items():
                    r[k] = float(v)
                r.update(params)
                r['epoch'] = i
                i+=1
                res.append(r)
# -

rdf = pd.DataFrame(res)

erdf = rdf[(rdf.groupby(['conf_num', 'arch']).f.transform(max)==rdf.f) ]
erdf = erdf[(erdf.groupby(['conf_num', 'arch']).epoch.transform(min)==erdf.epoch) ]

erdf.groupby(['conf_num', 'arch']).f.max().unstack()

erdf.shape

erdf.groupby('arch').f.max()

erdf.groupby(['optimizer', 'learning_rate', 'arch']).conf_num.nunique().unstack()

erdf[erdf.f>0].groupby(['optimizer', 'learning_rate', 'arch']).conf_num.nunique().unstack()

erdf.groupby(['optimizer', 'learning_rate', 'arch']).f.median().unstack()

erdf.groupby(['optimizer', 'learning_rate', 'arch']).f.max().unstack()

erdf.groupby(['dropout', 'arch']).f.median().unstack()

erdf.groupby(['hidden_dim', 'arch']).f.median().unstack()

erdf.groupby(['lstm_layer', 'arch']).f.median().unstack()

erdf.groupby(['lstm_layer', 'arch']).f.max().unstack()

erdf.groupby(['optimizer', 'arch']).f.median().unstack()

erdf[erdf.arch.str.contains('cnn')].groupby(['cnn_layer', 'arch']).f.median().unstack()

erdf[erdf.arch.str.contains('lstm')].groupby(['char_hidden_dim', 'arch']).f.median().unstack()

erdf.loc[erdf.char_seq_feature=='CNN', 'char_hidden_dim']=None
erdf.loc[erdf.char_seq_feature=='LSTM', 'cnn_layer']=None

erdf[erdf.optimizer=='SGD'].groupby(['momentum', 'arch']).f.median().unstack()

erdf[erdf.optimizer=='SGD'].groupby(['lr_decay', 'arch']).f.median().unstack()

(erdf[erdf.unit=='morph'].sort_values('f', ascending=False).head(20)[['f', 'p', 'r', 'char_seq_feature', 'lstm_layer', 'batch_size', 
                                                 'optimizer', 'learning_rate', 'lr_decay', 'momentum', 'char_hidden_dim', 'cnn_layer',
                                                 'hidden_dim', 'dropout', 'epoch']]).fillna('')

(erdf[erdf.unit=='morph'].sort_values('f', ascending=True).head(20)[['f', 'p', 'r', 'char_seq_feature', 'lstm_layer', 'batch_size', 
                                                 'optimizer', 'learning_rate', 'lr_decay', 'momentum', 'char_hidden_dim', 'cnn_layer',
                                                 'hidden_dim', 'dropout', 'epoch']]).fillna('')

(erdf[erdf.unit=='token'].sort_values('f', ascending=False).head(20)[['f', 'p', 'r', 'char_seq_feature', 'lstm_layer', 'batch_size', 
                                                 'optimizer', 'learning_rate', 'lr_decay', 'momentum', 'char_hidden_dim', 'cnn_layer',
                                                 'hidden_dim', 'dropout', 'epoch']]).fillna('')

(erdf[erdf.unit=='token'].sort_values('f', ascending=True).head(20)[['f', 'p', 'r', 'char_seq_feature', 'lstm_layer', 'batch_size', 
                                                 'optimizer', 'learning_rate', 'lr_decay', 'momentum', 'char_hidden_dim', 'cnn_layer',
                                                 'hidden_dim', 'dropout', 'epoch']]).fillna('')

lstm_confs[0]

erdf.groupby('arch').epoch.agg(['median', 'mean', 'std'])

sns.swarmplot(data=rdf[rdf.groupby(['arch', 'conf_num']).f.transform(max)==rdf.f].groupby(['arch', 'conf_num']).epoch.min().reset_index(), x='arch', y='epoch')

sns.swarmplot(data=rdf[(rdf.f>0.1) & (rdf.groupby(['optimizer', 'conf_num']).f.transform(max)==rdf.f)].groupby(['optimizer', 'conf_num']).epoch.min().reset_index(), x='optimizer', y='epoch')

sns.swarmplot(data=erdf.groupby(['char_seq_feature', 'unit','conf_num']).f.max().reset_index(), x='unit', y='f', hue='char_seq_feature', split=True)


sns.swarmplot(data=rdf[rdf.f>0].groupby(['conf_num', 'unit','batch_size']).f.max().reset_index(), x='unit', y='f', hue='batch_size', split=True)


sns.swarmplot(data=erdf, x='arch', y='f', hue='hidden_dim', split=True)


sns.swarmplot(data=erdf, x='arch', y='f', hue='lstm_layer', split=True)


sns.swarmplot(data=erdf, x='arch', y='f', hue='dropout', split=True)


sns.swarmplot(data=erdf, x='arch', y='f', hue='char_hidden_dim', split=True)

sns.swarmplot(data=erdf, x='arch', y='f', hue='cnn_layer', split=True)

sns.swarmplot(data=erdf[erdf.optimizer=='SGD'], x='learning_rate', y='f', split=True)

sns.swarmplot(data=erdf[erdf.optimizer=='Adam'], x='learning_rate', y='f', split=True)

sns.swarmplot(data=erdf[erdf.optimizer=='RMSProp'], x='learning_rate', y='f', split=True)

sns.swarmplot(data=erdf[erdf.optimizer=='SGD'], x='learning_rate', y='f', hue='lr_decay', split=True)

sns.swarmplot(data=erdf[(erdf.optimizer=='SGD') & (erdf.learning_rate<0.03)], x='unit', y='f', hue='lr_decay', split=True)

x = 7
int(x/2)

sns.swarmplot(data=erdf[erdf.optimizer=='SGD'], x='momentum', y='f', split=True)

erdf.groupby('arch').apply(lambda x: x.sort_values('f', ascending=False).head(2)).T

rdf[rdf.groupby('arch').f.transform(max)==rdf.f].sort_values('f', ascending=False).T

# ## TODO:
# 1. create more random configs (we will take them sequencially for order's sake)
# 1. ~~run more in another server~~
# 1. ~~match confs and results~~
# 1. initial analysis and maybe drop and add some HPs



StarcoderdataJupyterScriptsDedupFiltered