# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Evaluate all embeddings generated with parameter_search

import os
import pandas as pd
import numpy as np
from pathlib import Path
import datetime
import pickle
import seaborn as sns
import matplotlib.pyplot as plt

from evaluation_functions import nn, sil

# +
wd = os.getcwd()

DF = os.path.join(os.path.sep, str(Path(wd).parents[0]), "data", "processed", "df_focal_reduced.pkl")

OUT_COORDS = os.path.join(os.path.sep, str(Path(wd).parents[0]), "data", "interim", "parameter_search", "grid_search")
OUT_EVALS = os.path.join(os.path.sep, str(Path(wd).parents[0]), "data", "interim", "parameter_search", "grid_search_evals")
FIGURES = os.path.join(os.path.sep, str(Path(wd).parents[0]), "reports", "figures", "grid_search")

if (not os.path.isdir(OUT_COORDS)):
    print("Folder with UMAP coordinates doesn't exist: Missing ", OUT_COORDS)
    
for d in [OUT_EVALS, FIGURES]:
    if (not os.path.isdir(d)):
        os.mkdir(d)

# +
spec_df = pd.read_pickle(DF)
print(spec_df.shape)

labels = spec_df.call_lable.values
labeltypes = sorted(list(set(labels)))

# +
#outname = os.path.join(os.path.sep, OUT_EVALS, 'eval_table_5.csv')
#eval_table = pd.read_csv(outname, sep=";")
#already_evaluated = [x+'.csv' for x in eval_table[params].astype(str).agg('_'.join, axis=1)]
#not_evaluated = list(set(all_embedding_files) - set(already_evaluated))
# -

params = ['preprocess_type', 'metric_type', 'duration_method','min_dist', 'spread', 'n_neighbors', 'n_comps', 'input_type', 'denoised', 'n_mels', 'f_unit', 'bp_filtered', 'n_repeat']
k=5

all_embedding_files = list(sorted(os.listdir(OUT_COORDS)))
print(len(all_embedding_files))

# Check if some evaluation already exists
outname = os.path.join(os.path.sep, OUT_EVALS, 'eval_table_'+str(k)+'.csv')
if (os.path.exists(outname)):
    eval_table = pd.read_csv(outname, sep=";")
    #already_evaluated = [x+'.csv' for x in eval_table[params].astype(str).agg('_'.join, axis=1)]
    already_evaluated = list(eval_table['filename'])
    not_evaluated = list(set(all_embedding_files) - set(already_evaluated))

    print(len(not_evaluated))
    if (len(not_evaluated)>0):
        old_eval_table = eval_table.copy()
        all_embedding_files = not_evaluated

if len(all_embedding_files)>0:
    eval_colnames = params+ ['S_total'] + ['S_'+x for x in labeltypes] + ['Snorm_total'] + ['Snorm_'+x for x in labeltypes] + ['SIL_total'] + ['SIL_'+x for x in labeltypes]+['knncc_'+x for x in labeltypes]+['knn-acc_'+x for x in labeltypes]
    #print(eval_colnames)

    eval_table = np.zeros((len(all_embedding_files), len(eval_colnames)))
    eval_table = pd.DataFrame(eval_table, columns=eval_colnames)
    
    for i,embedding_file in enumerate(all_embedding_files):
        embedding = np.loadtxt(os.path.join(os.path.sep, OUT_COORDS, embedding_file),delimiter=";")

        embedding_params_string = embedding_file.replace('.csv', '')
        embedding_params_list = embedding_params_string.split('_')

        nn_stats = nn(embedding, labels, k=k)
        sil_stats = sil(embedding, labels)

        eval_vector = embedding_params_list + [nn_stats.get_S()] + list(nn_stats.get_ownclass_S()) + [nn_stats.get_Snorm()] + list(nn_stats.get_ownclass_Snorm()) + [sil_stats.get_avrg_score()] + list(sil_stats.get_score_per_class()) + list(nn_stats.knn_cc()) + list(nn_stats.knn_accuracy())
        eval_table.loc[i,:] = eval_vector

    eval_table['filename'] = all_embedding_files
    
    eval_table = pd.concat([old_eval_table, eval_table])


eval_table

eval_table['knncc_total'] = eval_table[['knncc_'+x for x in labeltypes]].mean(axis=1)
eval_table['knn-acc_total'] = eval_table[['knn-acc_'+x for x in labeltypes]].mean(axis=1)

outname = os.path.join(os.path.sep, OUT_EVALS, 'eval_table_'+str(k)+'.csv')
print(outname)
eval_table.to_csv(outname, sep=";",  index=False)

# ## Plot results

k=5

outname = os.path.join(os.path.sep, OUT_EVALS, 'eval_table_'+str(k)+'.csv')
eval_table = pd.read_csv(outname, sep=";")

eval_table["min_dist"] = pd.to_numeric(eval_table["min_dist"])
eval_table["n_neighbors"] = pd.to_numeric(eval_table["n_neighbors"])
eval_table["spread"] = pd.to_numeric(eval_table["spread"])
eval_table["n_comps"] = pd.to_numeric(eval_table["n_comps"])
eval_table["n_mels"] = pd.to_numeric(eval_table["n_mels"])
eval_table["n_repeat"] = pd.to_numeric(eval_table["n_repeat"])

eval_table

print(np.quantile(eval_table.S_total, 0.01))
print(np.quantile(eval_table.S_total, 0.99))

duration_method = eval_table.duration_method.copy()

# +
dur_dict = {'overlap-only': 'overlap',
           'timeshift-overlap': 'tshift-overlap',
           'timeshift-pad': 'tshift-pad',
           'pairwise-pad': 'pw-pad'}

dur_dict = {'overlap': 'ovl',
            'pad': 'pad',
            'overlap-only': 'ovl-o',
           'timeshift-overlap': 'ts-ovl',
           'timeshift-pad': 'ts-pad',
            'stretched': 'str',
           'pairwise-pad': 'pw-pad'}

duration_method = [dur_dict[x] if x in dur_dict.keys() else x for x in duration_method]
eval_table['duration_method'] = duration_method

# +
metric_dict = {'correlation': 'corr',
           'cosine': 'cosi',
           'euclidean': 'euclid',
           'manhattan': 'manh'}

metric_method = [metric_dict[x] if x in metric_dict.keys() else x for x in eval_table['metric_type']]
eval_table['metric_type'] = metric_method

# +
# Remove invalid spread
#eval_table = eval_table.loc[eval_table.spread!=0.1,:]
    
# Remove invalid input_type
#eval_table = eval_table.loc[eval_table.input_type!='mfccs',:]
#eval_table['input_type'] = [x if x!='zmfccs' else 'mfccs' for x in eval_table.input_type]
# -

calltypes = sorted(list(set(spec_df.call_lable)))
pal = sns.color_palette("Set2", n_colors=len(calltypes))

params = ['preprocess_type', 'metric_type', 'duration_method','min_dist', 'spread', 'n_neighbors', 'n_comps', 'input_type','denoised', 'n_mels', 'f_unit', 'bp_filtered', 'n_repeat']
p_default = dict(zip(params[:-1], ['zs', 'euclid', 'pad', 0.0, 1.0, 15, 3, 'melspecs', 'no', 40, 'dB', 'no']))

# +
param = 'denoised'
print(param)
outvar="knn-acc_total"

df = eval_table
boxplot = df[[param]+[outvar]].boxplot(by=param)
plt.ylim(30,70)
# -

print(np.min(eval_table.S_total))
print(np.max(eval_table.S_total))
print(np.min(eval_table.SIL_total))
print(np.max(eval_table.SIL_total))
print(np.min(eval_table.knncc_total))
print(np.max(eval_table.knncc_total))

y_lower_dict = {"SIL":-0.65,
                "SIL_total": -0.3,
                "S":0,
                "S_total": 30,
                "knncc":0,
                "knncc_total": 10,
                "knn-acc": 0,
                "knn-acc_total": 30}
y_upper_dict = {"SIL":0.65,
                "SIL_total": 0.3,
                "S":100,
                "S_total": 70,
                "knncc":100,
                "knncc_total": 45,
                "knn-acc": 100,
                "knn-acc_total": 70}

# +
# BOXPLOTS

for outvar in ['S_total', 'SIL_total']:
    for param in params[:-1]:
        other_params = set(params).difference([param, 'n_repeat'])
        boxplot = df[[param]+[outvar]].boxplot(by=param)

        plt.ylim(y_lower_dict[outvar], y_upper_dict[outvar])
        plt.suptitle('')
        plt.title('')
        plt.xlabel(param)
             
        plt.savefig(os.path.join(os.path.sep, FIGURES,'box_'+outvar+'_'+param+'.jpg'))
        plt.close()

# +
# LINE PLOTS

for out_v in ["SIL", "S", "knncc"]:
    outvars = [out_v+'_'+x for x in calltypes]
    #outvars = [out_v+'_total']+outvars
    
    for param in params[:-1]:
        other_params = set(params).difference([param, 'n_repeat'])

        df = eval_table

        #print(df.shape)
        means = df[[param, out_v+'_total']]
        df = df[[param]+outvars]

        melted = pd.melt(df, id_vars=param, value_vars=outvars)
        melted = melted.sort_values(by=param)

        sns.lineplot(x=param, y="value", hue="variable", data=melted, palette="Set2", hue_order=outvars, err_style='band')
        sns.lineplot(x=param, y=out_v+'_total', data=means, color='black')
        plt.ylabel(out_v)
        plt.ylim(y_lower_dict[out_v], y_upper_dict[out_v])
        
        lg = plt.legend(bbox_to_anchor=(1.05,1),loc=2, borderaxespad=0.)
        plt.savefig(os.path.join(os.path.sep, FIGURES,'line_'+out_v+'_'+param+'.jpg'), bbox_extra_artists=(lg,), bbox_inches='tight')
        plt.close()

# +
# Lineplots with error bars

for out_v in ["SIL", "S", "knncc"]:
    outvars = [out_v+'_'+x for x in calltypes]
    color_dict = dict(zip(outvars, pal))

    for param in params[:-1]:
        other_params = set(params).difference([param, 'n_repeat'])

        df = eval_table
        #print(df.shape)
        means = df[[param, out_v+'_total']]
        df = df[[param]+outvars]
        
        levels = sorted(list(set(df[param])))
        mean_df = df.groupby([param]).mean()
        std_df = df.groupby([param]).std()

        fig, ax = plt.subplots(figsize=(7, 4))

        for outvar in outvars:
            y = mean_df[outvar].values
            yerr = std_df[outvar].values
            ax.errorbar(levels, y, yerr=yerr,color=color_dict[outvar]) # linestype=ls

        ax.set_ylim(y_lower_dict[out_v], y_upper_dict[out_v])
        ax.set_title(param)
        plt.savefig(os.path.join(os.path.sep, FIGURES,'err_'+out_v+'_'+param+'.jpg'))
        plt.close()

# +
# Lineplots with error bars with Mean (?)

for out_v in ["SIL", "S", "knncc"]:
#for out_v in ["S"]:
    outvars = [out_v+'_'+x for x in calltypes]
    color_dict = dict(zip(outvars, pal))
    
    outvars = [out_v+'_total']+outvars
    color_dict[out_v+'_total'] = "black"
    #for param in params[0:2]:
    for param in params[:-1]:
        other_params = set(params).difference([param, 'n_repeat'])

        df = eval_table
        #print(df.shape)
        means = df[[param, out_v+'_total']]
        df = df[[param]+outvars]
        
        levels = sorted(list(set(df[param])))
        mean_df = df.groupby([param]).mean()
        std_df = df.groupby([param]).std()

        fig, ax = plt.subplots(figsize=(7, 4))

        for outvar in outvars:
            y = mean_df[outvar].values
            yerr = std_df[outvar].values
            ax.errorbar(levels, y, yerr=yerr,color=color_dict[outvar]) # linestype=ls

        ax.set_ylim(y_lower_dict[out_v], y_upper_dict[out_v])
        ax.set_title(param)
        plt.savefig(os.path.join(os.path.sep, FIGURES,'errmean_'+out_v+'_'+param+'.jpg'))
        plt.close()

# +
#df = df.groupby([param]).mean()
# -

eval_table.head(5)

import statsmodels.api as sm
from statsmodels.formula.api import ols

print(np.min(eval_table.S_total))
print(np.max(eval_table.S_total))

plot_params = ['preprocess_type',
               'metric_type',                          
               'duration_method',
               'denoised',              
               'n_mels',
               'f_unit',
               'bp_filtered']

# +
outvar = 'S_total'

for i, param in enumerate(plot_params):
    print(param)
    other_params = set(params).difference([param, 'n_repeat'])
    df = eval_table
    #for p in other_params:
    #    df = df.loc[df[p]==p_default[p],:]
        
    levels = sorted(list(set(df[param])))
    print(levels)
    # STATS
    mod = ols(outvar+'~'+param, data=df).fit()
    aov_table = sm.stats.anova_lm(mod, typ=2)
    print(aov_table)
    #print(mod.summary())
# -

# # S_total and SIL_total

# +
# BOXPLOTS
plot_params = ['n_mels',
               'f_unit',
               'preprocess_type',                        
               'duration_method',
               'denoised',               
               'bp_filtered',
               'metric_type']

title_dict = dict(zip(plot_params, ['Number of mels', 
                                    'Spectrogram intensity unit',
                                    'Level of spectrogram preprocessing',                                                                     
                                    'Dealing with variable duration', 
                                    'Denoising: Median subtraction',                                   
                                    'Denoising: Bandpass filtering',
                                    'UMAP: distance metric' 
                                   ]))
n_rows = 2
n_cols = 4

cs = list(range(0,n_cols)) * n_rows
rs_list = [[x]*n_cols for x in list(range(0,n_rows))]
rs = list()
for x in rs_list:
    for y in x:
        rs.append(y)

#fig.suptitle('Effect of different run parameters')
#for outvar in ['knn-acc_total']:
for outvar in ['S_total', 'SIL_total', 'knncc_total', 'knn-acc_total']:
#for outvar in ['S_total']:
    fig, axes = plt.subplots(n_rows, n_cols, figsize=(21, 10), sharey=True)
    for i,param in enumerate(plot_params):
        #print(param)
        other_params = set(params).difference([param, 'n_repeat'])
        
        df = eval_table
        #for p in other_params:
        #    df = df.loc[df[p]==p_default[p],:]
        #print(df.shape)
        #print(levels)
        
        levels = sorted(list(set(df[param])))
        #color_dict = {level: "red" if level == p_default[param] else "white" for level in levels}
        
        ax = sns.boxplot(ax=axes[rs[i], cs[i]], data=df, x=param, y=outvar, order=levels)#, palette = color_dict)
        #ax.set_ylabel(fontsize=18)
        for b,box in enumerate(ax.artists):
            if b==levels.index(p_default[param]):
                col = "red"
            else:
                col="black"
            box.set_edgecolor(col)
            box.set_facecolor("white")#

        # iterate over whiskers and median lines
            for j in range(6*b,6*(b+1)):
                ax.lines[j].set_color(col)
            
            
        ax.set_ylim(y_lower_dict[outvar], y_upper_dict[outvar])
        ax.set_title(title_dict[param], fontsize=20)
        ax.set_xlabel('', fontsize=18)
        ax.set_ylabel('', fontsize=18)
        ax.tick_params(labelsize=18)

    plt.tight_layout()
    plt.setp(axes[:, 0], ylabel=outvar)
    fig.delaxes(ax= axes[1,3]) 
    #fig.delaxes(ax= axes[2,3]) 
    plt.savefig(os.path.join(os.path.sep, FIGURES,'box_'+outvar+'_all_urghs.jpg'), bbox_inches='tight')
    #plt.close()
# -



StarcoderdataJupyterScriptsDedupFiltered