# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [py27]
#     language: python
#     name: Python [py27]
# ---

# +
# %matplotlib notebook

import itertools
import logging
from functools import partial
from collections import defaultdict, OrderedDict

import gensim
import matplotlib
from matplotlib import rc
import matplotlib.pyplot as plt
from matplotlib import colors as plt_colors
import numpy as np
import pandas as pnd
import os
from sklearn.cluster import *
from sklearn.preprocessing import normalize
from sklearn.cross_validation import train_test_split
from sklearn.decomposition import PCA, RandomizedPCA
from sklearn.manifold import TSNE
from sklearn import svm, metrics
from multiprocessing import Pool
from knub.thesis.util import *

pnd.set_option("display.max_colwidth", 200)
LIGHT_COLORS = ["#a6cee3", "#b2df8a", "#fb9a99", "#fdbf6f", "#cab2d6", "#ffff99"] # light colors
DARK_COLORS = ["#1f78b4", "#33a02c", "#e31a1c", "#ff7f00", "#6a3d9a"] # dark colors

rc('font', **{'family': 'serif', 'serif': ['Computer Modern']})
rc('text', usetex=True)
SIZE = 11
rc('font', size=SIZE)  # controls default text sizes
rc('axes', titlesize=SIZE)  # fontsize of the axes title
rc('axes', labelsize=SIZE)  # fontsize of the x and y labels
rc('xtick', labelsize=SIZE)  # fontsize of the tick labels
rc('ytick', labelsize=SIZE)  # fontsize of the tick labels
rc('legend', fontsize=SIZE)  # legend fontsize
rc('figure', titlesize=SIZE)  # fontsize of the figure title
def cm2inch(*tupl):
    inch = 2.54
    if isinstance(tupl[0], tuple):
        return tuple(i/inch for i in tupl[0])
    else:
        return tuple(i/inch for i in tupl)


# +
def evaluate_single(df_param):
    nr_inclusions = len(df_param)
    nr_succ = len(df_param[df_param.successful])
    return float(nr_succ) / nr_inclusions

def evaluate_df(df_param, print_eval):
    for method, df_group in df_param.groupby("method"):
        if "mixture" in method:
            continue
        succ_prob = evaluate_single(df_group)
        if print_eval:
            print "%s: %.2f" % (method, succ_prob)
    
def evaluate_file(f, print_eval=True):
    df_file = pnd.read_csv("/home/knub/Repositories/master-thesis/webapp/out/" + f, sep="\t", header=None)
    df_file.columns = ["inclusion_idx", "inclusion_id", "method", "words", "intruder", "selected_word", "successful"]
    evaluate_df(df_file, print_eval)
    return df_file


# -

for f_name in sorted([f for f in os.listdir("../webapp/out") if f.endswith(".txt")]):
    print f_name
    evaluate_file(f_name)
    print

# **Evaluating all results**

# +
dfs = []
for f_name in sorted([f for f in os.listdir("../webapp/out") if f.endswith(".txt")]):
    df_tmp = evaluate_file(f_name, print_eval=False)
    df_tmp["evaluator"] = f_name.replace(".txt", "").replace("20_", "")
    dfs.append(df_tmp)

df_all = pnd.concat(dfs)
        
evaluate_df(df_all, print_eval=True)


# +
def aggregate_samples(df_param):
    return all(df_param)

df_agreement = df_all.groupby(["inclusion_id", "method"], as_index=False)["successful"].agg(aggregate_samples).reset_index()
pnd.reset_option('display.max_rows')
df_agreement.groupby("method")["successful"].agg(["mean", "count"])
# -

df_agreement.groupby(["method"])["mean"].mean()


# +
def build_word_intrusion(df_param):
    models = sorted(list(set(df_param.method)))
    evaluators = sorted(list(set(df_param.evaluator)))
    
    df_return = pnd.DataFrame(columns=models, index=evaluators)
    df_return.index.name = "evaluator"
    df_return.columns.name = "model"
    
    print models
    for evaluator, df_group_evaluator in df_param.groupby("evaluator"):
        for model, df_group_model in df_group_evaluator.groupby("method"):
            df_return.set_value(evaluator, model, evaluate_single(df_group_model))
    return df_return

def plot_word_intrusion(df_param):
    df_param = pnd.melt(df_param.reset_index(), id_vars=['evaluator'])
    df_param = df_param.dropna()
    df_param = df_param.sort_values(["model", "value"])
    models = sorted(list(set(df_param["model"])))
    for m in models:
        df_param.loc[df_param["model"] == m, "offset"] = range(len(df_param["model"] == m))
    models = OrderedDict([(v, k+1) for k, v in enumerate(models)])
    scatters = np.linspace(-0.25, 0.25, 8)
    
    df_param["scatter_x"] = df_param.apply(lambda x: models[x.model] + scatters[x.offset], axis=1)
    plt.figure(figsize=cm2inch(12, 6), dpi=300)
    plt.xlim(0.6, 4.8)
    plt.ylim((0.,1.0))
    plt.xticks(models.values(), ["LDA", "LFTM", "TopicVec", "WELDA"], rotation='horizontal')
    plt.scatter(df_param["scatter_x"], df_param["value"], c="black", marker='x', s=20, alpha=1.0, lw = 0.5)
    width = 0.25
    for m in models:
        mean = df_param[df_param.model == m]["value"].mean()
        #plt.axhline(y=mean)
        #print models[m]
        plt.hlines(y=mean, xmin=models[m]-width, xmax=models[m]+width, linestyles="dashed", linewidth=0.5)
        plt.text(models[m] + width + 0.03, mean, "$%.2f$" % mean, verticalalignment="center")

df_no_mixture = df_all[df_all.method.apply(lambda x: "mixture" not in x)]
df_20 = df_no_mixture[df_no_mixture.method.apply(lambda x: "20" in x)]
df_50 = df_no_mixture[df_no_mixture.method.apply(lambda x: "20" not in x)]

df_word_intrusion_20 = build_word_intrusion(df_20)
plot_word_intrusion(df_word_intrusion_20)

df_word_intrusion_50 = build_word_intrusion(df_50)
plot_word_intrusion(df_word_intrusion_50)
# -

df_word_intrusion_20

df_word_intrusion_50

StarcoderdataJupyterScriptsDedupFiltered