# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import datetime
import json
import sys
sys.path.insert(0, '../src')

import warnings
from multiprocessing import Pool
from threading import Thread

import jsonlines
import pandas as pd
import seaborn as sns
from fuzzywuzzy import process
from intents import *
from matplotlib import pyplot as plt
from matplotlib.pyplot import figure
from p_tqdm import p_umap
from sklearn.metrics import (accuracy_score, confusion_matrix,
                             precision_recall_fscore_support)
from tqdm import tqdm
from utils import print_confusion_matrix
from IPython.display import clear_output
import random
from sklearn import preprocessing

le = preprocessing.LabelEncoder()
tqdm.pandas()
# -

verloop_intents = json.load(open("../data/verloop_intents.json"))

df_preds = pd.read_csv("../data/predictions_with_groundtruth.csv")
df_preds = df_preds[df_preds["MessageId"] != "zoJ8RNwzaLxPTzW9x"] # FALSE as a intent


def load_and_merge(field:str)->pd.DataFrame:
    output = []
    for obj in jsonlines.open(f"../data/davinci_{field}.jsonl"):
        output.append(obj)
    df_out = pd.DataFrame(output)
    df_out = df_out[["MessageId", "Message", "predicted", "logprobs"]].merge(
        df_preds[["MessageId", "VerdanPrediction", "Ground Truth"]], how="left"
    )
    clear_output()
    return df_out


# +
def measure_pref(df_out):
    le.fit(df_out["Ground Truth"])
    y_pred_gpt = le.transform(df_out["predicted"])
    y_pred_verdan = le.transform(df_out["VerdanPrediction"])
    y = le.transform(df_out["Ground Truth"])
    print(f"Number of samples : {len(df_out)}")
    print("Verdan Scores")
    print_scores(y_pred_verdan, y)
    print("GPT3 Scores")
    print_scores(y_pred_gpt, y)
    
def print_scores(y_pred, y):
    print(f'Accuracy : {accuracy_score(y_pred, y)}')
    print(f'PRF : {precision_recall_fscore_support(y_pred, y, average="micro")}')


# -

field = "Refunds"
df_out = load_and_merge(field) 
df_out["predicted"] = df_out["predicted"].apply(lambda x : x if x==field else f"Not {field}")
df_out["VerdanPrediction"] = df_out["VerdanPrediction"].apply(lambda x : x if x==field else f"Not {field}")
df_out["Ground Truth"] = df_out["Ground Truth"].apply(lambda x : x if x==field else f"Not {field}")
measure_pref(df_out)

field = "OrderDelivery"
df_out = load_and_merge(field)
df_out["predicted"] = df_out["predicted"].apply(lambda x : x if x==field else f"Not {field}")
df_out["VerdanPrediction"] = df_out["VerdanPrediction"].apply(lambda x : x if x==field else f"Not {field}")
df_out["Ground Truth"] = df_out["Ground Truth"].apply(lambda x : x if x==field else f"Not {field}")
measure_pref(df_out)

field = "Cancellations"
df_out = load_and_merge(field)
df_out["predicted"] = df_out["predicted"].apply(lambda x : x if x==field else f"Not {field}")
df_out["VerdanPrediction"] = df_out["VerdanPrediction"].apply(lambda x : x if x==field else f"Not {field}")
df_out["Ground Truth"] = df_out["Ground Truth"].apply(lambda x : x if x==field else f"Not {field}")
measure_pref(df_out)

field = "OrderReceiving"
df_out = load_and_merge(field)
df_out["predicted"] = df_out["predicted"].apply(lambda x : x if x==field else f"Not {field}")
df_out["VerdanPrediction"] = df_out["VerdanPrediction"].apply(lambda x : x if x==field else f"Not {field}")
df_out["Ground Truth"] = df_out["Ground Truth"].apply(lambda x : x if x==field else f"Not {field}")
measure_pref(df_out)

field = "OrderPlacement"
df_out = load_and_merge(field)
df_out["predicted"] = df_out["predicted"].apply(lambda x : x if x==field else f"Not {field}")
df_out["VerdanPrediction"] = df_out["VerdanPrediction"].apply(lambda x : x if x==field else f"Not {field}")
df_out["Ground Truth"] = df_out["Ground Truth"].apply(lambda x : x if x==field else f"Not {field}")
measure_pref(df_out)

StarcoderdataJupyterScriptsDedupFiltered