# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda env:py37]
#     language: python
#     name: conda-env-py37-py
# ---

# note: 
# - We found that kendall implementation in the pandas differ from v1.3.1 and 1.1.5
# - pandas does not have tau-c implementation
# - we use kendall from the scipy package
# - library version:
#     - pandas: 1.3.1
#     - scipy: 1.6.2

# # flickr8k

import pandas as pd
import numpy as np

# +
import scipy
from scipy import stats

def kendall_tau_b(x, y):
    tau, p_value = stats.kendalltau(x, y, variant="b")
    return tau

def kendall_tau_c(x, y):
    tau, p_value = stats.kendalltau(x, y, variant="c")
    return tau

scipy.__version__

# +
# # official vilbertscore

# df = pd.read_csv("../results/flickr8k.csv")
# flickr_precision = df["precision"].to_list()
# flickr_recall = df["recall"].to_list()
# flickr_f1 = df["f1"].to_list()
    
# with open("../data/flickr8k/scores.pkl", "rb") as f:
#     human = pickle.load(f, encoding='latin1')
# human = [(x-1)/3 for x in human]

# +
# custom vilbertscore
df = pd.read_csv("../data/processed/flickr8k/vilbertscore.csv")
flickr_precision = df["precision"].to_list()
flickr_recall = df["recall"].to_list()
flickr_f1 = df["f1"].to_list()
    
with open("../data/processed/flickr8k/annotations_avg.txt") as f:
    human = f.readlines()
    human = [ float(x.strip()) for x in human]
human = [(x-1)/3 for x in human]

df_final = pd.DataFrame([human, flickr_precision, flickr_recall, flickr_f1]).T
df_final = df_final.rename(columns={0:"human", 1:"precision", 2:"recall", 3:"f1"})
df_final.corr(method=kendall_tau_c)

# +
# df_final.corr(method=kendall_tau_b)
# -















# ### flat annotation

# +
# custom vilbertscore
df = pd.read_csv("../data/processed/flickr8k/vilbertscore.csv")
flickr_precision = df["precision"].to_list()
flickr_recall = df["recall"].to_list()
flickr_f1 = df["f1"].to_list()

flickr_precision_repeat3 = []
flickr_recall_repeat3 = []
flickr_f1_repeat_3 = []

for a, b, c in zip(flickr_precision, flickr_recall, flickr_f1):
    flickr_precision_repeat3.append(float(a))
    flickr_precision_repeat3.append(float(a))
    flickr_precision_repeat3.append(float(a))
    
    flickr_recall_repeat3.append(float(b))
    flickr_recall_repeat3.append(float(b))
    flickr_recall_repeat3.append(float(b))
    
    flickr_f1_repeat_3.append(float(c))
    flickr_f1_repeat_3.append(float(c))
    flickr_f1_repeat_3.append(float(c))
    
human_flat = []
with open("../data/processed/flickr8k/annotations.txt") as f:
    tmp = f.readlines()
    for item in tmp:
        a, b, c = item.strip().split(",")
        human_flat.append(float(a))
        human_flat.append(float(b))
        human_flat.append(float(c))
len(human_flat)
# -

df_final_flat = pd.DataFrame([human_flat, flickr_precision_repeat3, flickr_recall_repeat3, flickr_f1_repeat_3]).T
df_final_flat = df_final_flat.rename(columns={0:"human", 1:"precision", 2:"recall", 3:"f1"})
df_final_flat.corr(method=kendall_tau_c)

# +
# df_final_flat.corr(method=kendall_tau_b)
# -







# # capeval1k

import pandas as pd
import numpy as np

df = pd.read_csv("../data/raw/capeval1k/capeval1k_all_metrics.csv", index_col=False)
df = df.drop(["Unnamed: 0"], axis=1)

# +
df_vs = pd.read_csv("../data/processed/capeval1k/vilbertscore.csv", index_col=False)
df_vs_f1 = df_vs.drop(["precision", "recall"], axis=1)
df_vs_f1 = df_vs_f1.rename(columns={"f1":"vilbertscore"})

df_clip = pd.read_csv("../data/processed/capeval1k/clipcore.csv", index_col=False)
# -

df_merge = pd.concat([df, df_vs_f1, df_clip], axis=1)
df_merge.corr(method=kendall_tau_c)























# +
# import json

# with open("../data/processed/capeval1k/cand.txt") as f:
#     cap_test = f.readlines()
#     cap_test = [x.strip() for x in cap_test]

# with open("../data/processed/capeval1k/capeval_clip_result.json") as f:
#     clip = json.load(f)
# clip[0]

# list_clips = []
# list_refclips = []
# for check, y in zip(cap_test, clip):
#     if check == y["candiate_caption"]:
#         list_clips.append(y["CLIP-S"])
#         list_refclips.append(y["RefCLIP-S"])
        
# df = pd.DataFrame([list_clips, list_refclips]).T
# df = df.rename(columns={0:"clips", 1:"ref-clips"})
# df.to_csv("../data/processed/capeval1k/clipcore.csv", index=False)
# -



StarcoderdataJupyterScriptsDedupFiltered