# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

import pandas as pd
import numpy as np
import json


# +
def load_jsonl(fname):
    fin = open(fname, encoding="utf-8")
    data = []
    for line in fin:
        d = json.loads(line.strip())
        data.append(d)

    return data

def save_jsonl(data, filename):
    with open(filename, "w", encoding="utf-8") as fo:
        for idx, d in enumerate(data):
            fo.write(json.dumps(d, ensure_ascii=False))
            fo.write("\n")


# -

# ls ..

data = []
DIR = "../Datasets/WisesightSentiment/few-shot/"
for i in range(1, 5+1):
    d = load_jsonl(f"{DIR}/misp{i}/all.jsonl")
    for sent in d:
        del sent["id"]
        sent["meta"]["part"] = i
    d = pd.DataFrame(d)
    data.append(d)

print("#Annotators:", len(data))



# # Inter-Annotator Agreement
#
# <img width="500px" src="https://miro.medium.com/max/875/1*mHB6Ciljb4OnOacNWgc0aw.png">

import sklearn
from sklearn.metrics import cohen_kappa_score
import statsmodels
from statsmodels.stats.inter_rater import fleiss_kappa

# +
# from sklearn.metrics import cohen_kappa_score
# y_true = [2, 0, 2, 2, 0, 1]
# y_pred = [0, 0, 2, 2, 0, 2]
# 

# +
# from itertools import combinations
# pairs = list(combinations(range(5), 2))
# -

# ### Sentence-wise

# +
# import modules
import matplotlib.pyplot as mp
import pandas as pd
import seaborn as sb
import numpy as np

def has_misspelling(val):
    return int(len(val)==0)

mat = []
for p1 in range(5):
    row = []
    for p2 in range(5):
        if p2 >= p1:
            row.append(0)
            continue
        
        d1 = data[p1]
        d2 = data[p2]

        d = d1.merge(d2, on="data", how="inner")

        m1 = d["label_x"].apply(has_misspelling)
        m2 = d["label_y"].apply(has_misspelling)

        row.append(cohen_kappa_score(m1, m2))
        print(p1, p2, cohen_kappa_score(m1, m2), (m1!=m2).sum())
    mat.append(row)


# -

mask = np.triu(np.ones_like(mat))
# # plotting a triangle correlation heatmap
dataplot = sb.heatmap(mat, cmap="YlGnBu", annot=True, mask=mask, vmin=0, vmax=1)
fig = dataplot.get_figure()
# fig.savefig("../Figures/iaa1.png")





# ### Token-wise

# +
def get_segments(text, labels):
    seg = []
    pt = 0
    
    labels = sorted(labels, key=lambda x: x[0], reverse=False)
    for l in labels:
        seg.append({"text": text[pt:l[0]], "misp": False, "int":False, "s": pt, "t":l[0]})
        mint = (l[2]=='ตั้งใจ')
        seg.append({"text": text[l[0]:l[1]], "misp": True, "int":mint, "s": l[0], "t":l[1]})
        pt = l[1]
    
    seg.append({"text": text[pt:], "misp": False, "int":False, "s": pt, "t":len(text)})
    
    length = sum([len(s["text"]) for s in seg])
    assert(length==len(text))
    return seg

def overlap_segments(text, segments):
    idx = set()
    for seg in segments:
        for s in seg:
            idx.add(s["s"])
            idx.add(s["t"])
    
    idx = sorted(idx)
    newseg = []
    for i, _ in enumerate(idx):
        if i==0:
            continue
            
        newseg.append({
            "text": text[idx[i-1]:idx[i]], 
            "s": idx[i-1], 
            "t": idx[i],
        })
    
    o = []
    for seg in segments:
        ns = []
        for s in newseg:
            for ref in seg:
                if s["s"] >= ref["s"] and  s["s"] < ref["t"]:
                    break

            ns.append({
                "text": s["text"], 
                "misp": ref["misp"], 
                "int": ref["int"], 
                "s": s["s"], 
                "t": s["t"],
            })
        o.append(ns)
        
    return o

mat = []
for p1 in range(5):
    row = []
    for p2 in range(5):
        if p2 >= p1:
            row.append(0)
            continue
        
        d1 = data[p1]
        d2 = data[p2]

        d = d1.merge(d2, on="data", how="inner")
        s1 = []
        s2 = []
        for idx, sent in d.iterrows():
            seg1 = get_segments(sent["data"], sent["label_x"])
            seg2 = get_segments(sent["data"], sent["label_y"])
            seg = overlap_segments(sent["data"], [seg1, seg2])
            
            for s in seg[0]:
                if not s["misp"]:
                    s1.append(0)
                elif s["int"]:
                    s1.append(1)
                else:
                    s1.append(2)
                    
            for s in seg[1]:
                if not s["misp"]:
                    s2.append(0)
                elif s["int"]:
                    s2.append(1)
                else:
                    s2.append(2)
            

        row.append(cohen_kappa_score(s1, s2))
        print(p1, p2, cohen_kappa_score(s1, s2), (np.array(s1)!=np.array(s2)).sum()*100/len(s1))
        
    mat.append(row)
# -

mask = np.triu(np.ones_like(mat))
# # plotting a triangle correlation heatmap
dataplot = sb.heatmap(mat, cmap="YlGnBu", annot=True, mask=mask, vmin=0, vmax=1)
fig = dataplot.get_figure()
# fig.savefig("Figures/iaa2.png")



# # Intention Labelling Entropy across Sentences

from pythainlp.tokenize import word_tokenize
engine = "deepcut"
# word_tokenize("ฉันรักแมว", engine=engine)

reference = load_jsonl(f"{DIR}/missplling_train_wisesight_samples.jsonl")

reference = pd.DataFrame(reference)

O = reference
for i, d in enumerate(data):
    d = d[["data", "label"]]
    d.columns = ["text", f"misp{i}"]
    O = O.merge(d, on="text", how="left")



# +
import copy
from tqdm import tqdm
from itertools import groupby

def norm_word(word):
    groups = [list(s) for _, s in groupby(word)]
    ch = []
    extraToken = ""
    for g in groups:
        if len(g)>=3:
            extraToken = "<rep>"
        ch.append(g[0])  
        
    word = "".join(ch)+extraToken
    return word

def tolabel(n):
    if n==2:
        return "neg"
    elif n==1:
        return "neu"
    elif n==0:
        return "pos"
    else:
        raise(f"Unknow label: {n}")
        
merged = []
for idx, row in tqdm(O.iterrows(), total=len(O)):
    segs = []
    for i in range(5):
        if pd.isna([row[f"misp{i}"]]).all():
            seg = get_segments(row["text"], [])
        else:
            seg = get_segments(row["text"], row[f"misp{i}"])
        
        segs.append(seg)
    
#     seg2 = get_segments(sent["data"], sent["label_y"])
    o = overlap_segments(row["text"], segs)
    tokens = []
    # if row["text"]=="อีดออกกก ฟังเพลงล้ะมีโฆษณาเอ็มเคชีสซิ๊ดเเซ่บคือเหี้ยใร":
    #     print(o)
    for i in range(len(o[0])):
        s = copy.copy(o[0][i])
        
        mispProb = 0
        intProb = 0
        # assert(len(o)==5)
        for j in range(len(o)):
            if (o[j][i]["misp"]):
                mispProb += 1/3
            
            if (o[j][i]["int"]):
                intProb += 1/3
                
        assert(mispProb <= 1)
        assert(intProb <= 1)
        
        if (mispProb < 0.5) and (intProb < 0.5):
            continue
        

        s["int"] = intProb
        s["msp"] = mispProb
        
        if s["text"]=="ใร":
            print(s, row["text"])
        # s["misp"] = s["text"]
        # del s["text"]
        # s["int"] = (intProb > 0.5)
        
        
#         s["tokens"] = word_tokenize(s["text"], engine=engine)
        s["corr"] = None
        tokens.append(s)

    merged.append({
        "text": row["text"],
        "category": tolabel(row["meta"]["category"]),
        "misp_tokens": tokens
    })
    
merged = pd.DataFrame(merged)

# {"corr": "ไหม", "misp": "มั้ย", "int": true, "s": 67, "t": 71}
# -

tokenized = load_jsonl(f"{DIR}/../tokenized_train-misp-3000.jsonl")
tokenized = pd.DataFrame(tokenized)

# +
# merged

# +
# tokenized
# -

sents = merged.merge(tokenized[["text", "segments"]], on="text")

# +
from collections import defaultdict

cnt = defaultdict(int)
cntmsp = defaultdict(list)
cntint = defaultdict(list)

def cal_entropy(labels):
    s = pd.Series(labels)
    counts = s.value_counts()
    return entropy(counts)

for idx, sent in sents.iterrows():
    for seg in sent["segments"]:
        for w in seg[0]:
            cnt[norm_word(w)] += 1
            
    for m in sent["misp_tokens"]:
        norm = norm_word(m["text"])
        # cntmsp[norm].append(int(m["msp"] > 0.5))
        cntint[norm].append(int(m["int"] > 0.5))
        # cntint[norm].append(m["int"])

# +
from scipy.stats import entropy

mispconsis = {}
for m in cntint:
    if (cnt[m] < 5):
        continue 
        
    mispconsis[m] = cal_entropy(cntint[m])
# mispconsis
# -

mispconsis

values = {k: v for k, v in sorted(mispconsis.items(), key=lambda item: -item[1])}
x = []
y = []
for i, k in enumerate(values):
    x.append(k)
    y.append(values[k])
#     print(k, values[k])
#     if i > 10:
#         break



# +
# # import warnings
# # warnings.filterwarnings("ignore")

# import matplotlib.pyplot as plt
# import matplotlib.font_manager as fm

# font_dirs = ["../"]
# font_files = fm.findSystemFonts(fontpaths=font_dirs)
# for font_file in font_files:
#     fm.fontManager.addfont(font_file)

# +
# # set font
# import matplotlib.pyplot as plt
# plt.rcParams['font.family'] = 'TH Sarabun New'
# plt.rcParams['xtick.labelsize'] = 20.0
# plt.rcParams['ytick.labelsize'] = 20.0

# # mx = len(x)
# # plt.ylim(0, 1)
# # plt.xticks(rotation=90)
# # plt.rcParams["figure.figsize"] = (20,3)
# # plt.xticks([], [])
# # plt.bar(x[0:mx], y[0:mx])

# +
# y

# +
plt.rcParams["figure.figsize"] = (10,6)
plt.rcParams.update({'font.size': 25})

plt.ylabel("No. misspelt words")
plt.xlabel("Entropy of the annotated labels (intentional/unintentional)")
plt.hist(y, bins=10)
plt.savefig('../Figures/int_entropy.png')

# +
# save_jsonl(merged, f"{DIR}/train-misp-3000.jsonl")

# +
# from collections import defaultdict
# labels = defaultdict(int)

# for sent in unified:
#     labels[sent["category"]] += 1
# #     break

# labels
# -



# # Term Frequency



# +
from collections import defaultdict

cnt = defaultdict(int)
misp = defaultdict(int)

for idx, sent in sents.iterrows():
    mispFound = False
    for seg in sent["segments"]:
        for m, c in zip(seg[0], seg[1]):
            if m!=c:
                mispFound = True
                cnt["misp"] += 1
                n = norm_word(m)
                misp[n] += 1
            cnt["token"] += 1
            
    if mispFound:
        cnt["sent"] += 1
    # # break?

# +
print("%Mispelling Sentence:", cnt["sent"]*100/len(sents), cnt["sent"])

print()
print("#Misspelling:", cnt["misp"])
print("%Misspelling:", cnt["misp"]*100/cnt["token"])
print("#Unique Misspelling Tokens:", len(misp))

# +
# from transformers import XLMRobertaTokenizerFast
# tokenizer = XLMRobertaTokenizerFast.from_pretrained('xlm-roberta-base')

# import itertools

# cnt["misp_sub"] = 0
# cnt["corr_sub"] = 0
# unk = tokenizer.convert_tokens_to_ids(["<unk>"])[0]
# for sent in trainmisp:
#     s = [list(zip(seg[0], seg[1])) for seg in sent["segments"]]
#     tokens = list(itertools.chain(*s))
# #     misptokens = [t[0] for t in tokens]
# #     midx = tokenizer.convert_tokens_to_ids(misptokens)
# #     for i in range(len(midx)):
# #         if midx[i]==unk:
# #             t = tokenizer.tokenize("_"+misptokens[i])[1:]
# #             cnt["misp_sub"] += len(t)
# #         else:
# #             cnt["misp_sub"] += 1
    
#     cnt["misp_sub"] += len(tokens)
#     cnt["corr_sub"] += len(tokenizer.tokenize(sent["text"]))
#     # break

# print("%Subtokens Different", abs(cnt["misp_sub"]-cnt["corr_sub"])*100/cnt["corr_sub"])
# -

# ## Most common misspelling words

# +
sortedmisp = {k: v for k, v in sorted(misp.items(), key=lambda item: -item[1])}

for i, k in enumerate(sortedmisp):
    print(k, sortedmisp[k])
    if i > 10:
        break
# -



# +
x = [x for x in sortedmisp]
y = [sortedmisp[k] for k in sortedmisp]

mint = [int(np.average(cntint[k]) > 0.5) for k in sortedmisp]
c = ['b' if i==1 else 'r' for i in mint]

mx = 100
# plt.ylim(0, 1.2)
# plt.xticks(rotation=90)
# plt.rcParams["figure.figsize"] = (20,3)
plt.xticks([], [])
plt.bar(x[0:mx], y[0:mx], color=c[0:mx])
plt.ylabel("Frequency")
plt.xlabel(f"\nMisspelling terms (top {mx})")
plt.savefig("../Figures/tf.png")


# _x = np.array(range(len(x)))
# _y = (np.full(len(x), y[0]))/(_x+1) 
# plt.plot(_x[0:mx], _y[0:mx], "r")
# -

print("Common mispelling")
cc = 0
obs = 0
for i in range(len(x)):
    if cc <= 20:
        print(cc, x[i], y[i], 'int' if mint[i]==1 else 'un')
    cc += 1
    obs += y[i]

# +
print("Common intentional mispelling")
cc = 0
obs = 0
for i in range(len(x)):
    if mint[i]==1:
        if cc <= 15:
            print(cc, x[i], y[i])
        cc += 1
        obs += y[i]

print("#Intentional Words:", cc, obs)
# -

print("Common unintentional mispelling")
cc = 0
obs = 0
for i in range(len(x)):
    if mint[i]!=1:
        if cc <= 10:
            print(x[i], y[i])
        cc += 1
        obs += y[i]
print("#Unintentional Words:", cc, obs)



# # Sentiment Class and Misspelling

# +
from collections import defaultdict
from itertools import groupby

labels = defaultdict(int)
for idx, sent in sents.iterrows():
    mispFound = False
    for seg in sent["segments"]:
        for m, c in zip(seg[0], seg[1]):
            if m!=c:
                mispFound = True
    if mispFound:
        labels[sent["category"]] += 1

for l in labels:
    print(l, labels[l], labels[l]/cnt["sent"])
# -



StarcoderdataJupyterScriptsDedupFiltered