# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: 'Python 3.7.11 64-bit (''sem_eval'': conda)'
#     language: python
#     name: python3
# ---

try:
    from google.colab import drive
    drive.mount('/content/drive')
except ModuleNotFoundError as e:
    print("not in colab")
    pass
import os
base_dir = "/content/drive/MyDrive/semeval2022"
if not os.path.exists(base_dir):
  # !pip install -r requirements.txt
  base_dir = ""
else:
  # !pip install -r /content/drive/MyDrive/semeval2022/requirements.txt
  # !cp -rf /content/drive/MyDrive/semeval2022/*.py . 
  # !cp -rf /content/drive/MyDrive/semeval2022/utils .
  # !cp -rf /content/drive/MyDrive/semeval2022/model .

from utils.util import parse_args, get_reader, load_model, get_trainer, get_out_filename, write_eval_performance, get_tagset
import time
from torch.utils.data import DataLoader
from utils.util import wnut_iob
import os

checkpoint = os.path.join(base_dir, "roberta-train/lightning_logs/version_0/checkpoints/epoch=4-step=4784.ckpt")
checkpoint = os.path.join(base_dir, "roberta-train/lightning_logs/version_0/checkpoints/roberta-train_timestamp_1638429490.282762_final.ckpt")
#checkpoint = "./roberta-finetune/lightning_logs/version_0/checkpoints/"
checkpoint = os.path.join(base_dir, "roberta-finetune/lightning_logs/version_1/checkpoints/finetune_timestamp_1638516609.385282_final.ckpt")
train_file = os.path.join(base_dir, "training_data/EN-English/en_train.conll")
dev_file = os.path.join(base_dir, "training_data/EN-English/en_dev.conll")
output_dir = os.path.join(base_dir, "roberta-evaluate")
if not os.path.exists(output_dir):
    os.makedirs(output_dir)
iob_tagging = wnut_iob

test_data = get_reader(file_path=dev_file, target_vocab=wnut_iob, max_instances=-1, max_length=55, encoder_model="roberta-base")
model = load_model(checkpoint, tag_to_id=wnut_iob)
trainer = get_trainer(is_test=True)
out = trainer.test(model, test_dataloaders=DataLoader(test_data, batch_size=16, collate_fn=model.collate_batch))
eval_file, eval_detail_file = get_out_filename(output_dir, checkpoint, prefix="eval")
write_eval_performance(out, eval_file)

# +
batch_size = 8
test_dataloader = DataLoader(test_data, batch_size=batch_size, collate_fn=model.collate_batch)
sentences = test_data.sentences
ner_tags = test_data.ner_tags
pos_to_singel_word_map = test_data.pos_to_single_word_maps
f = open(eval_detail_file, "w")
f.write("token\tlabel\tpred\n")
f2 = open("en.pred.conll", "w")
for idx, batch in enumerate(test_dataloader):
    output = model.perform_forward_step(batch)
    pred_result = output["pred_results"]
    raw_pred_results = output["raw_pred_results"]
    for i in range(batch_size):
        sentence = sentences[idx*batch_size+i]
        pos_to_singel_word = pos_to_singel_word_map[idx*batch_size+i]
        ner_tag = ner_tags[idx*batch_size+i]
        input_ids = batch[0][i]
        pred_token_tag = pred_result[i]
        raw_pred_token_tag = raw_pred_results[i]
        metadata_token_tag = batch[3][i]
        meta_labels = []
        pred_labels = []
        sentence_subtokens = []
        for (start_pos, end_pos), (pred_start_pos, pred_end_pos) in zip(metadata_token_tag, pred_token_tag):
            sub_tokens = test_data.tokenizer.convert_ids_to_tokens(input_ids[start_pos: end_pos+1])
            sentence_subtokens.extend(sub_tokens)
            pred_sub_tokens = test_data.tokenizer.convert_ids_to_tokens(input_ids[start_pos: end_pos+1])
            tag = metadata_token_tag[(start_pos, end_pos)]
            pred_tag = pred_token_tag[(pred_start_pos, pred_end_pos)]
            for sub_token1, sub_token2 in zip(sub_tokens, pred_sub_tokens):
                meta_labels.append(tag)
                pred_labels.append(pred_tag)
                #f.write("{}{}{}{}{}{}{}".format(sub_token1, ",", tag, ",", sub_token2, ",", pred_tag))
                #f.write("\n")
        for (start, end) in pos_to_singel_word:
            single_word_tokens = sentence_subtokens[start:end]
            word = "".join(single_word_tokens)
            if word == "almada":
                print(sentence_subtokens)
                print(pos_to_singel_word)
            word_meta_tag = ner_tag[start]
            word_pred_tag = raw_pred_token_tag[start]
            f.write("{}\t{}\t{}".format(word, word_meta_tag, word_pred_tag))
            f2.write("{}\n".format(word_pred_tag))
            f.write("\n")
        f2.write("\n")
            
f.close() 
f2.close()
# -

import pandas as pd
result_data = pd.read_csv(eval_detail_file, delimiter="\t")

# +
error_data = result_data[result_data["label"] != result_data["pred"]]

print(len(error_data))
# -

test_data.tokenizer("among others , he was chairman of the standing conference of regional sports federations and member of the südwestrundfunk broadcasting council")

test_data.sentences[0].split(" ")

result_data



StarcoderdataJupyterScriptsDedupFiltered