# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="ITFHDGZalHiM" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 524} outputId="b2b1bdd9-362e-48b2-aeee-8e597fe00281"
import pandas as pd
# !pip install stanfordnlp
import stanfordnlp
# !pip install pytorch_pretrained_bert
import torch,gc

# + [markdown] id="VE30QzmGBeGC" colab_type="text"
# CoreNLP running on Python Server

# + id="Y2pQHQ_Vx4n8" colab_type="code" outputId="85201464-8d7e-4146-a759-24388cb1bcf9" colab={"base_uri": "https://localhost:8080/", "height": 1000}
# !echo "Downloading CoreNLP..."
# !wget "http://nlp.stanford.edu/software/stanford-corenlp-full-2018-10-05.zip" -O corenlp.zip
# !unzip corenlp.zip
# !mv ./stanford-corenlp-full-2018-10-05 ./corenlp

# Set the CORENLP_HOME environment variable to point to the installation location
import os
os.environ["CORENLP_HOME"] = "./corenlp"

# Import client module
from stanfordnlp.server import CoreNLPClient

# + id="jQ_xnuObx-Tq" colab_type="code" outputId="3e3ad2d0-ea3f-41fa-e68c-980ca9f0cbe4" colab={"base_uri": "https://localhost:8080/", "height": 70}
# Construct a CoreNLPClient with some basic annotators, a memory allocation of 4GB, and port number 9001
client = CoreNLPClient(annotators=['tokenize','ssplit', 'pos', 'lemma', 'ner'], memory='4G', endpoint='http://localhost:9001')
print(client)

# Start the background server and wait for some time
# Note that in practice this is totally optional, as by default the server will be started when the first annotation is performed
client.start()
import time; time.sleep(10)

# + [markdown] id="z7dcco3twlqN" colab_type="text"
# Add the Number of Rows to be considered as training data, test data and validation data in the input CSV which contains the training data

# + id="QGqQRV7dwq92" colab_type="code" colab={}
Trainrow = 1
testrow = 45
valrow = 49

# + [markdown] id="gxuT_e34DLnb" colab_type="text"
# Reading the input text from CSV file mentioned here. Change according to requirement.

# + id="Kn202rSM-MpG" colab_type="code" colab={}
import json
def format_to_lines():
    p_ct = 0
    i = 0
    for i in range(Trainrow):
        df_csv = pd.read_csv('./fincident.csv',encoding='cp1252')
        dataset = []
        d = _format_to_lines_new(df_csv,i)
        dataset.append(d)
        i += 1
        pt_file = "{:s}.{:s}.{:d}.json".format('bert_data', "train", p_ct)
        with open(pt_file, 'w') as save:
        # save.write('\n'.join(dataset))
          save.write(json.dumps(dataset))
          p_ct += 1
          print(p_ct)
          dataset = []



# + id="lKBI7ZxU8Qzf" colab_type="code" colab={}
REMAP = {"-lrb-": "(", "-rrb-": ")", "-lcb-": "{", "-rcb-": "}",
         "-lsb-": "[", "-rsb-": "]", "``": '"', "''": '"'}

import re
def clean(x):
    return re.sub(
        r"-lrb-|-rrb-|-lcb-|-rcb-|-lsb-|-rsb-|``|''",
        lambda m: REMAP.get(m.group()), x)
    
lower = True

def _format_to_lines_new(df_csv,i):
  source = []
  tgt = []
  flag = False
  print(df_csv['Texts'][i])
  ann = client.annotate(str(df_csv['Texts'][i]),annotators='tokenize,ssplit', output_format='json' )
  for sent in ann['sentences']:
    tokens = [t['word'] for t in sent['tokens']]
    if (lower):
        tokens = [t.lower() for t in tokens]
    if (tokens[0] == '@highlight'):
        flag = True
        continue
    if (flag):
        tgt.append(tokens)
        flag = False
    else:
        source.append(tokens)


  ann = client.annotate(str(df_csv['Analysis'][i]),annotators='tokenize,ssplit', output_format='json' )
  for sent in ann['sentences']:
    tokens = [t['word'] for t in sent['tokens']]
    if (lower):
        tokens = [t.lower() for t in tokens]
    if (tokens[0] == '@highlight'):
        flag = True
        continue
    if (flag):
        tgt.append(tokens)
        flag = False
    else:
        tgt.append(tokens)
    

  source = [clean(' '.join(sent)).split() for sent in source]
  tgt = [clean(' '.join(sent)).split() for sent in tgt]

  return {'src': source, 'tgt': tgt}

# + id="jM29IMF3JytO" colab_type="code" colab={}
from multiprocess import Pool
def format_to_bert(args):
    for i in range(Trainrow):
        a_lst = []
        for json_f in glob.glob(pjoin(args.raw_path, '*' + corpus_type + '.*.json')):
            real_name = json_f.split('/')[-1]
            a_lst.append((json_f, args, pjoin(args.save_path, real_name.replace('json', 'bert.pt'))))
        print(a_lst)
        pool = Pool(args.n_cpus)
        for d in pool.imap(_format_to_bert, a_lst):
            pass

        pool.close()
        pool.join()



# + [markdown] id="7WoQ2yY6Dc30" colab_type="text"
# Genereate JSON file of the trianing data from th CSV

# + id="KY9D4ibMw3Wy" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 205} outputId="9a2d0428-e368-4ea9-f98e-1616a192f4a2"
format_to_lines()


# + [markdown] id="ySCVJ3m4DpvF" colab_type="text"
# Different Modules which are required below for Preprocessing

# + id="Yd9SbZoMP17y" colab_type="code" colab={}
def _get_word_ngrams(n, sentences):
    """Calculates word n-grams for multiple sentences.
    """
    assert len(sentences) > 0
    assert n > 0

    # words = _split_into_words(sentences)

    words = sum(sentences, [])
    # words = [w for w in words if w not in stopwords]
    return _get_ngrams(n, words)


# + id="JnxFEX8zP8VM" colab_type="code" colab={}
def _get_ngrams(n, text):
    """Calcualtes n-grams.
    Args:
      n: which n-grams to calculate
      text: An array of tokens
    Returns:
      A set of n-grams
    """
    ngram_set = set()
    text_length = len(text)
    max_index_ngram_start = text_length - n
    for i in range(max_index_ngram_start + 1):
        ngram_set.add(tuple(text[i:i + n]))
    return ngram_set


# + id="GzIYN1pDPnQ6" colab_type="code" colab={}
def greedy_selection(doc_sent_list, abstract_sent_list, summary_size):
    def _rouge_clean(s):
        return re.sub(r'[^a-zA-Z0-9 ]', '', s)

    max_rouge = 0.0
    abstract = sum(abstract_sent_list, [])
    abstract = _rouge_clean(' '.join(abstract)).split()
    sents = [_rouge_clean(' '.join(s)).split() for s in doc_sent_list]
    evaluated_1grams = [_get_word_ngrams(1, [sent]) for sent in sents]
    reference_1grams = _get_word_ngrams(1, [abstract])
    evaluated_2grams = [_get_word_ngrams(2, [sent]) for sent in sents]
    reference_2grams = _get_word_ngrams(2, [abstract])

    selected = []
    for s in range(summary_size):
        cur_max_rouge = max_rouge
        cur_id = -1
        for i in range(len(sents)):
            if (i in selected):
                continue
            c = selected + [i]
            candidates_1 = [evaluated_1grams[idx] for idx in c]
            candidates_1 = set.union(*map(set, candidates_1))
            candidates_2 = [evaluated_2grams[idx] for idx in c]
            candidates_2 = set.union(*map(set, candidates_2))
            rouge_1 = cal_rouge(candidates_1, reference_1grams)['f']
            rouge_2 = cal_rouge(candidates_2, reference_2grams)['f']
            rouge_score = rouge_1 + rouge_2
            if rouge_score > cur_max_rouge:
                cur_max_rouge = rouge_score
                cur_id = i
        if (cur_id == -1):
            return selected
        selected.append(cur_id)
        max_rouge = cur_max_rouge

    return sorted(selected)



# + id="mIOJn85cQFKy" colab_type="code" colab={}
def cal_rouge(evaluated_ngrams, reference_ngrams):
    reference_count = len(reference_ngrams)
    evaluated_count = len(evaluated_ngrams)

    overlapping_ngrams = evaluated_ngrams.intersection(reference_ngrams)
    overlapping_count = len(overlapping_ngrams)

    if evaluated_count == 0:
        precision = 0.0
    else:
        precision = overlapping_count / evaluated_count

    if reference_count == 0:
        recall = 0.0
    else:
        recall = overlapping_count / reference_count

    f1_score = 2.0 * ((precision * recall) / (precision + recall + 1e-8))
    return {"f": f1_score, "p": precision, "r": recall}


# + id="yw0Sl56pPbJH" colab_type="code" colab={}
from pytorch_pretrained_bert import BertTokenizer


# + [markdown] id="GWzzl_OCE1yh" colab_type="text"
# Execute the below code for Preprocessing for PreSumm Only

# + id="E_DZhWoEaXDV" colab_type="code" colab={}
# For PreSumm

class BertData():
    def __init__(self):
        self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)

        self.sep_token = '[SEP]'
        self.cls_token = '[CLS]'
        self.pad_token = '[PAD]'
        self.tgt_bos = '[unused0]'
        self.tgt_eos = '[unused1]'
        self.tgt_sent_split = '[unused2]'
        self.sep_vid = self.tokenizer.vocab[self.sep_token]
        self.cls_vid = self.tokenizer.vocab[self.cls_token]
        self.pad_vid = self.tokenizer.vocab[self.pad_token]

    def preprocess(self, src, tgt, sent_labels, use_bert_basic_tokenizer=False, is_test=False):

        if ((not is_test) and len(src) == 0):
            print("returned none")
            return None

        original_src_txt = [' '.join(s) for s in src]

        idxs = [i for i, s in enumerate(src) if (len(s) > 5)]

        _sent_labels = [0] * len(src)
        for l in sent_labels:
            _sent_labels[l] = 1

        src = [src[i][:200] for i in idxs]
        sent_labels = [_sent_labels[i] for i in idxs]
        src = src[:200]
        sent_labels = sent_labels[:200]

        if ((not is_test) and len(src) < 3):
            print("returned none")
            return None

        src_txt = [' '.join(sent) for sent in src]
        text = ' {} {} '.format(self.sep_token, self.cls_token).join(src_txt)

        src_subtokens = self.tokenizer.tokenize(text)

        src_subtokens = [self.cls_token] + src_subtokens + [self.sep_token]
        src_subtoken_idxs = self.tokenizer.convert_tokens_to_ids(src_subtokens)
        _segs = [-1] + [i for i, t in enumerate(src_subtoken_idxs) if t == self.sep_vid]
        segs = [_segs[i] - _segs[i - 1] for i in range(1, len(_segs))]
        segments_ids = []
        for i, s in enumerate(segs):
            if (i % 2 == 0):
                segments_ids += s * [0]
            else:
                segments_ids += s * [1]
        cls_ids = [i for i, t in enumerate(src_subtoken_idxs) if t == self.cls_vid]
        sent_labels = sent_labels[:len(cls_ids)]

        tgt_subtokens_str = '[unused0] ' + ' [unused2] '.join(
            [' '.join(self.tokenizer.tokenize(' '.join(tt))) for tt in tgt]) + ' [unused1]'
        tgt_subtoken = tgt_subtokens_str.split()[:200]
        if ((not is_test) and len(tgt_subtoken) < 20):
            print("returned none <20")
            print(len(tgt_subtoken))
            return None

        tgt_subtoken_idxs = self.tokenizer.convert_tokens_to_ids(tgt_subtoken)

        tgt_txt = '<q>'.join([' '.join(tt) for tt in tgt])
        src_txt = [original_src_txt[i] for i in idxs]

        return src_subtoken_idxs, sent_labels, tgt_subtoken_idxs, segments_ids, cls_ids, src_txt, tgt_txt


# + [markdown] id="Z9xGn4a-E5Yi" colab_type="text"
# Execute the below code for Preprocessing for BertSumm Only

# + id="056VQK9NOruw" colab_type="code" colab={}
# For BertSum
class BertData():
    def __init__(self):
        self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
        self.sep_vid = self.tokenizer.vocab['[SEP]']
        self.cls_vid = self.tokenizer.vocab['[CLS]']
        self.pad_vid = self.tokenizer.vocab['[PAD]']

    def preprocess(self, src, tgt, oracle_ids):

        if (len(src) == 0):
            return None

        original_src_txt = [' '.join(s) for s in src]

        labels = [0] * len(src)
        for l in oracle_ids:
            labels[l] = 1

        idxs = [i for i, s in enumerate(src) if (len(s) > 5)]

        src = [src[i][:200] for i in idxs]
        labels = [labels[i] for i in idxs]
        src = src[:200]
        labels = labels[:200]

        print("length of label")
        print(len(labels))
        if (len(src) < 10):
            return None
        if (len(labels) == 0):
            return None

        src_txt = [' '.join(sent) for sent in src]
        # text = [' '.join(ex['src_txt'][i].split()[:self.args.max_src_ntokens]) for i in idxs]
        # text = [_clean(t) for t in text]
        text = ' [SEP] [CLS] '.join(src_txt)
        src_subtokens = self.tokenizer.tokenize(text)
        src_subtokens = src_subtokens[:510]
        src_subtokens = ['[CLS]'] + src_subtokens + ['[SEP]']

        src_subtoken_idxs = self.tokenizer.convert_tokens_to_ids(src_subtokens)
        _segs = [-1] + [i for i, t in enumerate(src_subtoken_idxs) if t == self.sep_vid]
        segs = [_segs[i] - _segs[i - 1] for i in range(1, len(_segs))]
        segments_ids = []
        for i, s in enumerate(segs):
            if (i % 2 == 0):
                segments_ids += s * [0]
            else:
                segments_ids += s * [1]
        cls_ids = [i for i, t in enumerate(src_subtoken_idxs) if t == self.cls_vid]
        labels = labels[:len(cls_ids)]

        tgt_txt = '<q>'.join([' '.join(tt) for tt in tgt])
        src_txt = [original_src_txt[i] for i in idxs]
        return src_subtoken_idxs, labels, segments_ids, cls_ids, src_txt, tgt_txt



# + [markdown] id="dKHpehAsEtX0" colab_type="text"
# Execute the below code for Preprocessing for PreSumm Only

# + id="_am-jsJYMm9m" colab_type="code" colab={}
## For PreSumm Only

def _format_to_bert():
  p_ct = 0
  for i in range(Trainrow):
    pt_file = "{:s}.{:s}.{:d}.json".format('bert_data', "train", p_ct)
    json_file = pt_file
    save_file = pt_file.replace('.json','.pt')
    p_ct += 1
    bert = BertData()

    jobs = json.load(open(json_file))
    datasets = []
    for d in jobs:
        source, tgt = d['src'], d['tgt']
        print("inside")
        oracle_ids = greedy_selection(source, tgt, 3)
        #elif (args.oracle_mode == 'combination'):
         #   oracle_ids = combination_selection(source, tgt, 3)
        print(oracle_ids)
        b_data = bert.preprocess(source, tgt, oracle_ids)
        if (b_data is None):
            print("None")
            continue
        src_subtoken_idxs, sent_labels, tgt_subtoken_idxs, segments_ids, cls_ids, src_txt, tgt_txt = b_data
        #for BertSumm
        #indexed_tokens, labels, segments_ids, cls_ids, src_txt, tgt_txt = b_data
        # for BertSumm
        #b_data_dict = {"src": indexed_tokens, "labels": labels, "segs": segments_ids, 'clss': cls_ids,
        #               'src_txt': src_txt, "tgt_txt": tgt_txt}
        b_data_dict = {"src": src_subtoken_idxs, "tgt": tgt_subtoken_idxs,
                       "src_sent_labels": sent_labels, "segs": segments_ids, 'clss': cls_ids,
                       'src_txt': src_txt, "tgt_txt": tgt_txt}
        print(b_data_dict)
        datasets.append(b_data_dict)
    torch.save(datasets, save_file)
    datasets = []
    gc.collect()


# + [markdown] id="eTHKvuh2EypV" colab_type="text"
# Execute the below code for Preprocessing for PreSumm Only

# + id="8GAvF7YdESPq" colab_type="code" colab={}
## For BertSumm Only

def _format_to_bert():
  p_ct = 0
  for i in range(Trainrow):
    pt_file = "{:s}.{:s}.{:d}.json".format('bert_data', "train", p_ct)
    json_file = pt_file
    save_file = pt_file.replace('.json','.pt')
    p_ct += 1
    bert = BertData()

    jobs = json.load(open(json_file))
    datasets = []
    for d in jobs:
        source, tgt = d['src'], d['tgt']
        print("inside")
        oracle_ids = greedy_selection(source, tgt, 3)
        #elif (args.oracle_mode == 'combination'):
         #   oracle_ids = combination_selection(source, tgt, 3)
        print(oracle_ids)
        b_data = bert.preprocess(source, tgt, oracle_ids)
        if (b_data is None):
            print("None")
            continue
        #for BertSumm
        indexed_tokens, labels, segments_ids, cls_ids, src_txt, tgt_txt = b_data
        # for BertSumm
        b_data_dict = {"src": indexed_tokens, "labels": labels, "segs": segments_ids, 'clss': cls_ids,
                       'src_txt': src_txt, "tgt_txt": tgt_txt}
        print(b_data_dict)
        datasets.append(b_data_dict)
    torch.save(datasets, save_file)
    datasets = []
    gc.collect()


# + id="rW7Ugu37EK8H" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 87} outputId="90d2b3f2-5212-40dc-cd41-7a211d680744"
_format_to_bert()

StarcoderdataJupyterScriptsDedupFiltered