"""
# About

This notebook added an splendid `External Dataset` shared by [@mlconsult](https://www.kaggle.com/mlconsult) to `MLM` shared by [@tungmphung](https://www.kaggle.com/tungmphung). Thanks to everyone willing to share their knowledge, let's make the DS community better ;)

**Please check the original notebooks and discussion here:**
- [Matching + Baseline + MLM w/ Evaluation Metric](https://www.kaggle.com/chienhsianghung/matching-baseline-mlm-w-evaluation-metric)
- [Coleridge Initiative| Naive CV (Local Validation)](https://www.kaggle.com/chienhsianghung/coleridge-initiative-naive-cv-local-validation)
- [Evaluation metric implementation](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/discussion/230091)
- [[Coleridge] Predict with Masked Dataset Modeling](https://www.kaggle.com/tungmphung/coleridge-predict-with-masked-dataset-modeling)
- [score 57ish with additional govt datasets](https://www.kaggle.com/mlconsult/score-57ish-with-additional-govt-datasets)

**Previous results from:**
[Matching + Baseline + MLM w/ Evaluation Metric](https://www.kaggle.com/chienhsianghung/matching-baseline-mlm-w-evaluation-metric)

|   | CV | LB |
| --- | --- | --- |
| og | 0.604 | 0.536 |
| og s1.5 |   | 0.535 |
| og s2.5 |   | 0.535 |
| pred_mlm_labels | 0.108 |   |
| lables_list | 0.548 |   |
| all | 0.261 |   |
| og2all | 0.454 |   |
| 150 400 |   | 0.536 |
| 200 300 |   | 0.536 |
| 2021 |   | 0.536 |
| 42 |   | 0.536 |
| 42 s1.5 |   | 0.535 |
| 42 .70 |   | 0.536 |
| 42 .80 |   | 0.536 |
| MATCH_ONLY |   | 0.533 |
| BASELINE_HELPING |   | 0.536 |
| 1 |   | 0.536 |
"""
"""
# Setting
"""
"""
# Install packages
"""
COMPUTE_CV = False
BS_CLEANING = False
eng_stws = []
with open('../input/engst-06061111/englishST.txt','r') as fff:
    for line in fff:
        line = line.strip().lower()
        if line != '':
            eng_stws.append(line)
eng_stws = list(set(eng_stws))
!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets
!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl
!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl
!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl


from IPython.display import clear_output
clear_output()
!pip install ../input/abb-whl/abbreviations-0.2.5-py3-none-any.whl

# !pip install ../input/simple-transformers-package/simpletransformers/ipykernel-5.5.5-py3-none-any.whl
# !pip install ../input/simple-transformers-package/simpletransformers/pydeck-0.6.2-py2.py3-none-any.whl
# !pip install ../input/simple-transformers-package/simpletransformers/streamlit-0.82.0-py2.py3-none-any.whl
# !pip install ../input/simple-transformers-package/simpletransformers/simpletransformers-0.61.5-py3-none-any.whl
!pip install --no-dependencies ../input/simple-transformers-package/simpletransformers/simpletransformers-0.61.5-py3-none-any.whl
import json
with open('../input/ner-dataset-for-show-us-the-data/ner_distant.json', 'r') as f:
    js = json.load(f)
print(js.keys())
js['sentences_not_including_dataset_all'][:10]
js['sentences_including_annotations_all'][:10]
from simpletransformers.ner import NERModel
import pandas as pd
import logging
import torch

# ログの設定
logging.basicConfig(level=logging.INFO)
transformers_logger = logging.getLogger("transformers")
transformers_logger.setLevel(logging.WARNING)
ner_model = NERModel(
    "roberta", "../input/ner-0605-st/content/outputs/best_model", use_cuda=True if torch.cuda.is_available() else False
)
predictions, raw_outputs = ner_model.predict(["Hermione was the best dataset study national in her class",
                                         "Final dataset study and its survey apperaed", "slosh model is great"])
predictions
# train_params = {
#     'max_seq_length': 512,
#     'max_length': 128,
#     'train_batch_size': 32,
#     'eval_batch_size': 32,
#     'num_train_epochs': 2,
#     'evaluate_during_training': True,
#     'evaluate_during_training_steps': 10000,
#     'use_multiprocessing': False,
#     'fp16': False,
#     'save_steps': -1,
#     'save_eval_checkpoints': False,
#     'save_model_every_epoch': False,
#     'no_cache': True,
#     'reprocess_input_data': True,
#     'overwrite_output_dir': True,
#     'preprocess_inputs': False,
#     'num_return_sequences': 1 ,
#     "output_dir": "./ner_outputs/",
#     "cache_dir": "./ner_cache_dir/",
# }

# model = NERModel(
#     "roberta", "roberta-base", use_cuda=True if torch.cuda.is_available() else False,
#     labels=['O','B-DATASET','I-DATASET'], args=train_params
# )
# import math, copy
# train_ner_num = math.floor(len(train_data) * 0.9)
# train_data_for_ner, test_data_for_ner = train_data[:train_ner_num], train_data[train_ner_num:]

# train_for_smpformer, test_data_for_smpformer = [], []

# for d in train_data_for_ner:
#     train_for_smpformer += d

# test_data_first_idx = copy.copy(test_data_for_ner[0][0][0])

# for d in test_data_for_ner:
#     for one_anno in d:
#         test_data_for_smpformer.append([one_anno[0] - test_data_first_idx,
#                                        one_anno[1], one_anno[2]])
# import math
# train_for_smpformer = pd.DataFrame(train_for_smpformer, columns=["sentence_id", "words", "labels"])
# test_data_for_smpformer = pd.DataFrame(test_data_for_smpformer, columns=["sentence_id", "words", "labels"])

# model.train_model(train_for_smpformer, eval_data=test_data_for_smpformer)

# # Evaluate the model
# print('eval ner')
# result, model_outputs, preds_list = model.eval_model(test_data_for_smpformer)
# predictions, raw_outputs = model.predict(["Hermione was the best dataset study national in her class",
#                                          "study"])

# import os
# from zipfile import ZipFile

# dirName = "./"
# zipName = "packages.zip"

# # Create a ZipFile Object
# with ZipFile(zipName, 'w') as zipObj:
#     # Iterate over all the files in directory
#     for folderName, subfolders, filenames in os.walk(dirName):
#         for filename in filenames:
#             if (filename != zipName):
#                 # create complete filepath of file in directory
#                 filePath = os.path.join(folderName, filename)
#                 # Add file to zip
#                 zipObj.write(filePath)
"""
# Import
"""
import os
import re
import json
import time
import datetime
import random
import glob
import importlib

import numpy as np
import pandas as pd

from tqdm.autonotebook import tqdm
import matplotlib.pyplot as plt
import seaborn as sns

import torch
from datasets import load_dataset
from transformers import AutoTokenizer, DataCollatorForLanguageModeling, \
AutoModelForMaskedLM, Trainer, TrainingArguments, pipeline

from typing import List
import string
from functools import partial
SEED=43
sns.set()
random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
"""
# Load data
"""
"""
# Literal Matching
"""
"""
### Create a Knowledge Bank
"""
all_labels = set()
common_data_words = ['study', 'adni', 'disease', 'longitudinal', 'neuroimaging', 'initiative', 
                     'alzheimer', 'international', 'science', 'mathematics', 'trends', 'aging', 
                     'baltimore', 'early', 'childhood', 'survey', 'national', 'continuum', 
                     'codes', 'rural', 'urban', ' adni ', 'data', 'common', 'model', 'trends',
                     'assesments','assesment','datasets','trend', 'research', 'code',
                     'databases',
                     'ibtracs', 'noaa', 'postsecondary', 'beginning', 'management', 'agricultural', 
                     'assessment', 'resource', 'students', 'optimum', 'interpolation', 'surface',
                     'temperature', 'database', 'program', 'census', 'enumeration', 'poll', 'polls', 'demographics',
                     'doctorates', 'progress', 'school', 'tide', 'coronavirus', 'analysis', 'dataset', 'analyze',
                     'cohort', 'cohorts']
critical_words = ['survey', 'study', 'data', 'database', 'dataset', 'databases', 'datasets',
                 'cohort', 'cohorts', 'program', 'programs', 'trand', 'trands', 'assessment', 'assessments',
                 'analysis', 'analytics', 'census', 'research', 'census', 'model']
with open('../input/filtered-public-data-csv/filered.csv', 'r') as fpd:
    for line in fpd:
        line = line.strip()
        if line != '' and len(line.split(' ')) > 2:
            split_w = line.split(' ')
            if split_w[0] in eng_stws:
                continue
            if split_w[-1] in eng_stws:
                continue
            if len(set(split_w) & set(common_data_words)) > 0: 
                all_labels.add(str(line).lower())
"""
#### Additional Govt Datasets
"""
adnl_govt_labels = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')

for line in adnl_govt_labels.title:
    line = line.strip().lower()
    if line != '' and len(line.split(' ')) > 0:
        split_w = line.split(' ')
        if split_w[0] in eng_stws:
            continue
        if split_w[-1] in eng_stws:
            continue
        all_labels.add(str(line).lower())    
    
all_labels = set(all_labels)
print(f'No. different labels: {len(all_labels)}')
"""
### Matching on test data
"""
def clean_text(txt):
    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()


def totally_clean_text(txt):
    txt = clean_text(txt)
    txt = re.sub(' +', ' ', txt)
    return txt

if not BS_CLEANING:
    def text_cleaning(text):
        '''
        Converts all text to lower case, Removes special charecters, emojis and multiple spaces
        text - Sentence that needs to be cleaned
        '''
        text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()
        text = re.sub(' +', ' ', text)
        emoji_pattern = re.compile("["
                                   u"\U0001F600-\U0001F64F"  # emoticons
                                   u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                                   u"\U0001F680-\U0001F6FF"  # transport & map symbols
                                   u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                                   "]+", flags=re.UNICODE)
        text = emoji_pattern.sub(r'', text)
        return text
else:
    def text_cleaning(text):
        '''
        Converts all text to lower case, Removes special charecters, emojis and multiple spaces
        text - Sentence that needs to be cleaned
        '''
        text = ''.join([k for k in text if k not in string.punctuation])
        text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()
        # text = re.sub("/'+/g", ' ', text)
        return text


def read_json_pub(filename, train_data_path, output='text'):
    json_path = os.path.join(train_data_path, (filename+'.json'))
    headings = []
    contents = []
    combined = []
    with open(json_path, 'r') as f:
        json_decode = json.load(f)
        for data in json_decode:
            headings.append(data.get('section_title'))
            contents.append(data.get('text'))
            combined.append(data.get('section_title'))
            combined.append(data.get('text'))
    
    all_headings = ' '.join(headings)
    all_contents = ' '.join(contents)
    all_data = '. '.join(combined)
    
    if output == 'text':
        return all_contents
    elif output == 'head':
        return all_headings
    else:
        return all_data
"""
#### Ken Matching
"""
"""
# Masked Dataset Modeling
"""
"""
### Paths and Hyperparameters
"""
"""
# Transform data to MLM format
"""
"""
### Load model and tokenizer
"""
"""
### Auxiliary functions
"""
import pdb
def jaccard_similarity(s1, s2):
    l1 = s1.split(" ")
    l2 = s2.split(" ")    
    intersection = len(list(set(l1).intersection(l2)))
    union = (len(l1) + len(l2)) - intersection
    if union == 0:
        return 0
    return float(intersection) / union

def clean_paper_sentence(s):
    """
    This function is essentially clean_text without lowercasing.
    """
    s = re.sub('[^A-Za-z0-9]+', ' ', str(s)).strip()
    s = re.sub(' +', ' ', s)
    return s

def shorten_sentences(sentences):
    """
    Sentences that have more than MAX_LENGTH words will be split
    into multiple sentences with overlappings.
    """
    short_sentences = []
    for sentence in sentences:
        words = sentence.split()
        if len(words) > MAX_LENGTH:
            for p in range(0, len(words), MAX_LENGTH - OVERLAP):
                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))
        else:
            short_sentences.append(sentence)
    return short_sentences

connection_tokens = {'s', 'of', 'and', 'in', 'on', 'for', 'data', 'dataset', 'survey', 'statistics'}
def find_mask_candidates(sentence):
    """
    Extract masking candidates for Masked Dataset Modeling from a given $sentence.
    A candidate should be a continuous sequence of at least 2 words, 
    each of these words either has the first letter in uppercase or is one of
    the connection words ($connection_tokens). Furthermore, the connection 
    tokens are not allowed to appear at the beginning and the end of the
    sequence.
    """
    def candidate_qualified(words):
        while len(words) and words[0].lower() in connection_tokens:
            words = words[1:]
        while len(words) and words[-1].lower() in connection_tokens:
            words = words[:-1]
        
        return len(words) >= 2
    
    candidates = []
    
    phrase_start, phrase_end = -1, -1
    for id in range(1, len(sentence)):
        word = sentence[id]
        if word[0].isupper() or word in connection_tokens:
            if phrase_start == -1:
                phrase_start = phrase_end = id
            else:
                phrase_end = id
        else:
            if phrase_start != -1:
                if candidate_qualified(sentence[phrase_start:phrase_end+1]):
                    candidates.append((phrase_start, phrase_end))
                phrase_start = phrase_end = -1
    
    if phrase_start != -1:
        if candidate_qualified(sentence[phrase_start:phrase_end+1]):
            candidates.append((phrase_start, phrase_end))
    return candidates
len(all_labels)
print([l for l in all_labels if len(l.strip()) < 6])
"""
### Transform
"""
"""
### Predict
"""
"""
# Baseline Model
"""
def read_append_return(filename, train_files_path, output='text'):
    """
    Function to read json file and then return the text data from them and append to the dataframe
    """
    json_path = os.path.join(train_files_path, (filename+'.json'))
    headings = []
    contents = []
    combined = []
    with open(json_path, 'r') as f:
        json_decode = json.load(f)
        for data in json_decode:
            headings.append(data.get('section_title'))
            contents.append(data.get('text'))
            combined.append(data.get('section_title'))
            combined.append(data.get('text'))
    
    all_headings = ' '.join(headings)
    all_contents = ' '.join(contents)
    all_data = '. '.join(combined)
    
    if output == 'text':
        return all_contents
    elif output == 'head':
        return all_headings
    else:
        return all_data
    
    
def text_cleaning(text):
    '''
    Converts all text to lower case, Removes special charecters, emojis and multiple spaces
    text - Sentence that needs to be cleaned
    '''
    text = ''.join([k for k in text if k not in string.punctuation])
    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()
    # text = re.sub("/'+/g", ' ', text)
    return text
import tempfile
from typing import Dict, Iterable, List, Tuple

import torch

from allennlp.common.util import JsonDict
from allennlp.data import (
    DataLoader,
    DatasetReader,
    Instance,
    Vocabulary,
    TextFieldTensors,
)
from allennlp.data.data_loaders import SimpleDataLoader, MultiProcessDataLoader
from allennlp.data.fields import LabelField, TextField
from allennlp.data.token_indexers import TokenIndexer, SingleIdTokenIndexer
from allennlp.data.tokenizers import Token, Tokenizer, WhitespaceTokenizer
from allennlp.models import Model
from allennlp.modules import TextFieldEmbedder, Seq2VecEncoder
from allennlp.modules.text_field_embedders import BasicTextFieldEmbedder
from allennlp.modules.token_embedders import Embedding
from allennlp.modules.seq2vec_encoders import BagOfEmbeddingsEncoder, LstmSeq2VecEncoder
from allennlp.nn import util
from allennlp.predictors import Predictor
from allennlp.training.metrics import CategoricalAccuracy
from allennlp.training.optimizers import AdamOptimizer
from allennlp.training.trainer import Trainer, GradientDescentTrainer
from allennlp.training.util import evaluate
import math
import torch
from allennlp.modules.token_embedders.embedding import _read_embeddings_from_text_file


class ClassificationTsvReader(DatasetReader):
    def __init__(
        self,
        sentence_including_annotations: List[str],
        sentence_not_including_dataset: List[str],
        tokenizer: Tokenizer = None,
        token_indexers: Dict[str, TokenIndexer] = None,
        max_tokens: int = None,
        **kwargs
    ):
        super().__init__(**kwargs)
        self.tokenizer = tokenizer or WhitespaceTokenizer()
        self.token_indexers = token_indexers or {"tokens": SingleIdTokenIndexer()}
        self.max_tokens = max_tokens
        
        self.train_sent_including_dataset, self.dev_sent_including_dataset, self.train_sent_not_including_dataset, self.dev_sent_not_including_dataset = \
         self._data_splitter_with_train_and_dev(sentence_including_annotations, sentence_not_including_dataset)
        
        print('length_of_data')
        print('train including dataset sents', len(self.train_sent_including_dataset))
        print('dev including dataset sents', len(self.dev_sent_including_dataset))
        print('train not including dataset sents', len(self.train_sent_not_including_dataset))
        print('dev not including dataset sents', len(self.dev_sent_not_including_dataset))
        
    def _data_splitter_with_train_and_dev(self, sentence_including_annotations, sentence_not_including_dataset):
        train_frac, dev_frac = 0.85, 0.15
        sentences_including_annotations_num = len(sentence_including_annotations)
        train_sentences_including_annotations = sentence_including_annotations[:math.floor(sentences_including_annotations_num * train_frac)]
        dev_sentences_including_annotations = sentence_including_annotations[math.floor(sentences_including_annotations_num * train_frac):]
    
        sentences_not_including_annotations_num = len(sentence_not_including_dataset)
        train_sentences_not_including_annotations = sentence_not_including_dataset[:math.floor(sentences_not_including_annotations_num * train_frac)]
        dev_sentences_not_including_annotations = sentence_not_including_dataset[math.floor(sentences_not_including_annotations_num * train_frac):]
        
        return train_sentences_including_annotations, \
               dev_sentences_including_annotations, \
               train_sentences_not_including_annotations, \
               dev_sentences_not_including_annotations
    
    def text_to_instance(self, text: str, label: str = None) -> Instance:
        tokens = self.tokenizer.tokenize(text)
        if self.max_tokens:
            tokens = tokens[: self.max_tokens]
        text_field = TextField(tokens, self.token_indexers)
        fields = {'text': text_field}
        if label:
            fields['label'] = LabelField(label)
        return Instance(fields)
               
        
    def _read(self, train_or_dev: str) -> Iterable[Instance]:
        if train_or_dev == 'train':
            train_data = list()
            
            for sent in self.train_sent_including_dataset:
                dataset_including_flag = '1'
                yield self.text_to_instance(sent, dataset_including_flag)
            for sent in self.train_sent_not_including_dataset:
                dataset_including_flag = '0'
                yield self.text_to_instance(sent, dataset_including_flag)
            
            return train_data
            
        elif train_or_dev == 'dev':
            dev_data = list()
            for sent in self.dev_sent_including_dataset:
                dataset_including_flag = '1' 
                yield self.text_to_instance(sent, dataset_including_flag)
            for sent in self.dev_sent_not_including_dataset:
                dataset_including_flag = '0'
                yield self.text_to_instance(sent, dataset_including_flag)
            
            return dev_data
        else:
            raise NotImplementedError

class SimpleClassifier(Model):
    def __init__(
        self, vocab: Vocabulary, embedder: TextFieldEmbedder, encoder: Seq2VecEncoder
    ):
        super().__init__(vocab)
        self.embedder = embedder
        self.encoder = encoder
        num_labels = vocab.get_vocab_size("labels")
        self.classifier = torch.nn.Linear(encoder.get_output_dim(), num_labels)
        self.accuracy = CategoricalAccuracy()

    def forward(
        self, text: TextFieldTensors, label: torch.Tensor = None) -> Dict[str, torch.Tensor]:

        # Shape: (batch_size, num_tokens, embedding_dim)
        embedded_text = self.embedder(text)
        # Shape: (batch_size, num_tokens)
        mask = util.get_text_field_mask(text)
        # Shape: (batch_size, encoding_dim)
        encoded_text = self.encoder(embedded_text, mask)
        # Shape: (batch_size, num_labels)
        logits = self.classifier(encoded_text)
        # Shape: (batch_size, num_labels)
        probs = torch.nn.functional.softmax(logits)
        # Shape: (1,)
        output = {"probs": probs}
        if label is not None:
            self.accuracy(logits, label)
            output['loss'] = torch.nn.functional.cross_entropy(logits, label)
        return output

    def get_metrics(self, reset: bool = False) -> Dict[str, float]:
        return {"accuracy": self.accuracy.get_metric(reset)}


def build_dataset_reader(sentence_including_annotations: List,
        sentence_not_including_dataset: List,
                        max_tokens: int) -> DatasetReader:
    return ClassificationTsvReader(sentence_including_annotations, sentence_not_including_dataset, 
                                   max_tokens=max_tokens)


def read_data(reader: DatasetReader) -> Tuple[List[Instance], List[Instance]]:
    print("Reading data")
    training_data = list(reader.read("train"))
    validation_data = list(reader.read("dev"))
    return training_data, validation_data


def build_vocab(instances: Iterable[Instance]) -> Vocabulary:
    print("Building the vocabulary")
    return Vocabulary.from_instances(instances)


def build_model(vocab: Vocabulary) -> Model:
    print("Building the model")
    vocab_size = vocab.get_vocab_size("tokens")
    # '../input/glove6b/glove.6B.300d.txt'
    
    embed_matrix = _read_embeddings_from_text_file(
        file_uri='../input/glove6b/glove.6B.300d.txt',
        embedding_dim=300,
        vocab=vocab
    )
    token_embedding = Embedding(num_embeddings=vocab.get_vocab_size('tokens'),
                                embedding_dim=300,
                                weight=embed_matrix)
    
    embedder = BasicTextFieldEmbedder(
        {"tokens": token_embedding}
    )
    encoder = LstmSeq2VecEncoder(input_size=300, hidden_size=300, num_layers=1, bidirectional=True)
    # BagOfEmbeddingsEncoder(embedding_dim=300)
    return SimpleClassifier(vocab, embedder, encoder)


def build_data_loaders(dataset_reader) -> Tuple[DataLoader, DataLoader]:
    if torch.cuda.is_available():
        train_loader =  MultiProcessDataLoader(dataset_reader, data_path='train', batch_size=64, shuffle=True,
                                              cuda_device=0)
        dev_loader =  MultiProcessDataLoader(dataset_reader, data_path='dev', batch_size=64, shuffle=False,
                                            cuda_device=0)
    else:
        train_loader =  MultiProcessDataLoader(dataset_reader, data_path='train', batch_size=64, shuffle=True)
        dev_loader =  MultiProcessDataLoader(dataset_reader, data_path='dev', batch_size=64, shuffle=False)
    return train_loader, dev_loader


def build_trainer(
    model: Model,
    serialization_dir: str,
    train_loader: DataLoader,
    dev_loader: DataLoader,
) -> Trainer:
    if torch.cuda.is_available():
        model.cuda()
    
    parameters = [(n, p) for n, p in model.named_parameters() if p.requires_grad]
    optimizer = AdamOptimizer(parameters, lr=0.01)  # type: ignore
    trainer = GradientDescentTrainer(
        model=model,
        serialization_dir=serialization_dir,
        data_loader=train_loader,
        validation_data_loader=dev_loader,
        num_epochs=5,
        optimizer=optimizer,
        cuda_device=0 if torch.cuda.is_available() else -1
    )
    return trainer


def run_training_loop(sentence_including_annotations: List[str], 
                      sentence_not_including_dataset: List[str],
                     max_tokens: int):
    dataset_reader = build_dataset_reader(sentence_including_annotations, sentence_not_including_dataset,
                                         max_tokens)
    train, dev = dataset_reader.read('train'), dataset_reader.read('dev')
    vocab = build_vocab(train)
    vocab.extend_from_instances(dev)
    model = build_model(vocab)

    train_loader, dev_loader = build_data_loaders(dataset_reader)
    train_loader.index_with(vocab)
    dev_loader.index_with(vocab)

    # You obviously won't want to create a temporary file for your training
    # results, but for execution in binder for this guide, we need to do this.
    with tempfile.TemporaryDirectory() as serialization_dir:
        trainer = build_trainer(model, serialization_dir, train_loader, dev_loader)
        trainer.train()

    return model, dataset_reader
sentences_including_annotations_all = js['sentences_including_annotations_all']
sentences_not_including_dataset_all = js['sentences_not_including_dataset_all']
print(len(sentences_including_annotations_all))
print(len(sentences_not_including_dataset_all))
model, dataset_reader = run_training_loop(sentences_including_annotations_all, sentences_not_including_dataset_all,
                                         max_tokens=250)
vocab = model.vocab
from allennlp.predictors import Predictor
from allennlp.training.util import evaluate


class SentenceClassifierPredictor(Predictor):
    def predict(self, sentence: str) -> JsonDict:
        return self.predict_json({"sentence": sentence})
    
    def predict_batch(self, sentences: List[str]):
        return self.predict_batch_json([{"sentence": sentence} for sentence in sentences])

    def _json_to_instance(self, json_dict: JsonDict) -> Instance:
        sentence = json_dict["sentence"]
        return self._dataset_reader.text_to_instance(sentence)

predictor = SentenceClassifierPredictor(model, dataset_reader)

output = predictor.predict_batch(["A good movie!", "Slosh model is needed"])
print(output)
from nltk import sent_tokenize
"""
## Aggregate final predictions and write submission file
"""
"""
# Evaluation Metric
"""
# Asthetics
import warnings
warnings.filterwarnings('ignore', category=DeprecationWarning)
warnings.filterwarnings('ignore', category=FutureWarning)

# Basic
import pandas as pd
pd.set_option('display.max_columns', None)
import numpy as np
import json
import os
import random
from tqdm.autonotebook import tqdm
import string
import re
from functools import partial

# Visualizations
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
sns.set(style="whitegrid")
from wordcloud import WordCloud, STOPWORDS


RANDOM_SEED = 30
def seed_everything(seed=RANDOM_SEED):
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    random.seed(seed)
seed_everything()
train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')
sample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')
train_files_path = '../input/coleridgeinitiative-show-us-the-data/train'
test_files_path = '../input/coleridgeinitiative-show-us-the-data/test'
def read_append_return(filename, train_files_path=train_files_path, output='text'):
    json_path = os.path.join(train_files_path, (filename+'.json'))
    headings = []
    contents = []
    combined = []
    with open(json_path, 'r') as f:
        json_decode = json.load(f)
        for data in json_decode:
            headings.append(data.get('section_title'))
            contents.append(data.get('text'))
            combined.append(data.get('section_title'))
            combined.append(data.get('text'))
    
    all_headings = ' '.join(headings)
    all_contents = ' '.join(contents)
    all_data = '. '.join(combined)
    
    if output == 'text':
        return all_contents
    elif output == 'head':
        return all_headings
    else:
        return all_data
tqdm.pandas()
train_df['text'] = train_df['Id'].progress_apply(read_append_return)
tqdm.pandas()
sample_sub['text'] = sample_sub['Id'].progress_apply(partial(read_append_return, train_files_path=test_files_path))
sample_sub.head()
def text_cleaning(text):
    '''
    Converts all text to lower case, Removes special charecters, emojis and multiple spaces
    text - Sentence that needs to be cleaned
    '''
    text = ''.join([k for k in text if k not in string.punctuation])
    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()
    text = re.sub(' +', ' ', text)
    emoji_pattern = re.compile("["
                               u"\U0001F600-\U0001F64F"  # emoticons
                               u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                               u"\U0001F680-\U0001F6FF"  # transport & map symbols
                               u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                               "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    return text
tqdm.pandas()
train_df['text'] = train_df['text'].progress_apply(text_cleaning)
def clean_text(txt):
    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()
temp_1 = [x.lower() for x in train_df['dataset_label'].unique()]
'''
Idea below of also using the 'dataset_title' is burrowed from
https://www.kaggle.com/josephassaker/coleridge-initiative-eda-na-ve-submission
'''
temp_2 = [x.lower() for x in train_df['dataset_title'].unique()]
temp_3 = [x.lower() for x in train_df['cleaned_label'].unique()]

existing_labels = set(temp_1 + temp_2 + temp_3 ) | all_labels
len(existing_labels)
one_word_label = []
many_word_label = []

for extist_label in existing_labels:
    sp_ext = extist_label.strip().split(' ')
    if len(sp_ext) == 1:
        one_word_label.append(' '.join(sp_ext))
    else:
        many_word_label.append(' '.join(sp_ext))
len(one_word_label)
many_word_label[:10]
id_list = []
lables_list = []
from abbreviations import schwartz_hearst

def batcher(iterable, n=1):
    l = len(iterable)
    for ndx in range(0, l, n):
        yield iterable[ndx:min(ndx + n, l)]

for index, row in tqdm(sample_sub.iterrows()):
    sample_text = row['text']

    abb_pairs = {}
    '''
    {'QC': 'quality control', 'MDS': 'multidimensional scaling', 'MLMA': 'mixed-linear-model association',
    'ii': 'included in COGENT; and', 'CHARGE': 'Cohorts for Heart and Aging Research in Genomic Epidemiology', 
    'FHS': 'Framingham Heart Study, Second Generation Cohort', 'CHS': 'Cardiovascular Health Study',
    'NCNG': 'Norwegian Cognitive NeuroGenetics', 'lincRNA': 'long intergenic non-coding RNA', 
    'PING': 'Pediatric Imaging, Neurocognition, and Genetics', 'ADNI': "Alzheimer's Disease Neuroimaging Initiative", 
    'KDP': 'Kernel density plot'}
    '''
    row_id = row['Id']
    temp_df = train_df[train_df['text'] == text_cleaning(sample_text)]
    cleaned_labels = temp_df['cleaned_label'].to_list()
    for known_label in existing_labels:
        if known_label in sample_text.lower():
            cleaned_labels.append(clean_text(known_label))
    cleaned_labels = [clean_text(x).replace(',', ' ') for x in cleaned_labels]
    cleaned_labels = set(cleaned_labels)

    try:
        labels_from_ner = list()
        sents = sent_tokenize(sample_text)
        sents_num = len(sents)
        sents = sents[: sents_num // 2]
        for batched_sents in batcher(sents, 64):
            whether_sent_includes_dataset = predictor.predict_batch(batched_sents)

            filtered_texts = list()

            for sent, if_dataset_include_result in zip(batched_sents, whether_sent_includes_dataset):
                result =     [
                (vocab.get_token_from_index(label_id, "labels"), prob)
                for label_id, prob in enumerate(if_dataset_include_result["probs"])]

                if result[1][1] > 0.98:
                    filtered_texts.append(sent)
                
                if 'et al' in sent:
                    filtered_texts.append(sent)
                
                try:
                    if len(set(sent.lower().split(' ')) & set(common_data_words)) > 0 or '(' in sent:
                        pairs = schwartz_hearst.extract_abbreviation_definition_pairs(doc_text=sent) 
                        if len(pairs) > 0:
                            filtered_texts.append(sent)
                            for k, v in pairs.items():
                                abb_pairs.update({k.lower(): v.lower()})
                except:
                    pass
            
            filtered_texts = list(set(filtered_texts))
            if len(filtered_texts) > 0:
                res = ner_model.predict(filtered_texts)
                res = res[0]

                tmp_ds_label = []
                for sent_result in res:
                    for one_word_pred in sent_result:
                        for word, pred_label in one_word_pred.items():
                            if pred_label in ['B-DATASET', 'I-DATASET']:
                                tmp_ds_label.append(word)
                            if pred_label == 'O':
                                if len(tmp_ds_label) > 1: # critical params
                                    if tmp_ds_label[0] not in eng_stws and tmp_ds_label[-1] not in eng_stws:
                                        labels_from_ner.append(' '.join(tmp_ds_label))
                                tmp_ds_label.clear()

        labels_from_ner = [clean_text(label) for label in labels_from_ner if len(label.split(' ')) > 2]
        filtered_labels = []

        for label in sorted(labels_from_ner, key=len, reverse=True):
            if len(filtered_labels) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered_labels):
                filtered_labels.append(label)

        filtered_labels = [label.replace(',', ' ') for label in filtered_labels if label.split(' ')[0] not in eng_stws
                          and label.split(' ')[-1] not in eng_stws]
        

        '''
        ['floodplain management and hazard mitigation', 'north carolina division of transportation', 
        'nc sea level rise risk management study', 'erosion storm surge and sea level rise',
        'usgs digital shoreline analysis system', 'area of change from shoreline movement', 
        'periodic reporting of sea level trends', 'estuarine shoreline change and erosion', 
        'storm surge vulnerability assessment', 'albemarle pamlico estuarine system', 
        'oceanic atmospheric administration', 'sea level affecting marshes model',
        'digital flood insurance rate maps', 'national flood insurance program',
        'cape hatteras national seashore', 'coastal elevation data quality',
        'sea level rise inundation maps', 'storm surge vulnerability maps', 
        'tidal or wind tide inundation', 'nc floodplain mapping program', 'coastal vulnerability index', 
        'coastal observation station', 'surge simulation modeling', 'area of shoreline change', 
        'linear regression rate', 'sea level rise mapping', 
        'coastal erosion study', 'envelope of water', 'noaa ocm slr']
        '''
    except:
        filtered_labels = list()
    
    set_list = set(cleaned_labels)
    
    
    # critical_words
    final_filtered_from_ner = [d for d in filtered_labels if d.lower().split(' ')[-1] in critical_words]
    print('** filtered ner prediction **', final_filtered_from_ner)
    print('** cleaned str match **',cleaned_labels)
    
    filtered_abb = [v.lower() for v in abb_pairs.values()]
    filtered_abb = [v for v in filtered_abb if v.split(' ')[-1] not in eng_stws and v.split(' ')[0] not in eng_stws]
    filtered_abb = [clean_text(v) for v in filtered_abb if len(set(v.split(' ')) & set(critical_words)) > 0]
    print('** filtered_abb **', filtered_abb)
    
    entire_labels = []
    entire_labels += [clean_text(label) for label in list(set_list)]
    
    if len(entire_labels) == 0 :
        entire_labels += [clean_text(label) for label in list(set(filtered_abb))]
        entire_labels = list(set(entire_labels))
        
        if len(entire_labels) == 0:

            for label in sorted(final_filtered_from_ner, key=len, reverse=True):
                if len(entire_labels) == 0 or all(jaccard_similarity(label, got_label) < 0.50 for got_label in entire_labels):
                    entire_labels.append(label)
        entire_labels = entire_labels[:5]
    
    entire_labels.sort()
    print('**final_labels aggregated**', entire_labels)
    lables_list.append('|'.join(entire_labels))
    id_list.append(row_id)
sample_sub['PredictionString'] = lables_list
sample_sub[['Id', 'PredictionString']].to_csv('submission.csv', index=False)
sample_sub.head()
assert len(id_list) == len(lables_list)
assert len(sample_sub) == len(id_list)
# submission = pd.DataFrame()
assert len(sample_sub['Id']) == len(id_list)
sample_sub[['Id', 'PredictionString']].head()
# submission.to_csv('submission.csv', index=False)
"""
|   | CV | LB |
| --- | --- | --- |
| s57 adnl_govt_labels |   | 0.573 |
| 42 w/ adnl_govt_labels |   | 0.557 |
| adnl_govt_labels_26897 | 0.136 |   |
| adnl_govt_labels |   | 0.556 |
| adnl_govt_labels KEN |   |   |
| 42 w/ adnl_govt_labels KEN |   | 0.557 |
| **RKM adnl_govt_labels w/ 42** | **0.514** | **0.574** |
| RKM adnl_govt_labels BS_CLEANING w/ 42 |   | 0.568 |
| RKM adnl_govt_labels_26897 |   | 0.244 |
| RKM adnl_govt_labels w/ 42_48 |   | 0.574 |
| **RKM adnl_govt_labels w/ 42_36** |   | **0.574** |
| RKM adnl_govt_labels w/ 42_60 |   | 0.574 |
| RKM adnl_govt_labels w/ 42_24 |   | 0.573 |
| RKM adnl_govt_labels w/ 42_60 |   |   |
"""