# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

import spacy
import gensim
import xml.etree.ElementTree as ET

print(f'gensim version: {gensim.__version__}')
glove_model = gensim.models.KeyedVectors.load_word2vec_format('data/glove.6B.50d.txt')
spacy_nlp = spacy.load('en_core_web_sm')

qa = ET.parse('../data/CRF_training/qa_test.xml')
root = qa.getroot()


def wmd(qa_pair):
    """
        Function used to compute Word Moving Distance between question and each candidate
    """
    
    qa_pair_tokens = []
    for item in qa_pair:
        qa_pair_tokens.append([token.text.lower() for token in spacy_nlp(item) if not token.is_stop])
    
    question = qa_pair[0]
    question_tokens =  qa_pair_tokens[0]
    
    print(f'\nQuestion: {question}\n')
    for candidate_tokens, candidate in zip(qa_pair_tokens, qa_pair):
        print('-' * 50)
        print(f'Comparing to: {candidate}')
        dist = glove_model.wmdistance(question_tokens, candidate_tokens)
        print(f'distance = {dist:.4f}')


pair = []
for qa_pair in root:
        pair = []
        pair = [qa_pair.find('question').text.split('\n')[1].strip()]
        pair.extend([candidate.text.split('\n')[1].strip() for candidate in qa_pair.findall('candidate')])
        
        wmd(pair)



StarcoderdataJupyterScriptsDedupFiltered