# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# %cd ..

from gensim.corpora import WikiCorpus, MmCorpus, Dictionary
from nltk.tokenize import sent_tokenize, word_tokenize
import os
import random
import codecs
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
import re
from gensim.test.utils import common_texts
from gensim.corpora.dictionary import Dictionary
from gensim.models import LdaModel
from gensim.test.utils import common_texts
from gensim.corpora.dictionary import Dictionary

# ### Steps:
# 1) Load Wikipedia Corpus. [(Title: Sentence), ... ]
# 2) Tokenize each sentence in Wikipedia corpus.
# 3) 

# ### Load WikiCorpus

ex = '\nA None\nAndorra is a coprincipality in which the office of head of state is jointly held ex officio by the French president and the bishop of the Roman Catholic diocese of U'

re.sub(r'[\n]', '', ex)

if re.search(r'\n', ex):
    print(re.search(r'^([^=]*)=', ex).group(1))


def clean_sentence(sent):
    return re.sub(r"\n|\'", '', sent)


def tokenize(text, token_min_len, token_max_len, lower):
    # override original method in wikicorpus.py
    return [token for token in sent_tokenize(text)
            if len(token) >= 20 and not token.startswith('_')]


ex = 'sdlfkjds sdlkjf. sdlfkj sdfsd sdf.'
re.sub(r'[^\w\s.]', '', ex)

wiki = WikiCorpus('data/simplewiki-latest-pages-articles.xml.bz2', tokenizer_func=tokenize)
wiki.metadata = False

wiki.metadata = True

wiki_text = wiki.get_texts()


def obtain_abstract(sents):
    return 


wiki_text = wiki.get_texts()
sents = []
length = 100
i = 0
for t in wiki_text:
    title = t[1][1]
    sents_topic = [(title, clean_sentence(s)) for s in t[0] if s[0] != '*']
    sents.extend(sents_topic)
    i += 1
    if i > length:
        break

# ### Tokenize WikiCorpus

# +
# wiki_tokens

# +
# wiki_tokens = [word_tokenize(s[1]) for s in sents]
# wiki_dict = Dictionary(wiki_tokens)
# wiki_corpus = [wiki_dict.doc2bow(t) for t in wiki_tokens]
# -

# ### Train LDA on WikiCorpus

lda = LdaModel(wiki_corpus, num_topics=10)

# ### Train LSI on WikiCorpus

lsi = models.LsiModel(wiki_corpus, id2word=wiki_dict, num_topics=2)
index = MatrixSimilarity(lsi[wiki_corpus])


# ### Predict document

def predict_doc(doc, lsi, index, wiki_dict):
    sim_df = pd.DataFrame()
    doc_tokens = word_tokenize(doc)
    doc_vec_bow = wiki_dict.doc2bow(doc_tokens)
    vec_lsi = lsi[doc_vec_bow]
    sims = index[vec_lsi]
    sim_df['sents'] = wiki_tokens
    sim_df['sim'] = sims
    return sim_df.sort_values(by='sim', ascending=False)



predict_doc('April is my favorite month.', lsi, index, wiki_dict)

# ## LSI Model

# +
# Create a corpus from a list of texts
common_dictionary = Dictionary(common_texts)
common_corpus = [common_dictionary.doc2bow(text) for text in common_texts]

# Train the model on the corpus.
# lda = LdaModel(common_corpus, num_topics=10)

# +
# for i in wiki.sample_texts(50, length=10000):
#     print(i)

# +
from gensim.test.utils import common_texts
from gensim.corpora.dictionary import Dictionary

common_dictionary = Dictionary(common_texts)
# -

print(common_dictionary[0])


# +
def clean(doc):
    stop_free = " ".join([i for i in doc.lower().split() if i not in stopwords])
    normalized = " ".join(lemma.lemmatize(word,'v') for word in stop_free.split())
    x = normalized.split()
    y = [s for s in x if len(s) > 2]
    return y

def get_related_documents(term, top, corpus):
    clean_docs = [clean_doc(doc) for doc in corpus]
    related_docid = []
    test_term = [ldamodel.id2word.doc2bow(doc) for doc in clean_docs]
    doc_topics = ldamodel.get_document_topics(test_term, minimum_probability=0.20)
    term_topics =  ldamodel.get_term_topics(term, minimum_probability=0.000001)
    for k,topics in enumerate(doc_topics):
        if topics:
            topics.sort(key = itemgetter(1), reverse=True)
            if topics[0][0] == term_topics[0][0]:
                related_docid.append((k,topics[0][1]))
 
    related_docid.sort(key = itemgetter(1), reverse=True)
    for j,doc_id in enumerate(related_docid):
        print (doc_id[1],"\n\n",docs_test[doc_id[0]])
        if j == (top-1):
            break


# +
# Create a corpus from a list of texts
common_dictionary = Dictionary(common_texts)
common_corpus = [common_dictionary.doc2bow(text) for text in common_texts]

# Train the model on the corpus.
lda = LdaModel(common_corpus, num_topics=10)
# -

lda

other_texts = [['computer', 'time', 'graph'], ['survey', 'response', 'eps'],['human', 'system', 'computer']]
other_corpus = [common_dictionary.doc2bow(text) for text in other_texts]
unseen_doc = other_corpus[0]
vector = lda[unseen_doc]

from gensim import models
from gensim.similarities import MatrixSimilarity
# https://radimrehurek.com/gensim/tut3.html

lsi = models.LsiModel(common_corpus, id2word=common_dictionary, num_topics=2)

index = MatrixSimilarity(lsi[common_corpus])

doc = "Human computer interface. How are you?"
vec_bow = common_dictionary.doc2bow(doc.lower().split())
vec_lsi = lsi[vec_bow] 


sims = index[vec_lsi]
print(list(enumerate(sims)))

# +
# common_corpus
# -



StarcoderdataJupyterScriptsDedupFiltered