# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# <center>
# <h1><b>Word Embedding Based Answer Evaluation System for Online Assessments (WebAES)</b></h1>
# <h3>A smart system to automate the process of answer evaluation in online assessments.</h3>
# <h5>LDA Model for topic extraction</h5>

# +
# For training LDA model using text8 corpus
from gensim import corpora, models
import gensim.downloader as api

# To perform text pre-processing
import string

# Natural Language Toolkit
import nltk
from nltk.corpus import stopwords
nltk.download('stopwords')

# Set of stopwords in English
en_stopwords = set(stopwords.words('english'))

# +
# Load text8 corpus and convert to list of documents
text8_corpus = api.load('text8')
text8_corpus = [doc for doc in text8_corpus]

# List containing list of tokens from each document of text8 corpus
list_of_list_of_tokens = []

# For each document in the text8 corpus
for i in range(len(text8_corpus)):
    # Remove stopwords from each document
    text8_corpus[i] = [w for w in text8_corpus[i] if w not in en_stopwords]
    
    # Add list of tokens for document to list of list of tokens
    list_of_list_of_tokens.append(text8_corpus[i])

# +
# Create a text corpus using list of list of tokens
dictionary_LDA = corpora.Dictionary(list_of_list_of_tokens)
corpus = [dictionary_LDA.doc2bow(list_of_tokens) for list_of_tokens in list_of_list_of_tokens]

# LDA model to extract 100 topics from the text corpus
num_topics = 100
# %time lda_model = models.LdaModel(corpus, num_topics=num_topics, id2word=dictionary_LDA, passes=4, alpha=[0.01]*num_topics, eta=[0.01]*len(dictionary_LDA.keys()))

# Save trained LDA model
lda_model.save('WebAES_LDA_Model.model')
# -

# Load previously trained LDA model
lda_model =  models.LdaModel.load('WebAES_LDA_Model.model')

# +
# Pre-process text
new_text = 'The different domains of Artificial Intelligence include machine learning, neural networks, robotics, expert systems, fuzzy logic, and natural language processing. Machine Learning is the science of getting computers to act by feeding them data so that they can learn a few tricks on their own, without being explicitly programmed to do so. Neural Networks are a set of algorithms and techniques, modelled in accordance with the human brain. Neural Networks are designed to solve complex and advanced machine learning problems. Robotics is a subset of AI, which includes different branches and application of robots. These Robots are artificial agents acting in a real-world environment. An AI Robot works by manipulating the objects in its surrounding, by perceiving, moving and taking relevant actions. An expert system is a computer system that mimics the decision-making ability of a human. It is a computer program that uses artificial intelligence (AI) technologies to simulate the judgment and behaviour of a human or an organization that has expert knowledge and experience in a particular field. Fuzzy logic is an approach to computing based on “degrees of truth” rather than the usual “true or false” (1 or 0) Boolean logic on which the modern computer is based. Fuzzy logic Systems can take imprecise, distorted, noisy input information. Natural Language Processing (NLP) refers to the Artificial Intelligence method that analyses natural human language to derive useful insights in order to solve problems.'
# Remove punctuations
new_text = new_text.translate(str.maketrans('', '', string.punctuation)).lower()
# Remove stopwords
new_text = ' '.join([w for w in new_text.split() if not w.lower() in en_stopwords])
# Create list of tokens
new_text_tokens = new_text.split()

# Get probability of belonging to each topic using LDA model
topic_probs = lda_model[dictionary_LDA.doc2bow(new_text_tokens)]

# Display all topics and their probability
print('All topics: ', topic_probs)

# Select index of topic with maximum topic probability
max_prob = 0
max_prob_topic = 0
for topic in topic_probs:
    topic_index, topic_prob = topic[0], topic[1]
    if topic_prob > max_prob:
        max_prob = topic_prob
        max_prob_topic = topic_index

# Display topic index with maximum topic probability
print('Index of topic with max. probability: ', max_prob_topic)

# +
# Pre-process text
new_text = 'The different domains of Artificial Intelligence include machine learning, neural networks, robotics, expert systems, fuzzy logic, and natural language processing.'
# Remove punctuations
new_text = new_text.translate(str.maketrans('', '', string.punctuation)).lower()
# Remove stopwords
new_text = ' '.join([w for w in new_text.split() if not w.lower() in en_stopwords])
# Create list of tokens
new_text_tokens = new_text.split()

# Get probability of belonging to each topic using LDA model
topic_probs = lda_model[dictionary_LDA.doc2bow(new_text_tokens)]

# Display all topics and their probability
print('All topics: ', topic_probs)

# Select index of topic with maximum topic probability
max_prob = 0
max_prob_topic = 0
for topic in topic_probs:
    topic_index, topic_prob = topic[0], topic[1]
    if topic_prob > max_prob:
        max_prob = topic_prob
        max_prob_topic = topic_index

# Display topic index with maximum topic probability
print('Index of topic with max. probability: ', max_prob_topic)

# +
# Pre-process text
new_text = 'Human Computer Interaction is the study of the interaction between human users and computer systems.'
# Remove punctuations
new_text = new_text.translate(str.maketrans('', '', string.punctuation)).lower()
# Remove stopwords
new_text = ' '.join([w for w in new_text.split() if not w.lower() in en_stopwords])
# Create list of tokens
new_text_tokens = new_text.split()

# Get probability of belonging to each topic using LDA model
topic_probs = lda_model[dictionary_LDA.doc2bow(new_text_tokens)]

# Display all topics and their probability
print('All topics: ', topic_probs)

# Select index of topic with maximum topic probability
max_prob = 0
max_prob_topic = 0
for topic in topic_probs:
    topic_index, topic_prob = topic[0], topic[1]
    if topic_prob > max_prob:
        max_prob = topic_prob
        max_prob_topic = topic_index

# Display topic index with maximum topic probability
print('Index of topic with max. probability: ', max_prob_topic)

StarcoderdataJupyterScriptsDedupFiltered