# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ### Practical 2
#
# ### Creating a bag of words model by processing raw wikipedia text

# Implementing bag of words model
import nltk
import numpy as np # matrix is required to vectorize the words
import bs4 as bs
import urllib.request
import re

# Scrape the web data
raw_html = urllib.request.urlopen('https://en.wikipedia.org/wiki/Natural_language_processing')
raw_html = raw_html.read()
print(raw_html)

article_html = bs.BeautifulSoup(raw_html, 'lxml')
print(article_html)

article_paragraphs = article_html.find_all('p') # find all stuff with the tag p
print(article_paragraphs)

article_text = ''
for para in article_paragraphs:
    article_text += para.text
print(article_text)

# Split corpus into individual sentences
corpus = nltk.sent_tokenize(article_text)
print(corpus)

# We will remove punctuation marks, space, and turn all text into lower case
for i in range(len(corpus)):
    corpus[i] = corpus[i].lower() # Turn into lower case
    corpus[i] = re.sub(r'\W', ' ', corpus[i]) # Eliminate the blank spaces
    corpus[i] = re.sub(r'\s+', ' ', corpus[i]) # Eliminate the punctuations
print(corpus)

# Tokenize the sentences in the corpus and create a dictionary wirh sentences and their frequencies
wordfreq = {}
for sentence in corpus:
    tokens = nltk.word_tokenize(sentence) 
    for t in tokens:
        if t not in wordfreq.keys():
            wordfreq[t] = 1
        else:
            wordfreq[t] += 1
print(wordfreq)

# Filter down to 200 most frequently ocurring words:
import heapq
most_freq = heapq.nlargest(200, wordfreq, key=wordfreq.get)
print(most_freq)

# Convert the sentence in the corpus into their corresponding vector representation:
sent_vectors = []
for sentence in corpus:
    sent_tokens = nltk.word_tokenize(sentence)
    sent_vec = []
    for word in most_freq:
        if word in sent_tokens:
            sent_vec.append(1)
        else:
            sent_vec.append(0)
    sent_vectors.append(sent_vec)
print(sent_vectors)

sent_vectors = np.asarray(sent_vectors)
print(sent_vectors)

StarcoderdataJupyterScriptsDedupFiltered