# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + cell_id="00001-80603ce7-5e43-4aaa-9c2e-fc333c20a663" execution_millis=4 execution_start=1603914492064 output_cleared=false source_hash="bbacf3a0" tags=[] id="09g62Azr5U1R" executionInfo={"status": "ok", "timestamp": 1603916886579, "user_tz": 420, "elapsed": 1675, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggu3bTBcysUxxtmnLI9n_OEGc92nfpZXGRAhDZ5vQ=s64", "userId": "12984101785953050215"}}
import nltk
import numpy as np

# + cell_id="00002-c68972ca-99d0-4fb8-8b2c-1c296e224a4a" execution_millis=15 execution_start=1603914493054 output_cleared=false source_hash="2f335b27" tags=[] id="kOWus-a95U1Z" executionInfo={"status": "ok", "timestamp": 1603916888361, "user_tz": 420, "elapsed": 849, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggu3bTBcysUxxtmnLI9n_OEGc92nfpZXGRAhDZ5vQ=s64", "userId": "12984101785953050215"}}
import requests

url = "http://www.gutenberg.org/files/2554/2554.txt"

response = requests.get(url)
raw_html = response.content
text = raw_html.decode("utf-8-sig")

# + [markdown] cell_id="00005-0e3e3085-9f12-4073-b9bf-a699a216c404" tags=[] id="OXQvFzyj5U1f"
# ### Bag of Words
#
# Bag of Words based encoding or TF-IDF vector is a frequentist based approach to NLP applications.

# + cell_id="00006-381941ee-0649-4a76-9841-1923426edffe" execution_millis=248 execution_start=1603914496124 output_cleared=false source_hash="abe7455d" tags=[] id="iSDR5KNt5U1g" executionInfo={"status": "ok", "timestamp": 1603917274958, "user_tz": 420, "elapsed": 2459, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggu3bTBcysUxxtmnLI9n_OEGc92nfpZXGRAhDZ5vQ=s64", "userId": "12984101785953050215"}}
import urllib.request

url = "https://www.gutenberg.org/files/829/829-0.txt" # gulliver's travels
#Alt
#url = 'https://www.gutenberg.org/files/2701/2701-0.txt' # Moby Dick

file = urllib.request.urlopen(url)
text = [line.decode('utf-8') for line in file]
text = ''.join(text)

# + [markdown] cell_id="00007-e8474323-77fb-4af2-b76b-cf21b17e2b07" tags=[] id="S3hK5Ldw5U1j"
# #### Tokenize

# + cell_id="00008-c92e3a1e-acf6-4466-8344-0ca3510a6b75" execution_millis=2119 execution_start=1603914497851 output_cleared=false source_hash="dd04af38" tags=[] id="43g6RdUf5U1k" executionInfo={"status": "ok", "timestamp": 1603917279960, "user_tz": 420, "elapsed": 1306, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggu3bTBcysUxxtmnLI9n_OEGc92nfpZXGRAhDZ5vQ=s64", "userId": "12984101785953050215"}} outputId="243880a7-7b85-4f86-98b9-379827ce3afb" colab={"base_uri": "https://localhost:8080/"}
nltk.download('punkt')
from nltk import word_tokenize
tokens = word_tokenize(text)

# + cell_id="00009-e513d93c-17df-4f64-ba5e-748f1628283d" execution_millis=218 execution_start=1603914500757 output_cleared=false source_hash="204a6f56" tags=[] id="oc9TB80i5U1p" executionInfo={"status": "ok", "timestamp": 1603917288906, "user_tz": 420, "elapsed": 647, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggu3bTBcysUxxtmnLI9n_OEGc92nfpZXGRAhDZ5vQ=s64", "userId": "12984101785953050215"}}
import string
tokens = [word for word in tokens if word.isalpha()]
table = str.maketrans('', '', string.punctuation)
tokens = [w.translate(table) for w in tokens]
tokens = [word.lower() for word in tokens]

# + [markdown] cell_id="00010-128bff7f-6c9c-41c7-af8b-34e14d7c8cda" tags=[] id="d-kUPb7k5U1s"
# Removing **stop-words** and **stemming**

# + cell_id="00011-cdf4882f-23c8-4056-852c-7aa7f2e398d9" execution_millis=2889 execution_start=1603914502637 output_cleared=false source_hash="203b141e" tags=[] id="Syfo7BOb5U1t" executionInfo={"status": "ok", "timestamp": 1603917291637, "user_tz": 420, "elapsed": 1571, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggu3bTBcysUxxtmnLI9n_OEGc92nfpZXGRAhDZ5vQ=s64", "userId": "12984101785953050215"}} outputId="842060a1-2a34-4b07-c1d2-e90caa8c4d71" colab={"base_uri": "https://localhost:8080/"}
from nltk.corpus import stopwords
nltk.download('stopwords')
stop_words = set(stopwords.words('english'))
tokens = [w for w in tokens if not w in stop_words]

from nltk.stem.porter import PorterStemmer
porter = PorterStemmer()
tokens = [porter.stem(word) for word in tokens]
tokens[200:202]

# + [markdown] cell_id="00012-044ce2ce-6815-490b-9f18-0645470e543f" tags=[] id="NjzIKn_85U1w"
# **Understanding the vocabulary** 
#
# * A vocabulary of a document represents all the words in that document and the frequency they appear.
# * `FreqDist` class
#

# + cell_id="00013-fdd3703a-21ab-4f5c-bef8-ecccb46953e3" execution_millis=40 execution_start=1603914505591 output_cleared=false source_hash="60a04f4f" tags=[] id="2Yv0397E5U1x" executionInfo={"status": "ok", "timestamp": 1603917300119, "user_tz": 420, "elapsed": 683, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggu3bTBcysUxxtmnLI9n_OEGc92nfpZXGRAhDZ5vQ=s64", "userId": "12984101785953050215"}} outputId="9ceba789-32ca-425d-af6c-26b40c38e744" colab={"base_uri": "https://localhost:8080/"}
from nltk.probability import FreqDist

word_counts = FreqDist(tokens)
word_counts

# + [markdown] cell_id="00014-72d4693b-04ed-488c-ae66-8f0ce7199aed" tags=[] id="RhDlzRqE5U1z"
# **Scoring words with frequency**

# + cell_id="00015-02387729-4804-4ce6-9161-419c503be606" execution_millis=9 execution_start=1603914506105 output_cleared=false source_hash="d999d5d7" tags=[] id="eapZUCqh5U10" executionInfo={"status": "ok", "timestamp": 1603917302419, "user_tz": 420, "elapsed": 513, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggu3bTBcysUxxtmnLI9n_OEGc92nfpZXGRAhDZ5vQ=s64", "userId": "12984101785953050215"}} outputId="54d8f9b6-ffa1-4158-dcd9-aa00c84730f5" colab={"base_uri": "https://localhost:8080/"}
top = 100
vocabulary = word_counts.most_common(top)

vocabulary[:10]

# + cell_id="00016-a40281d3-efa8-460f-bbe0-e4090eee870a" execution_millis=4 execution_start=1603914507044 output_cleared=false source_hash="a2651e8b" tags=[] id="SO1eID8a5U14" executionInfo={"status": "ok", "timestamp": 1603917303987, "user_tz": 420, "elapsed": 599, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggu3bTBcysUxxtmnLI9n_OEGc92nfpZXGRAhDZ5vQ=s64", "userId": "12984101785953050215"}} outputId="70e04656-fbc9-4c47-a1f0-36bf5282b984" colab={"base_uri": "https://localhost:8080/"}
voc_size = len(vocabulary)
doc_vector = np.zeros(voc_size)

word_vector = [(idx,word_counts[word[0]]) for idx, word in enumerate(vocabulary) if word[0] in word_counts.keys()] 
word_vector[10]

# + cell_id="00017-36e9c930-b190-4fa9-9b2d-dbda227ca607" execution_millis=415 execution_start=1603914507882 output_cleared=false source_hash="ae320f3a" tags=[] id="NStzHzfH5U17" executionInfo={"status": "ok", "timestamp": 1603917335032, "user_tz": 420, "elapsed": 3213, "user": {"displayName": "Y<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ggu3bTBcysUxxtmnLI9n_OEGc92nfpZXGRAhDZ5vQ=s64", "userId": "12984101785953050215"}} outputId="48efa19f-9fe3-400a-81c5-8f1a91f05975" colab={"base_uri": "https://localhost:8080/"}
# Generating a model of Bag of Words

from nltk import sent_tokenize

docs = sent_tokenize(text)[703:706]
docs

from sklearn.feature_extraction.text import CountVectorizer

count_vectorizer=CountVectorizer(stop_words='english')
word_count_vector=count_vectorizer.fit_transform(docs)
word_count_vector.shape
word_count_vector.toarray()
count_vectorizer.get_feature_names()

# + id="DqneBlSy5exp"


StarcoderdataJupyterScriptsDedupFiltered