# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # 1. Named Entity Recognization

# +
import nltk

#nltk.download('maxent_ne_chunker')
#nltk.download('words')

# +
  
paragraph = """The Taj Mahal was built by Emperor <NAME>"""
               
               
# POS Tagging
words = nltk.word_tokenize(paragraph)


# -

tagged_words = nltk.pos_tag(words)

# Named entity recognition
namedEnt = nltk.ne_chunk(tagged_words)
namedEnt.draw()

orga = """
ORGANIZATION	Georgia-Pacific Corp., WHO
PERSON	<NAME>, President Obama
LOCATION	Murray River, Mount Everest
DATE	June, 2008-06-29
TIME	two fifty a m, 1:30 p.m.
MONEY	175 million Canadian Dollars, GBP 10.40
PERCENT	twenty pct, 18.75 %
FACILITY	Washington Monument, Stonehenge
GPE	South East Asia, 
"""

words = nltk.word_tokenize(orga)

tagged_words_new = nltk.pos_tag(words)

nameentity = nltk.ne_chunk(tagged_words_new)

nameentity.draw()

# # 2. Text Visuatization

# Bag of Words

# +
import nltk
import re

paragraph = """Thank you all so very much. Thank you to the Academy. 
               Thank you to all of you in this room. I have to congratulate 
               the other incredible nominees this year. The Revenant was 
               the product of the tireless efforts of an unbelievable cast
               and crew. First off, to my brother in this endeavor, Mr. Tom 
               Hardy. Tom, your talent on screen can only be surpassed by 
               your friendship off screen … thank you for creating a t
               ranscendent cinematic experience. Thank you to everybody at 
               Fox and New Regency … my entire team. I have to thank 
               everyone from the very onset of my career … To my parents; 
               none of this would be possible without you. And to my 
               friends, I love you dearly; you know who you are. And lastly,
               I just want to say this: Making The Revenant was about
               man's relationship to the natural world. A world that we
               collectively felt in 2015 as the hottest year in recorded
               history. Our production needed to move to the southern
               tip of this planet just to be able to find snow. Climate
               change is real, it is happening right now. It is the most
               urgent threat facing our entire species, and we need to work
               collectively together and stop procrastinating. We need to
               support leaders around the world who do not speak for the 
               big polluters, but who speak for all of humanity, for the
               indigenous people of the world, for the billions and 
               billions of underprivileged people out there who would be
               most affected by this. For our children’s children, and 
               for those people out there whose voices have been drowned
               out by the politics of greed. I thank you all for this 
               amazing award tonight. Let us not take this planet for 
               granted. I do not take tonight for granted. Thank you so very much."""
               
# -

#  we need BOW model becaues any ml or dl algo only understands numbers or float so the BOWM is a necessaty

data = nltk.sent_tokenize(paragraph)

words = nltk.word_tokenize(paragraph)

len(words)

for i in range(len(data)):
    data[i] = data[i].lower()
    #lowercasing the dataset
    data[i] = re.sub(r"\W",' ',data[i])
    # puncuation chars
    data[i] = re.sub(r"\s+",' ',data[i])

data

# +
#Creating The histogram
# -

#per line tokenizing
word2count = {}
for datas in data:
    words = nltk.word_tokenize(datas)
    for word in words:
        if word not in word2count.keys():
            word2count[word] = 1
            #jodi na thake tale atleast 
            #it has apppeared once
            
        else:
            word2count[word] += 1
            #jodi thake tale count barachi            

word2count

len(word2count)

import heapq
# used to find the N most frequent words in the dict

# +

freq_words = heapq.nlargest(100, 
                          word2count, 
                          key=word2count.get)
# -

freq_words

# +
#Creating the BOG model

X = [] # it will contain the BOW model

# in a BOw each words is a vector 1 - apppears & 0- not appears
for datas in data:
    vector = []
    for word in freq_words:
        if word in nltk.word_tokenize(datas):
            vector.append(1)
        else:
            vector.append(0)
    X.append(vector)
# -

len(data)

import numpy as np
np.array(X)

np.array(X).shape

X = np.asarray(X)
X

# # Text modelling using TF - IDF Model

import nltk
import re
import heapq
import numpy as np

paragraph = """Thank you all so very much. Thank you to the Academy. 
               Thank you to all of you in this room. I have to congratulate 
               the other incredible nominees this year. The Revenant was 
               the product of the tireless efforts of an unbelievable cast
               and crew. First off, to my brother in this endeavor, Mr. Tom 
               Hardy. Tom, your talent on screen can only be surpassed by 
               your friendship off screen … thank you for creating a t
               ranscendent cinematic experience. Thank you to everybody at 
               Fox and New Regency … my entire team. I have to thank 
               everyone from the very onset of my career … To my parents; 
               none of this would be possible without you. And to my 
               friends, I love you dearly; you know who you are. And lastly,
               I just want to say this: Making The Revenant was about
               man's relationship to the natural world. A world that we
               collectively felt in 2015 as the hottest year in recorded
               history. Our production needed to move to the southern
               tip of this planet just to be able to find snow. Climate
               change is real, it is happening right now. It is the most
               urgent threat facing our entire species, and we need to work
               collectively together and stop procrastinating. We need to
               support leaders around the world who do not speak for the 
               big polluters, but who speak for all of humanity, for the
               indigenous people of the world, for the billions and 
               billions of underprivileged people out there who would be
               most affected by this. For our children’s children, and 
               for those people out there whose voices have been drowned
               out by the politics of greed. I thank you all for this 
               amazing award tonight. Let us not take this planet for 
               granted. I do not take tonight for granted. Thank you so very much."""


# Tokenize sentences
dataset = nltk.sent_tokenize(paragraph)
for i in range(len(dataset)):
    dataset[i] = dataset[i].lower()
    dataset[i] = re.sub(r'\W',' ',dataset[i])
    dataset[i] = re.sub(r'\s+',' ',dataset[i])

#per line tokenizing
# Creating word histogram
word2count = {}
for data in dataset:
    words = nltk.word_tokenize(data)
    for word in words:
        if word not in word2count.keys():
            word2count[word] = 1
            #jodi na thake tale atleast 
            #it has apppeared once
        else:
            word2count[word] += 1
            #jodi thake tale count barachi  

# Selecting best 100 features
freq_words = heapq.nlargest(100,
                            word2count,
                            key=word2count.get)

# +
#Creating the BOG model

X = [] # it will contain the BOW model

# in a BOw each words is a vector 1 - apppears & 0- not appears
for datas in data:
    vector = []
    for word in freq_words:
        if word in nltk.word_tokenize(datas):
            vector.append(1)
        else:
            vector.append(0)
    X.append(vector)
# -

X = np.asarray(X)
X

# +
# IDF Dictionary

word_idfs = {}
for word in freq_words:
    doc_count = 0
    for data in dataset:
        if word in nltk.word_tokenize(data):
            doc_count += 1
    word_idfs[word] = np.log(len(dataset)/(1+doc_count))
    # +1 is kindof like a bias
# -

len(word_idfs)

# TF Matrix
tf_matrix = {}
for word in freq_words:
    doc_tf = []
    for data in dataset:
        frequency = 0
        for w in nltk.word_tokenize(data):
            if word == w:
                frequency += 1
        tf_word = frequency/len(nltk.word_tokenize(data))
        doc_tf.append(tf_word)
    tf_matrix[word] = doc_tf   

# TF - IDF Calculation
tfidf_matrix  = []
for word in tf_matrix.keys():
    tfidf = []
    for value in tf_matrix[word]:
        score = value * word_idfs[word]
        tfidf.append(score)
    tfidf_matrix.append(tfidf)  

X = np.asarray(tfidf_matrix).astype('float64')
X

X.shape

X = X.T
X.shape



StarcoderdataJupyterScriptsDedupFiltered