# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import json
import numpy as np
import os
import re
import nltk

words = set(nltk.corpus.words.words())

stopwords=['','(',')','{','}','\\','--',':']

# +
tweets=[]
for line in open("20110123-000.json", 'r'):
    try:
        tweets.append(json.loads(line))
    except:
        continue
        
tweets[14]        
# -

test_tweets= []
for line in open("20110123-006.json", 'r'):
    try:
        test_tweets.append(json.loads(line))
    except:
        continue

# # Making language model:

from nltk.util import ngrams
import string
from nltk import SimpleGoodTuringProbDist
from math import log

corpus = nltk.corpus.reader.plaintext.PlaintextCorpusReader(".", "sample.txt")
test = nltk.corpus.reader.plaintext.PlaintextCorpusReader(".", "test_data.txt")
print(test)
corpus

# ## Unigram perplexity with add 1 smoothing\ Laplacian smoothing

unigram= [w.strip(string.punctuation).lower() for w in corpus.words()]
len(unigram)
#unigram[:20]

spl= int(95*len(unigram)/100)
train_corpus= unigram[:spl]
test_corpus= unigram[spl:]

freq_1gram= nltk.FreqDist(train_corpus)
len_train= len(train_corpus)
vocab= len(set(train_corpus))


def unigram_prob_with_add1smoothing(word):
    return (freq_1gram[word]+1)/(len_train+vocab)


# +
def entropy(n, text):
        e = 0.0
        text = ["<s>"] + text + ["</s>"]
        for i in range(n - 1, len(text)):
            context = text[i - n + 1:i]
            token = text[i]
            #print(str(context)+"    "+token)
            e += logprob(token, context)
        return e / float(len(text) - (n - 1))


def logprob(word, context):
    if len(context)==0:
        p=unigram_prob_with_add1smoothing(word)
    elif len(context)==1:
        p=bigram_prob_with_add1smoothing(context[0], word)
    else:
        p=trigram_prob_with_add1smoothing(context[0], context[1], word)
    return -p*log(p , 2)


def perplexity(n, text):
      return pow(2.0, entropy(n, text))


# -

perplexity(1, test_corpus)

perplexity(1, train_corpus)

# ## Bigram Perplexity with add 1 smoothing

bigram= [a for a in ngrams(unigram, 2, pad_left=True,pad_right=True,left_pad_symbol='<s>', right_pad_symbol="</s>")]
len(bigram)

cfreq_2gram = nltk.ConditionalFreqDist(bigram)
cprob_2gram = nltk.ConditionalProbDist(cfreq_2gram, nltk.MLEProbDist)
def bigram_prob_with_add1smoothing(word1, word2):
    cprob_2gram_add1=float((((1+cfreq_2gram[word1][word2])/(len(cfreq_2gram)+sum(cfreq_2gram[word1].values())))))
    return cprob_2gram_add1



perplexity(2, test_corpus)

perplexity(2, train_corpus)


def bigram_prob_with_gtsmoothing(word1, word2):
    return SimpleGoodTuringProbDist(word1, word2)


# +
def entropy(n, text):
        e = 0.0
        text = ["<s>"] + text + ["</s>"]
        for i in range(n - 1, len(text)):
            context = text[i - n + 1:i]
            token = text[i]
            #print(str(context)+"    "+token)
            e += logprob_gt(token, context)
        return e / float(len(text) - (n - 1))


def logprob_gt(word, context):
    if len(context)==0:
        p=unigram_prob_with_gtsmoothing(word)
    elif len(context)==1:
        p=bigram_prob_with_gtsmoothing(context[0], word)
    else:
        p=trigram_prob_with_gtsmoothing(context[0], context[1], word)
    return -p*log(p , 2)


def perplexity(n, text):
      return pow(2.0, entropy(n, text))


# -

# ## Trigram perplexity with add 1 smoothing

trigram= [a for a in ngrams(unigram, 3, pad_left= True, pad_right=True, left_pad_symbol= "<s>", right_pad_symbol="</s>")]
len(trigram)

trigrams_as_bigrams=[]
trigrams_as_bigrams.extend([((t[0],t[1]), t[2]) for t in trigram])
cfreq_3gram = nltk.ConditionalFreqDist(trigrams_as_bigrams)
cprob_3gram = nltk.ConditionalProbDist(cfreq_3gram, nltk.MLEProbDist)
def trigram_prob_with_add1smoothing(w1, w2, w3):
    cprob_3gram_add1=float((((1+cfreq_3gram[(w1,w2)][w3])/(len(cfreq_3gram)+sum(cfreq_3gram[(w1,w2)].values())))))
    return cprob_3gram_add1


perplexity(3, test_corpus)

perplexity(3, train_corpus)

# ## Trying to generate text from corpus:

from collections import Counter, OrderedDict, defaultdict
from operator import itemgetter
import random

# +
test_unigram = [w.strip(string.punctuation).lower() for w in test.words()]

test_bigram= [a for a in ngrams(test_unigram, 2)]
len(test_bigram)

test_trigram= [a for a in ngrams(test_unigram, 3)]
len(test_trigram)

test_frequency= nltk.FreqDist(test_bigram)
test_sorted_freq= OrderedDict(reversed(sorted(test_frequency.items(), key= itemgetter(1))))

test_cfreq_2gram= nltk.ConditionalFreqDist(test_bigram)

test_cprob_2gram= nltk.ConditionalProbDist(test_cfreq_2gram, nltk.MLEProbDist)
# -

counts = Counter(corpus.words())
total_count = len(corpus.words())

counts.most_common(20)

for word in counts:
    counts[word] /= float(total_count)

# generating 100 words at random of the language
text=[]
for _ in range(100):
    r = random.random()
    accumulator = .0
 
    for word, freq in counts.items():
        accumulator += freq
 
        if accumulator >= r:
            text.append(word)
            break
print(' '.join(text))

# +
model = defaultdict(lambda: defaultdict(lambda : 0))
for sentence in corpus.sents():
    for w1, w2, w3 in ngrams(sentence, 3, pad_left= True, pad_right=True):
        model[(w1, w2)][w3] +=1

#transform counts to probabilities
for w1_w2 in model:
    total_count = float(sum(model[w1_w2].values()))
    for w3 in model[w1_w2]:
        model[w1_w2][w3] /= total_count


# +
text = [None, None]
sentence_finished = False
while not sentence_finished:
    r= random.random()
    accumulator = 0.0
    for word in model[tuple(text[-2:])].keys():
        accumulator += model[tuple(text[-2:])][word]
        if(accumulator > r):
            text.append(word)
            break
        if(text[-2:] == [None, None]):
            sentence_finished = True
            
print(" ".join([t for t in text if t]))            
# -









StarcoderdataJupyterScriptsDedupFiltered