# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="jxWY-dyTGpUp" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="0bc957e9-b91e-41ad-b435-dc4f91c1dad8"
# google drive settings 
from google.colab import drive 
drive.mount('/content/gdrive')

# + id="J-rA0tLwGpl3" colab_type="code" colab={}
# google drive settings 
# %%capture
# %cd /content/gdrive/My\ Drive/NLP

# + id="p5wt0IsQCQE5" colab_type="code" colab={}
from collections import defaultdict
import numpy as np
import pandas as pd
import nltk

# + id="JKmtMgnwGsCz" colab_type="code" colab={}
first_week = pd.read_csv("tweet_data/first_week.csv")
second_week = pd.read_csv("tweet_data/second_week.csv")
third_week = pd.read_csv("tweet_data/third_week.csv")
fourth_week = pd.read_csv("tweet_data/fourth_week.csv")

# + id="gNvY0PALauZT" colab_type="code" colab={}
first_week = first_week[first_week.clean_text.isna()==False] # drop all tweets without text
second_week = second_week[second_week.clean_text.isna()==False] # drop all tweets without text
third_week = third_week[third_week.clean_text.isna()==False] # drop all tweets without text
fourth_week = fourth_week[fourth_week.clean_text.isna()==False] # drop all tweets without text


# + id="AQ-b_EkcCshP" colab_type="code" colab={}
class LM:
 
  def __init__(self, smoothing, n_grams_size):
    self.smoothing = smoothing
    self.n_grams_size = n_grams_size
    self.counts = defaultdict(lambda: defaultdict(lambda: smoothing))
    self.START = '_***_'
    self.STOP = '_STOP_'
 
  def fit(self, document):
    corpus = [line.strip().split() for line in document]
  
    for sentence in corpus:
      tokens = [self.START for _ in range(self.n_grams_size)] + sentence + [self.STOP]
      for n_grams_tuple in nltk.ngrams(tokens, self.n_grams_size):
        self.counts[n_grams_tuple[:-1]][n_grams_tuple[-1]] += 1
  
  def _sample_next_word(self, history_tuple, iters=50):
    for _ in range(iters):
      keys, values = zip(*self.counts[history_tuple].items())
      values = np.array(values)
      values /= values.sum()
      sample = np.random.multinomial(1, values) 
    
    return keys[np.argmax(sample)]
  
  def generate(self, words_list = []):
    result = [self.START for _ in range(self.n_grams_size)]
    if words_list: 
      result.extend(words_list)

    history_tuple = tuple([result[i] for i in range(-self.n_grams_size+1, 0, 1)])
    next_word = self._sample_next_word(history_tuple)
    result.append(next_word)
  
    while next_word != self.STOP:
      history_tuple = tuple(result[i] for i in range(-self.n_grams_size+1, 0, 1))
      next_word = self._sample_next_word(history_tuple)
      result.append(next_word)
  
    return ' '.join(result[self.n_grams_size:-1])


# + id="A__j0YwvGoYB" colab_type="code" colab={}
def create_lm(dataset, n=4): 
  tweets = dataset['clean_tweet'].to_list()
  lm = LM(smoothing=0.001, n_grams_size=n)
  lm.fit(document=tweets)
  return lm 


# + [markdown] id="JGsvfjtrweYr" colab_type="text"
# ## 7-gram Model

# + id="YssQH8PyGAqk" colab_type="code" colab={}
#create language model for each week 
w1 = create_lm(first_week, n=7)
w2 = create_lm(second_week, n=7)
w3 = create_lm(third_week, n=7)
w4 = create_lm(fourth_week, n=7)

# + id="Jmg-qH70MrrC" colab_type="code" colab={}
number_of_tweets = 5

# + id="PMlcnah9MkWN" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} outputId="eb582a9b-e136-4e79-f5ff-17a5c472900d"
print(np.unique([w1.generate() for _ in range(number_of_tweets)]))

# + id="1HDTl3xCKp6d" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} outputId="6b4cfadb-d688-4b09-b2d5-8a4ba5c37151"
print(np.unique([w2.generate() for _ in range(number_of_tweets)]))

# + id="bl5WUELBL79A" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 101} outputId="1bcbee66-ad8c-4211-a9d7-0983c1626abc"
print(np.unique([w3.generate() for _ in range(number_of_tweets)]))

# + id="vHcH8kZIL-iO" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} outputId="9b41d524-8f29-4de4-f16c-4ab0d4dd24dc"
print(np.unique([w4.generate() for _ in range(number_of_tweets)]))

# + id="iBvevziXZJ5D" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 101} outputId="87a142d2-13ed-4095-ff69-490e398e96cb"
print(np.unique([w1.generate(['coronavirus', 'è']) for _ in range(number_of_tweets)]))

# + id="_UUFObYSZJyV" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} outputId="2b4d6d5a-c271-4cc7-ccf8-d9883c679eab"
print(np.unique([w2.generate(['coronavirus', 'è']) for _ in range(number_of_tweets)]))

# + id="wEkkgimcZJmR" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} outputId="44546a03-1238-429f-c9c5-a612353d6e0a"
print(np.unique([w3.generate(['coronavirus', 'è']) for _ in range(number_of_tweets)]))

# + id="NTCP1RkRZB_r" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} outputId="290df0c6-591c-4bbc-f204-1dca42223985"
print(np.unique([w4.generate(['coronavirus', 'è']) for _ in range(number_of_tweets)]))

# + [markdown] id="CjncCSYpwaAy" colab_type="text"
# ## 4-gram Model

# + id="LbDfnR-3fiOG" colab_type="code" colab={}
#create language model for each week 
w1_n4 = create_lm(first_week, n=4)
w2_n4 = create_lm(second_week, n=4)
w3_n4 = create_lm(third_week, n=4)
w4_n4 = create_lm(fourth_week, n=4)

# + id="1CEhF4G8cFCf" colab_type="code" colab={}
number_of_tweets = 5

# + id="l1F3RNWngk5Y" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} outputId="5743b74e-c832-4c8f-e1d4-45cb5c0fd3dd"
print(np.unique([w1_n4.generate(['conte']) for _ in range(number_of_tweets)]))

# + id="qgsAYnszgnR6" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} outputId="52b4c542-9376-4752-f635-cbc7107f5186"
print(np.unique([w2_n4.generate(['conte']) for _ in range(number_of_tweets)]))

# + id="8iInVMC7gnsY" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} outputId="a960c19f-c70a-48b5-bf4a-9ce8f68c2537"
print(np.unique([w3_n4.generate(['conte']) for _ in range(number_of_tweets)]))

# + id="OrM5U0IrgoPW" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 121} outputId="518f385d-a20c-44dd-bf46-14428f3fe54d"
print(np.unique([w4_n4.generate(['conte']) for _ in range(number_of_tweets)]))

# + id="0wzR-n2irD_N" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 101} outputId="21bf42a6-d654-4fed-da23-9d8ce57dcc98"
print(np.unique([w4_n4.generate(['conte']) for _ in range(5)]))

# + id="SoIr3xBwcI0j" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered