# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + colab={"base_uri": "https://localhost:8080/"} id="L7vqmAE5ym4U" executionInfo={"status": "ok", "timestamp": 1611042719987, "user_tz": 480, "elapsed": 6175, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GimIntWE1SvxJeoWOj0PwvUuB_-simILm2JI8het08=s64", "userId": "04899005791450850059"}} outputId="ba2b0ca1-5197-485f-8ec7-da8eb90d5941"
#@title ## Download Kaggle Dataset
#@markdown Dataset: Annotated Corpus for Named Entity Recognition <br>
#@markdown [https://www.kaggle.com/therohk/million-headlines](https://www.kaggle.com/therohk/million-headlines)
#@markdown <br><br>
#@markdown News headlines published over a period of seventeen years.
#@markdown <br><br>
#@markdown Setup your [Kaggle API](https://www.kaggle.com/docs/api) access to download the dataset.
# !mkdir -p ~/.kaggle
# !cp /content/drive/MyDrive/kaggle/kaggle.json ~/.kaggle/kaggle.json
# !chmod 600 ~/.kaggle/kaggle.json
# !kaggle datasets download -d therohk/million-headlines
# !unzip -o /content/million-headlines

# + colab={"base_uri": "https://localhost:8080/"} id="AWg1An-_yhM6" executionInfo={"status": "ok", "timestamp": 1611042721383, "user_tz": 480, "elapsed": 7557, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GimIntWE1SvxJeoWOj0PwvUuB_-simILm2JI8het08=s64", "userId": "04899005791450850059"}} outputId="1816af43-a416-47c0-c7fb-6a5f6708eeb9"
import pandas as pd
import numpy as np
import tensorflow as tf
import functools

import matplotlib.pylab as plt

from tensorflow.keras.layers import Embedding, Dense, Flatten
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam

print("tensorflow      ", tf.__version__)
print("pandas          ", pd.__version__)
print("numpy           ", np.__version__)
print("matplotlib.pylab", plt.__version__)

# + colab={"base_uri": "https://localhost:8080/", "height": 34} id="-1b4eVBh2pU7" executionInfo={"status": "ok", "timestamp": 1611042722759, "user_tz": 480, "elapsed": 8923, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GimIntWE1SvxJeoWOj0PwvUuB_-simILm2JI8het08=s64", "userId": "04899005791450850059"}} outputId="3df0808f-aae0-40e5-f7f4-863fb14f9491"
headlines = pd.read_csv("abcnews-date-text.csv")["headline_text"]
headlines[0]

# + colab={"base_uri": "https://localhost:8080/"} id="up-WrxazDnBL" executionInfo={"status": "ok", "timestamp": 1611042723410, "user_tz": 480, "elapsed": 9566, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GimIntWE1SvxJeoWOj0PwvUuB_-simILm2JI8het08=s64", "userId": "04899005791450850059"}} outputId="86edaaec-2ec5-44a8-99a7-e97265e1b1bc"
window = [tuple([h[i], h[i+1], h[i+2]]) for h in headlines[:10000] for i in range(len(h) - 2)]
window = pd.DataFrame(window)
vocab = window[0].unique()
vocab = {v: i for i, v in enumerate(vocab)}
X = window.iloc[:,:2].applymap(lambda t: vocab[t]).to_numpy()
y = window.iloc[:,-1:].applymap(lambda t: vocab[t]).to_numpy()

X.shape, y.shape 

# + colab={"base_uri": "https://localhost:8080/"} id="4YsTIFfFZD2n" executionInfo={"status": "ok", "timestamp": 1611042726596, "user_tz": 480, "elapsed": 12745, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GimIntWE1SvxJeoWOj0PwvUuB_-simILm2JI8het08=s64", "userId": "04899005791450850059"}} outputId="06a6ba16-41e4-4ba5-82f3-cd63b7f01dce"
vocab_size = len(vocab)
embedding_dim = 2

model = Sequential([
    Embedding(vocab_size, embedding_dim, input_length=2, name="embedding"),
    Flatten(),
    Dense(vocab_size, activation="softmax")
])

adam = Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False)
model.compile(optimizer=adam, loss="categorical_crossentropy", metrics=["accuracy"])

output_array = model.get_layer("embedding")(X)
output_array.shape

# + colab={"base_uri": "https://localhost:8080/", "height": 265} id="MhV9nt9jZ7BB" executionInfo={"status": "ok", "timestamp": 1611042726941, "user_tz": 480, "elapsed": 13083, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GimIntWE1SvxJeoWOj0PwvUuB_-simILm2JI8het08=s64", "userId": "04899005791450850059"}} outputId="1a1bd426-803a-43a3-9abe-dd3d7aa41c7f"
import matplotlib.pylab as plt

idx_to_calc = list(vocab.values())
idx_to_calc = np.array([idx_to_calc, idx_to_calc]).T

translator = {v:k for k,v in vocab.items()}
preds = model.get_layer("embedding")(idx_to_calc)

plt.scatter(preds[:, 0, 0], preds[:, 0, 1], alpha=0)
for i, idx in enumerate(idx_to_calc):
  plt.text(preds[i, 0, 0], preds[i, 0, 1], translator[idx[0]])


# + colab={"base_uri": "https://localhost:8080/"} id="p9djX-7JbkHa" executionInfo={"status": "ok", "timestamp": 1611043550570, "user_tz": 480, "elapsed": 836705, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GimIntWE1SvxJeoWOj0PwvUuB_-simILm2JI8het08=s64", "userId": "04899005791450850059"}} outputId="858f0d59-8d46-4b55-c517-125f6f7d1793"
from sklearn.preprocessing import OneHotEncoder

y_one_hot = OneHotEncoder(sparse=False).fit_transform(y)
model.fit(X, y_one_hot, epochs=30, verbose=1)

# + colab={"base_uri": "https://localhost:8080/", "height": 265} id="2rB80wchbmUx" executionInfo={"status": "ok", "timestamp": 1611043556243, "user_tz": 480, "elapsed": 634, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GimIntWE1SvxJeoWOj0PwvUuB_-simILm2JI8het08=s64", "userId": "04899005791450850059"}} outputId="cab1e7eb-60ec-4dbe-fdae-74fde2fab85e"
preds = model.get_layer("embedding")(idx_to_calc)
plt.scatter(preds[:, 0, 0], preds[:, 0, 1], alpha=0)
for i, idx in enumerate(idx_to_calc):
  plt.text(preds[i, 0, 0], preds[i, 0, 1], translator[idx[0]])

StarcoderdataJupyterScriptsDedupFiltered