# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="y-8yg9DvTmce" colab_type="code" colab={}
# %matplotlib inline
import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
from scipy.spatial.distance import cdist

# + id="BGwnrmfSTrn9" colab_type="code" colab={}
# from tf.keras.models import Sequential  # This does not work!
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense, GRU, Embedding
from tensorflow.python.keras.optimizers import Adam
from tensorflow.python.keras.preprocessing.text import Tokenizer
from tensorflow.python.keras.preprocessing.sequence import pad_sequences

# + id="tIT1GROATvFw" colab_type="code" outputId="a73270a5-f7a5-44f7-924d-840695585389" colab={"base_uri": "https://localhost:8080/", "height": 35}
tf.__version__

# + id="n_874Lo5TzUS" colab_type="code" outputId="b8280025-84ef-4836-8e0f-fb65f3345139" colab={"base_uri": "https://localhost:8080/", "height": 35}
tf.keras.__version__

# + id="S5isfTF6T3K_" colab_type="code" colab={}
import imdb

# + id="ga-lit1DT_ce" colab_type="code" outputId="4247152d-94c1-4006-a3ed-4e9ebfdba2bf" colab={"base_uri": "https://localhost:8080/", "height": 69}
imdb.maybe_download_and_extract()

# + id="EiemxMsDUSUk" colab_type="code" colab={}
x_train_text, y_train = imdb.load_data(train=True)
x_test_text, y_test = imdb.load_data(train=False)

# + id="XwzEu3cjUYvQ" colab_type="code" outputId="e0f939b6-a68f-4784-9180-ac2511e6e593" colab={"base_uri": "https://localhost:8080/", "height": 52}
print("Train-set size: ", len(x_train_text))
print("Test-set size:  ", len(x_test_text))

# + id="JEwUAZ0RUdk1" colab_type="code" colab={}
data_text = x_train_text + x_test_text

# + id="u30_aQPoUkSI" colab_type="code" outputId="17d5a285-35e1-48a9-861e-87196c4d746a" colab={"base_uri": "https://localhost:8080/", "height": 55}
x_train_text[1]

# + id="upVK7HNuUqGF" colab_type="code" outputId="a1c0e080-3a29-452f-df59-e414bfb98fe8" colab={"base_uri": "https://localhost:8080/", "height": 35}
y_train[1]

# + id="e_dsDCoMUwwU" colab_type="code" colab={}
num_words = 10000

# + id="7GNj1fkLU339" colab_type="code" colab={}
tokenizer = Tokenizer(num_words=num_words)

# + id="OwV7auU4U9fe" colab_type="code" outputId="b3df2592-1095-4942-ebe8-ac613857bfd6" colab={"base_uri": "https://localhost:8080/", "height": 52}
# %%time
tokenizer.fit_on_texts(data_text)

# + id="p_gEgR8dVFlH" colab_type="code" colab={}
if num_words is None:
    num_words = len(tokenizer.word_index)

# + id="i5vtzuo1VOYp" colab_type="code" outputId="b90f7b6c-0fae-45da-e421-e263afe25a4e" colab={"base_uri": "https://localhost:8080/", "height": 1000}
tokenizer.word_index

# + id="t_bMyGFpVTcU" colab_type="code" colab={}
x_train_tokens = tokenizer.texts_to_sequences(x_train_text)

# + id="JHFAjQLCVs6a" colab_type="code" outputId="62e380fe-fac3-4721-835b-783b38d462e3" colab={"base_uri": "https://localhost:8080/", "height": 55}
x_train_text[1]

# + id="1XcaGQCvVxyH" colab_type="code" outputId="ef87676a-4ed6-4243-ec0d-16ad190dcdfe" colab={"base_uri": "https://localhost:8080/", "height": 1000}
np.array(x_train_tokens[1])

# + id="t7w0AMpHWiPz" colab_type="code" colab={}
x_test_tokens = tokenizer.texts_to_sequences(x_test_text)


# + id="okzxeaiXWlvN" colab_type="code" colab={}
num_tokens = [len(tokens) for tokens in x_train_tokens + x_test_tokens]
num_tokens = np.array(num_tokens)

# + id="Deh-2qpWWspQ" colab_type="code" outputId="c04fd445-600b-429f-fe97-c3c6736e55bd" colab={"base_uri": "https://localhost:8080/", "height": 35}
np.mean(num_tokens)


# + id="IPE2KrLBWylN" colab_type="code" outputId="87beb063-db31-4f45-d3eb-ed76179798ee" colab={"base_uri": "https://localhost:8080/", "height": 35}
np.max(num_tokens)


# + id="pEJrsZFlW1m0" colab_type="code" outputId="ed1029a8-1935-4953-c1c5-ace382a0d676" colab={"base_uri": "https://localhost:8080/", "height": 35}
max_tokens = np.mean(num_tokens) + 2 * np.std(num_tokens)
max_tokens = int(max_tokens)
max_tokens

# + id="xyUNdYEFW7W4" colab_type="code" outputId="a1dd0951-a091-47e0-e0bc-9f69821e9788" colab={"base_uri": "https://localhost:8080/", "height": 35}
np.sum(num_tokens < max_tokens) / len(num_tokens)

# + id="OGR98RyrW_sH" colab_type="code" colab={}
pad = 'pre'

# + id="yN5RgZmrXEVq" colab_type="code" colab={}
x_train_pad = pad_sequences(x_train_tokens, maxlen=max_tokens,
                            padding=pad, truncating=pad)

# + id="VxnfAP8bXI3t" colab_type="code" colab={}
x_test_pad = pad_sequences(x_test_tokens, maxlen=max_tokens,
                           padding=pad, truncating=pad)

# + id="IIFojpUAXP6F" colab_type="code" outputId="d5b501f4-d71e-499c-a459-37ab6cb35b38" colab={"base_uri": "https://localhost:8080/", "height": 35}
x_train_pad.shape

# + id="b3FT6gRHXRIA" colab_type="code" outputId="516840e7-cf06-42a0-991c-d49f430dcb78" colab={"base_uri": "https://localhost:8080/", "height": 35}
x_test_pad.shape


# + id="ma2q04KnXUV2" colab_type="code" outputId="b16114e7-dcaa-41fc-e639-5d4885daacf8" colab={"base_uri": "https://localhost:8080/", "height": 1000}
np.array(x_train_tokens[1])

# + id="-wRrZXpwXcKT" colab_type="code" outputId="eb032ee6-dffd-4de0-b665-1e9f240a82a3" colab={"base_uri": "https://localhost:8080/", "height": 884}
x_train_pad[1]

# + id="24e4gLnFXghS" colab_type="code" colab={}
idx = tokenizer.word_index
inverse_map = dict(zip(idx.values(), idx.keys()))


# + id="bquK6zXcXvdB" colab_type="code" colab={}
def tokens_to_string(tokens):
    # Map from tokens back to words.
    words = [inverse_map[token] for token in tokens if token != 0]
    
    # Concatenate all words.
    text = " ".join(words)

    return text


# + id="7th5snR4Xzzx" colab_type="code" outputId="fede5805-9619-4b89-d07d-2b4672315a89" colab={"base_uri": "https://localhost:8080/", "height": 55}
x_train_text[1]

# + id="M-e9H_CNX8O2" colab_type="code" outputId="6b6ee288-a93f-497f-ee73-d5a60700df46" colab={"base_uri": "https://localhost:8080/", "height": 55}
tokens_to_string(x_train_tokens[1])

# + id="MhBvz3EFX-aD" colab_type="code" colab={}
model = Sequential()

# + id="X91-12ARYCie" colab_type="code" colab={}
embedding_size = 8

# + id="nMBwcy4hYFu2" colab_type="code" outputId="29c23b3e-9daf-41b7-b855-b9bc2bf8a8b5" colab={"base_uri": "https://localhost:8080/", "height": 107}
model.add(Embedding(input_dim=num_words,
                    output_dim=embedding_size,
                    input_length=max_tokens,
                    name='layer_embedding'))

# + id="IAfhROjlYJvx" colab_type="code" outputId="46076547-e512-43a0-de50-4499befff4e3" colab={"base_uri": "https://localhost:8080/", "height": 89}
model.add(GRU(units=16, return_sequences=True))

# + id="3o85eMPHYO9A" colab_type="code" colab={}
model.add(GRU(units=8, return_sequences=True))

# + id="DzLXSZ8zYYLF" colab_type="code" colab={}
model.add(GRU(units=4))

# + id="4gA3K4U6YaGI" colab_type="code" colab={}
model.add(Dense(1, activation='sigmoid'))

# + id="9lkTxp6_YfTl" colab_type="code" colab={}
optimizer = Adam(lr=1e-3)

# + id="8UC_Zry3YgjZ" colab_type="code" outputId="92e2887f-d915-4728-f573-28052c318c40" colab={"base_uri": "https://localhost:8080/", "height": 89}
model.compile(loss='binary_crossentropy',
              optimizer=optimizer,
              metrics=['accuracy'])

# + id="Xg9gByRwYku3" colab_type="code" outputId="4f038b93-7c68-4021-d926-b4e97203458c" colab={"base_uri": "https://localhost:8080/", "height": 329}
model.summary()

# + id="jlNERlTBYr_5" colab_type="code" outputId="9e5c084a-dd34-4345-817c-9f001d051875" colab={"base_uri": "https://localhost:8080/", "height": 225}
# %%time
model.fit(x_train_pad, y_train,
          validation_split=0.05, epochs=4, batch_size=64)

# + id="M5w4FzgScOI_" colab_type="code" outputId="af5e2d64-f5e7-4f83-bed5-a30ed91fd3e0" colab={"base_uri": "https://localhost:8080/", "height": 69}
# %%time
result = model.evaluate(x_test_pad, y_test)

# + id="ZlRezmWkcq2V" colab_type="code" outputId="1e4e85ba-3a3e-4b13-fe59-dce7d5211792" colab={"base_uri": "https://localhost:8080/", "height": 35}
print("Accuracy: {0:.2%}".format(result[1]))

# + id="z8IClF_Tcy-l" colab_type="code" outputId="e7c0d51f-0639-4ac5-b5eb-872131bb79e3" colab={"base_uri": "https://localhost:8080/", "height": 381}
# %%time
y_pred = model.predict(x=x_test_pad[0:1000])
y_pred = y_pred.T[0]

# + id="APetRl_sc8FX" colab_type="code" colab={}
cls_pred = np.array([1.0 if p>0.5 else 0.0 for p in y_pred])

# + id="n2Lz7LOodAcS" colab_type="code" colab={}
cls_true = np.array(y_test[0:1000])

# + id="lEDNyGS9dDoy" colab_type="code" colab={}
incorrect = np.where(cls_pred != cls_true)
incorrect = incorrect[0]

# + id="TQKcThxMdHJQ" colab_type="code" outputId="651c962d-b00f-46f8-e786-ff2a0e03532b" colab={"base_uri": "https://localhost:8080/", "height": 35}
len(incorrect)

# + id="PW-wt9-YdKQ_" colab_type="code" outputId="3d081935-4ce1-4fc1-b14e-abb8ab961fc2" colab={"base_uri": "https://localhost:8080/", "height": 35}
idx = incorrect[0]
idx

# + id="Mnqg2CUTdLkA" colab_type="code" outputId="450b6162-fc6f-4a4c-a27d-cf2c68aeab31" colab={"base_uri": "https://localhost:8080/", "height": 55}
text = x_test_text[idx]
text

# + id="srSgSdsqdTe9" colab_type="code" outputId="65aeb30d-d908-40d0-d65a-3b000c3cb3e4" colab={"base_uri": "https://localhost:8080/", "height": 35}
y_pred[idx]

# + id="5Bix1oGsdU3-" colab_type="code" outputId="6277d206-49ca-43d2-e56d-7f741fc15fcb" colab={"base_uri": "https://localhost:8080/", "height": 35}
cls_true[idx]

# + id="kSesar2PdYWA" colab_type="code" colab={}
text1 = "This movie is fantastic! I really like it because it is so good!"
text2 = "Good movie!"
text3 = "Maybe I like this movie."
text4 = "Meh ..."
text5 = "If I were a drunk teenager then this movie might be good."
text6 = "Bad movie!"
text7 = "Not a good movie!"
text8 = "This movie really sucks! Can I get my money back please?"
texts = [text1, text2, text3, text4, text5, text6, text7, text8]

# + id="I22XRhQUddNJ" colab_type="code" colab={}
tokens = tokenizer.texts_to_sequences(texts)

# + id="AfoeDJ1mdjMA" colab_type="code" outputId="83d4847c-be7a-40db-fb80-8806c2de6396" colab={"base_uri": "https://localhost:8080/", "height": 35}
tokens_pad = pad_sequences(tokens, maxlen=max_tokens,
                           padding=pad, truncating=pad)
tokens_pad.shape

# + id="uchPXJw9dovJ" colab_type="code" outputId="125f6e36-0424-4035-e219-f0b510d2182a" colab={"base_uri": "https://localhost:8080/", "height": 156}
model.predict(tokens_pad)

# + id="OlPUAlGmdqet" colab_type="code" colab={}
layer_embedding = model.get_layer('layer_embedding')

# + id="i76V9ILwdwh4" colab_type="code" colab={}
weights_embedding = layer_embedding.get_weights()[0]

# + id="BngERaPvdyzz" colab_type="code" outputId="61cb39e5-a6ab-4995-88d9-3f17fc83ff41" colab={"base_uri": "https://localhost:8080/", "height": 35}
weights_embedding.shape

# + id="I0VFuC5Od2OL" colab_type="code" outputId="59f54cd7-7cd6-483c-bac0-4abb53eec0a4" colab={"base_uri": "https://localhost:8080/", "height": 35}
token_good = tokenizer.word_index['good']
token_good

# + id="mN7AQb6Kd39I" colab_type="code" outputId="8be63d59-4bbf-4a02-b67b-85c<PASSWORD>f44" colab={"base_uri": "https://localhost:8080/", "height": 35}
token_great = tokenizer.word_index['great']
token_great

# + id="BcE9ujAXeAld" colab_type="code" outputId="8d069a59-1224-4c34-ce8b-217376160808" colab={"base_uri": "https://localhost:8080/", "height": 52}
weights_embedding[token_good]

# + id="Vusg1zw1eB50" colab_type="code" outputId="3bbc4578-79dc-4836-98e9-14fd81a8d228" colab={"base_uri": "https://localhost:8080/", "height": 52}
weights_embedding[token_great]

# + id="nP9ZbRSYeGsH" colab_type="code" colab={}
token_bad = tokenizer.word_index['bad']
token_horrible = tokenizer.word_index['horrible']

# + id="Qgb7wYkBePOD" colab_type="code" outputId="8d7b2845-72d4-4d9f-a15c-4273e81542b6" colab={"base_uri": "https://localhost:8080/", "height": 52}
weights_embedding[token_bad]

# + id="JYqZPThUeQon" colab_type="code" outputId="f7a78b67-866a-4d75-cd72-0a854e5c405b" colab={"base_uri": "https://localhost:8080/", "height": 52}
weights_embedding[token_horrible]


# + id="GKryzmKieYZU" colab_type="code" colab={}
def print_sorted_words(word, metric='cosine'):
    """
    Print the words in the vocabulary sorted according to their
    embedding-distance to the given word.
    Different metrics can be used, e.g. 'cosine' or 'euclidean'.
    """

    # Get the token (i.e. integer ID) for the given word.
    token = tokenizer.word_index[word]

    # Get the embedding for the given word. Note that the
    # embedding-weight-matrix is indexed by the word-tokens
    # which are integer IDs.
    embedding = weights_embedding[token]

    # Calculate the distance between the embeddings for
    # this word and all other words in the vocabulary.
    distances = cdist(weights_embedding, [embedding],
                      metric=metric).T[0]
    
    # Get an index sorted according to the embedding-distances.
    # These are the tokens (integer IDs) for words in the vocabulary.
    sorted_index = np.argsort(distances)
    
    # Sort the embedding-distances.
    sorted_distances = distances[sorted_index]
    
    # Sort all the words in the vocabulary according to their
    # embedding-distance. This is a bit excessive because we
    # will only print the top and bottom words.
    sorted_words = [inverse_map[token] for token in sorted_index
                    if token != 0]

    # Helper-function for printing words and embedding-distances.
    def _print_words(words, distances):
        for word, distance in zip(words, distances):
            print("{0:.3f} - {1}".format(distance, word))

    # Number of words to print from the top and bottom of the list.
    k = 10

    print("Distance from '{0}':".format(word))

    # Print the words with smallest embedding-distance.
    _print_words(sorted_words[0:k], sorted_distances[0:k])

    print("...")

    # Print the words with highest embedding-distance.
    _print_words(sorted_words[-k:], sorted_distances[-k:])


# + id="zIDLoVTweame" colab_type="code" outputId="8f1381bd-fd1f-4539-f1f2-52bedc155077" colab={"base_uri": "https://localhost:8080/", "height": 166}
print_sorted_words('great', metric='cosine')

# + id="EQcmFPpPeiAP" colab_type="code" outputId="5203f0d3-05db-463b-b6ef-4b51bd7634ef" colab={"base_uri": "https://localhost:8080/", "height": 399}
print_sorted_words('worst', metric='cosine')

StarcoderdataJupyterScriptsDedupFiltered