"""
## Importing Libraries
"""
!pip install tensor-sensor
import tsensor
import numpy as np

from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Embedding, GlobalAveragePooling1D

import warnings
warnings.filterwarnings("ignore")
"""
## Tokenization
"""
# Tokenising sentences
sentences = [
    'The quick brown fox jumps over the lazy dog.'
]

tokenizer = Tokenizer()
tokenizer.fit_on_texts(sentences)
print("Word Index       :", tokenizer.word_index)
train_sequence = tokenizer.texts_to_sequences(sentences)
train_sequence = np.array(train_sequence)
print("Sentence Sequence :", train_sequence)
"""
## Creating Embedding Layer
"""
# Create a random embedding layer
embedding = Embedding(input_dim=len(train_sequence[0]), output_dim=128)
# Get the embeddings of the train sample
train_sample = embedding(train_sequence)
print("Shape of Input          :", train_sequence.shape)
print("Shape of Embedded Input :", train_sample.shape)
with tsensor.explain(fontname='Hack', dimfontname='Hack'):
    train_sample = embedding(train_sequence)
train_sample[0]
"""
## Average Across Tokens
"""
GlobalAveragePooling1D()(train_sample)
with tsensor.explain(fontname='Hack', dimfontname='Hack'):
    z = GlobalAveragePooling1D()(train_sample)
"""
## Create Word Embeddings for More than One Sentence
"""
# More than one sentence

test_corpus = [
    'The quick brown fox jumps over the lazy dog.',
    'The quick brown fox.',
    'The lazy dog',
    'The dog',
    'Dog and the fox',
    'Hello, world!'
]

encoded_sentences = tokenizer.texts_to_sequences(test_corpus)
i = 1
for sentence, encoded_sentence in zip(test_corpus, encoded_sentences):
    print("Sentence",str(i) + " :", sentence)
    print("Sequence   :", encoded_sentence)
    print("---------------------------------------------------------")
    i+=1
"""
## Padding Sequences
"""
# Length of each sentence in the corpus
print("Length :", [len(sentence) for sentence in encoded_sentences])
# Length of the longest sentence
print("Max Length :", max([len(sentence) for sentence in encoded_sentences]))
MAX_SEQUENCE_LENGTH = 9
# Padding sequence that are shorter than the longest sequence
X = pad_sequences(encoded_sentences, maxlen=MAX_SEQUENCE_LENGTH)
X