import numpy as np

"""
Explanation: Chapter 6.1.1 - One-hot encoding of words and characters
End of explanation
"""


samples = ['The cat sat on the mat.', 'The dog ate my homework.']

# Intializing token index as an empty dictionary
token_index = {}

"""
Explanation: Initializing an example
End of explanation
"""


# Testing result of function .split()
samples[0].split()

for sample in samples:
    # Each sample is split into words. 
    # Currently, punctuation is not ommited
    for word in sample.split():
        if word not in token_index:
            # Assign a unique index to each unique word
            # Index 0 is not assigned to anything.
            token_index[word] = len(token_index) + 1

token_index

"""
Explanation: Splitting the sentence (word level)
End of explanation
"""


# Taking into consideration only first 10 words in each sentence.
max_length = 10

# Initializing the result array with zeros
# It will be of shape (number_of_samples, max_length_taken_into_consideration, number_of_unique_words)
results = np.zeros((len(samples), max_length, max(token_index.values()) + 1))

# Enumerating through samples and words
# One-hot encoding
for i, sample in enumerate(samples):
    for j, word in list(enumerate(sample.split()))[:max_length]:
        index = token_index.get(word)
        results[i, j, index] = 1.

results

"""
Explanation: Vectoring the example
End of explanation
"""


import string

samples = ['The cat sat on the mat.', 'The dog ate my homework.']

# Assigning all prinatable ASCII characters
characters = string.printable  

characters

# Tokenizing the characters
token_index = dict(zip(characters, range(1, len(characters) + 1)))

token_index

# Take intro consideration only first 50 character of the sentence
max_length = 50

results = np.zeros((len(samples), max_length, max(token_index.values()) + 1))

for i, sample in enumerate(samples):
    for j, character in enumerate(sample[:max_length]):
        index = token_index.get(character)
        results[i, j, index] = 1.

results

"""
Explanation: Splitting the sentence (character level)
End of explanation
"""


# Importing Keras Tokenizer
from keras.preprocessing.text import Tokenizer

samples = ['The cat sat on the mat.', 'The dog ate my homework.']

# Intializing tokenizer, which will take into account only 1000 most commonly used words.
tokenizer = Tokenizer(num_words = 1000)

# Building the dictionary
tokenizer.fit_on_texts(samples)

# Turning the sequences to an array of integers corresponding to the unique words
sequences = tokenizer.texts_to_sequences(samples)

sequences

# Representing the data as one-hot encoded.
one_hot_results = tokenizer.texts_to_matrix(samples, 
                                            mode = 'binary')

one_hot_results

# Retrieving the index
word_index = tokenizer.word_index

word_index

"""
Explanation: Keras Tokenizer
End of explanation
"""


samples = ['The cat sat on the mat.', 'The dog ate my homework.']

# Storing words as vector of size 1000.
# Possible hash collisions and the accuracy of the encoding will drop.

dimensionality = 1000
max_length = 10

results = np.zeros((len(samples), max_length, dimensionality))

for i, sample in enumerate(samples):
    for j, word in list(enumerate(sample.split()))[:max_length]:
        # Hash the word into a "random" integer index
        # that is between 0 and 1000
        index = abs(hash(word)) % dimensionality
        results[i, j, index] = 1.

results

# 2 examples, each of size 10 (or less, but encoding will persist) with one-hot hashed words
results.shape

"""
Explanation: One-hot hashing
"A variant of one-hot encoding is the so-called "one-hot hashing trick", which can be used when the number of unique tokens in your vocabulary is too large to handle explicitly. Instead of explicitly assigning an index to each word and keeping a reference of these indices in a dictionary, one may hash words into vectors of fixed size. This is typically done with a very lightweight hashing function. The main advantage of this method is that it does away with maintaining an explicit word index, which saves memory and allows online encoding of the data (starting to generate token vectors right away, before having seen all of the available data). The one drawback of this method is that it is susceptible to "hash collisions": two different words may end up with the same hash, and subsequently any machine learning model looking at these hashes won't be able to tell the difference between these words. The likelihood of hash collisions decreases when the dimensionality of the hashing space is much larger than the total number of unique tokens being hashed."
End of explanation
"""


