# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda env:tensorflow]
#     language: python
#     name: conda-env-tensorflow-py
# ---

# +
import builtins

def print(*args, **kwargs):
    return builtins.print(*args, **kwargs, end='\n\n')


# -

from tensorflow.keras.preprocessing.text import Tokenizer

sentences = [
    'I love my Dog',
    'I love my Cat',
    'You love my Dog!!'
]

tokenizer = Tokenizer(num_words=100)
tokenizer.fit_on_texts(sentences)
word_index = tokenizer.word_index
print(word_index)

# ----------------

# +
sentences = [
    'I love my Dog',
    'I love my Cat',
    'You love my Dog!!',
    'Do you think my dog is amazing?'
]

tokenizer = Tokenizer(num_words=100)
tokenizer.fit_on_texts(sentences)
word_index = tokenizer.word_index

# sentence to list of tokens/numbers
sequences = tokenizer.texts_to_sequences(sentences)

print(word_index)
print(sequences)

# +
test_data = [
    'i really love my dog',
    'my dog loves my manatee'
]

test_seq = tokenizer.texts_to_sequences(test_data)
print(test_seq)
# -

# ______________

# +
sentences = [
    'I love my Dog',
    'I love my Cat',
    'You love my Dog!!',
    'Do you think my dog is amazing?'
]

# special value for unseen word
tokenizer = Tokenizer(num_words=100, oov_token="<OOV>")
tokenizer.fit_on_texts(sentences)
word_index = tokenizer.word_index

# sentence to list of tokens/numbers
sequences = tokenizer.texts_to_sequences(sentences)

print(word_index)
print(sequences)

test_data = [
    'i really love my dog',
    'my dog loves my manatee'
]

test_seq = tokenizer.texts_to_sequences(test_data)
print(test_seq)

# +
# padding to make words uniform
# -

from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.preprocessing.text import Tokenizer

# +
sentences = [
    'I love my Dog',
    'I love my Cat',
    'You love my Dog!!',
    'Do you think my dog is amazing?'
]

# special value for unseen word
tokenizer = Tokenizer(num_words=100, oov_token="<OOV>")
tokenizer.fit_on_texts(sentences)
word_index = tokenizer.word_index

# sentence to list of tokens/numbers
sequences = tokenizer.texts_to_sequences(sentences)

# padding 
paddded = pad_sequences(sequences)
print(paddded)

# padding after the sentence
paddded = pad_sequences(sequences, padding='post')
print(paddded)

# matrix len is the same as the sentence with maximum length
# to change it
paddded = pad_sequences(sequences, padding='post', maxlen=5)
print(paddded)

# loose info from end instead of front
paddded = pad_sequences(sequences, padding='post',
                       truncating='post', maxlen=5)
print(paddded)

# print(word_index)
# print(sequences)
# print(paddded)
# -

# _____________

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences

sentences = [
    'I love my Dog',
    'I love my Cat',
    'You love my Dog!!',
    'Do you think my dog is amazing?'
]

# +
sentences = [
    'I love my Dog',
    'I love my Cat',
    'You love my Dog!!',
    'Do you think my dog is amazing?'
]


sequences = tokenizer.texts_to_sequences(sentences)

padded = pad_sequences(sequences, maxlen=5)

tokenizer = Tokenizer(num_words=100, oov_token="<OOV>")
tokenizer.fit_on_texts(sentences)
word_index = tokenizer.word_index

sequences = tokenizer.texts_to_sequences(sentences)

padded = pad_sequences(sequences, maxlen=5)

print(f"Word Index: {word_index}")
print(f"Sequences: {sequences}")
print(f"Padded Sequences: \n{padded}")

# +
# testing

test_data = [
    'i really love my dog',
    'my dog loves my manatee'
]

test_seq = tokenizer.texts_to_sequences(test_data)
print(f"Test Sequences: {test_seq}")

padded = pad_sequences(test_seq, maxlen=10)
print(f"Padded test sequence: \n {padded}")
# -

stopwords = ["a", "about", "above", "after", "again", "against", "all", "am", "an", "and", "any", "are", "as", "at", 
             "be", "because", "been", "before", "being", "below", "between", "both", "but", "by", "could", "did", 
             "do", "does", "doing", "down", "during", "each", "few", "for", "from", "further", "had", "has", 
             "have", "having", "he", "he'd", "he'll", "he's", "her", "here", "here's", "hers", "herself", "him", 
             "himself", "his", "how", "how's", "i", "i'd", "i'll", "i'm", "i've", "if", "in", "into", "is", "it", 
             "it's", "its", "itself", "let's", "me", "more", "most", "my", "myself", "nor", "of", "on", "once", 
             "only", "or", "other", "ought", "our", "ours", "ourselves", "out", "over", "own", "same", "she", "she'd", 
             "she'll", "she's", "should", "so", "some", "such", "than", "that", "that's", "the", "their", "theirs", "them", 
             "themselves", "then", "there", "there's", "these", "they", "they'd", "they'll", "they're", "they've", "this", 
             "those", "through", "to", "too", "under", "until", "up", "very", "was", "we", "we'd", "we'll", "we're", "we've",
             "were", "what", "what's", "when", "when's", "where", "where's", "which", "while", "who", "who's", "whom", "why", 
             "why's", "with", "would", "you", "you'd", "you'll", "you're", "you've", "your", "yours", "yourself", "yourselves"]

StarcoderdataJupyterScriptsDedupFiltered