"""
<blockquote class="twitter-tweet"><p lang="en" dir="ltr">NLP with 1.6m tweets, positive and negative tweets, can we create some ML that can predict if a tweet is either? 280 chars with links, mentions, and ugly spellling. <a href="https://twitter.com/hashtag/letsgo?src=hash&amp;ref_src=twsrc%5Etfw">#letsgo</a> <a href="https://twitter.com/hashtag/datascience?src=hash&amp;ref_src=twsrc%5Etfw">#datascience</a> <a href="https://twitter.com/hashtag/ML?src=hash&amp;ref_src=twsrc%5Etfw">#ML</a> <a href="https://twitter.com/hashtag/needmoreGPUs?src=hash&amp;ref_src=twsrc%5Etfw">#needmoreGPUs</a> <a href="https://twitter.com/hashtag/bigdata?src=hash&amp;ref_src=twsrc%5Etfw">#bigdata</a></p>&mdash; Jonathan Henson (@sciencetition) <a href="https://twitter.com/sciencetition/status/1305008902212399108?ref_src=twsrc%5Etfw">September 13, 2020</a></blockquote> <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>
"""
"""
**Lazy or Smart?** 
This was an exercise in creating a lighter, stripped down ML, asking the question why DS projects often follow the same script. I took out what is usually considered important steps, objects, ideals, and tested to see if there was a need.

The most important take away: **Stopwords** need to be taken into account with respect to the target data. It might make sense for articles, books, paragraphs. But for tweets? Turns out perhaps not.


"""

import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

from sklearn.metrics import confusion_matrix, classification_report, accuracy_score
import itertools

from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.text import Tokenizer

from wordcloud import WordCloud

import tensorflow as tf
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.layers import Conv1D, Bidirectional, LSTM, Dense, Input, Dropout, SpatialDropout1D

import nltk
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
import re

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))
        

"""
**Load and Drop** 
Here we load our tweets into our dataframe. Not even going to give the columns proper names before we drop them. In the end all we want is sentiment and text. Print a peek to make sure we are good to go.
"""
df = pd.read_csv('../input/sentiment140/training.1600000.processed.noemoticon.csv',encoding = 'latin',header=None)
df.columns = ['sentiment', 'x1', 'x2', 'x3', 'x4', 'text']
df = df.drop(['x1', 'x2', 'x3', 'x4'], axis=1)

print(df)
"""
**Workable data**
4's and 0's are not very ML friendly. Since the data does not include neutral sentiment, or anything in between lets just label them Negative and Positive.
"""
#function to change labels to Positive or Negative
Convert_Sentiment = {0: "Negative", 4: "Positive"}
def Sentiment_change(lcol):
  return Convert_Sentiment[lcol]
df.sentiment = df.sentiment.apply(lambda x: Sentiment_change(x))
"""
**Distribution** 
In the real world we are given data that is usually highly skewed. We were given a gift here, 50/50 and no need to change samples.
"""
#lets look at our distribution
val_count = df.sentiment.value_counts()
print(val_count)
"""
**Cleaning up the data** 
Typically you would stem or lemmatize, remove links, mentions, and stop words. Call it a day and your data is clean. However with tweets, our data is different. We dont have full sentences, or even coherent thoughts sometimes. 

Stop words: A list of words commonly used which have little meaning. Well, it does have meaning, and if the entire context of a Positive or Negative tweet is derived from 280 chars, we are going to leave the stop words in. 

Twitter specific spam removed: HTTPS, HTTP, @mentions, AMP(mobile page text), and quot (meaningless)

![](https://www.kdnuggets.com/images/cartoon-machine-learning-class.jpg)
"""
## Clean data. Typically stop words would be used. This study includes all words, excluding stop words in a 280 char tweet is too restrictive.
## Remove words with no meaning, mentions and urls. 
cleaning = "amp\S+|quot\S+|@\S+|https?:\S+|http?:\S|[^A-Za-z0-9]"
def preprocess(text):
  text = re.sub(cleaning, ' ', str(text).lower()).strip()
  tokens = []
  for token in text.split():
    tokens.append(token)
  return " ".join(tokens)
df.text = df.text.apply(lambda x: preprocess(x))
"""
**Get a visual, make sure they look different**
"""
plt.figure(figsize=(20, 20))
cloud = WordCloud(max_words=1000, width=1000, height=800)
cloud.generate(" ".join(df[df.sentiment == 'Positive'].text))
plt.imshow(cloud, interpolation='bilinear')
plt.show()
"""
# Now - stop word
Do you see the "now". thats a big one for Negative tweets and not for Positive ones, and it just happens to be a stop word. 
"""
plt.figure(figsize=(20, 20))
cloud = WordCloud(max_words=1000, width=1000, height=800)
cloud.generate(" ".join(df[df.sentiment == 'Negative'].text))
plt.imshow(cloud, interpolation='bilinear')
plt.show()
"""
**80/20 as is the tradition**
"""
#setup for training and testing splits. 
Training_percent = 0.8
MAX_SEQUENCE_LENGTH = 30
train_data, test_data = train_test_split(df, test_size=1-Training_percent, random_state=7) 
"""
**TOKENS**

Each tweet is going to contain multiple words, here we basically split the tweet into its individual words so we can further process.
"""
tokens = Tokenizer()
tokens.fit_on_texts(train_data.text)
word_index = tokens.word_index
"""
Manipulating our train and tests, filled with tokens and padded to ensure identical shapes
"""
#pesky +1 to account for 0 starting.
vocab_size = len(tokens.word_index) + 1
print("# of words :", vocab_size)
x_train = pad_sequences(tokens.texts_to_sequences(train_data.text),maxlen=MAX_SEQUENCE_LENGTH)
x_test = pad_sequences(tokens.texts_to_sequences(test_data.text), maxlen=MAX_SEQUENCE_LENGTH)

labels = train_data.sentiment.unique().tolist()

encoder = LabelEncoder()
encoder.fit(train_data.sentiment.to_list())
"""
Creating our Y-Train, reshaping, and taking a peek to make sure we are on the right track. 
"""
#Create y-train
y_train = encoder.transform(train_data.sentiment.to_list())
y_test = encoder.transform(test_data.sentiment.to_list())
y_train = y_train.reshape(-1, 1)
y_test = y_test.reshape(-1, 1)

print("Y Train shape: ",y_train.shape)
print("Y Test shape: ",y_test.shape)
"""
# What is the right number?

We are at the point where I feel like pathways really open up, we will be using GLOVE, but there are other options. We are using ADAM but there are other options. What is going to be your Learning Rate, how many Epochs are going to be run, will something as simple as batch size change our model drastically?


![](https://i.imgflip.com/1j9mml.jpg)
"""
GLOVE_EMB = '../input/glove6b300dtxt/glove.6B.300d.txt'
EMBEDDING_DIM = 300
LR = .005
BATCH_SIZE = 2048
EPOCHS = 15
#start embedding, open glove and resolve total vectors
embed_index = {}
f = open(GLOVE_EMB, encoding="utf-8")
for line in f:
  values = line.split()
  word = values[0]
  vectors = np.asarray(values[1:], dtype='float32')
  embed_index[word] = vectors
f.close()
print('Word vectors: ',len(embed_index))
#create the matrix and fill with 0s, size of total vocab words and the embeded dimentions from the Glove file used. 
emb_matrix = np.zeros((vocab_size, EMBEDDING_DIM))

#get each word, get corresponding glove value, write if value exists to the embedding matrix
for word, i in word_index.items():
  embedding_vector = embed_index.get(word)
  if embedding_vector is not None:
    emb_matrix[i] = embedding_vector
"""
# Trainable is FALSE.

When trying different combinations of settings, trainable = true led to an extremely over fit model, which also took 5x as long to run. Don't do it. 

# No callbacks.
This was tested, and including callbacks did not result in a better model. I am choosing to exclude callbacks and keep the model cleaner.
"""
#create the embedding layer with given dimensions, vectors. Trainable false is used, if using trainable = true the model will over fit significantly
embedding_layer = tf.keras.layers.Embedding(vocab_size,EMBEDDING_DIM,weights=[emb_matrix],input_length=MAX_SEQUENCE_LENGTH,trainable=False)
"""
# Is anyone an expert in this area?
I am using the LSTM model, with pretty standard settings. Most models seem to have additional layers, but I found them not to be helpful in producing an accurate model.
"""
#sequence creation. 
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedding_sequences = embedding_layer(sequence_input)
x = SpatialDropout1D(0.2)(embedding_sequences)
x = Conv1D(64, 5, activation='relu')(x)
x = Bidirectional(LSTM(64, dropout=0.1, recurrent_dropout=0.1))(x)
x = Dense(512, activation='relu')(x)
outputs = Dense(1, activation='sigmoid')(x)

model = tf.keras.Model(sequence_input, outputs)
model.compile(optimizer=Adam(learning_rate=LR), loss='binary_crossentropy',metrics=['accuracy'])
print(model.summary())
"""
# GPU and Coffee time
![](https://imgs.xkcd.com/comics/compiling.png)
"""
#Here we go
history = model.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=EPOCHS, validation_data=(x_test, y_test))
"""
I picked too many Epochs, 10 would have been enough. The val_accuracy fluctuated a bit after 10 epochs but wasn't sustained improvement. 
"""
s, (at, al) = plt.subplots(2, 1)
at.plot(history.history['accuracy'], c='b')
at.plot(history.history['val_accuracy'], c='r')
at.set_title('model accuracy')
at.set_ylabel('accuracy')
at.set_xlabel('epoch')
at.legend(['LSTM_train', 'LSTM_val'], loc='upper left')

al.plot(history.history['loss'], c='m')
al.plot(history.history['val_loss'], c='c')
al.set_title('model loss')
al.set_ylabel('loss')
al.set_xlabel('epoch')
al.legend(['train', 'val'], loc='upper left')
plt.show()
def decode_sentiment(score):
    return "Positive" if score > 0.5 else "Negative"


scores = model.predict(x_test, verbose=1, batch_size=10000)
y_pred_1d = [decode_sentiment(score) for score in scores]
"""
**Confusion Matrix**

Here we can see where the model got things right and wrong. Very close to even on Positive and Negative with a slightly better job at spotting Positive tweets.

# over all about an 83%

When computed with stopwords, all else being equal, the model showed 79%. 
"""
def plot_confusion_matrix(cm, classes,title='Confusion matrix',cmap=plt.cm.Blues):
    cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title, fontsize=20)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, fontsize=13)
    plt.yticks(tick_marks, classes, fontsize=13)

    fmt = '.2f'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.ylabel('True label', fontsize=17)
    plt.xlabel('Predicted label', fontsize=17)
    

cnf_matrix = confusion_matrix(test_data.sentiment.to_list(), y_pred_1d)
plt.figure(figsize=(6, 6))
plot_confusion_matrix(cnf_matrix, classes=test_data.sentiment.unique(), title="Confusion matrix")
plt.show()

print(classification_report(list(test_data.sentiment), y_pred_1d))

model.save_weights("model.h5")