# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ## Disaster Tweets
# * A **Natural  Language Processing** task that detemines is talking about natural disaster or not using the [Kaggle](https://www.kaggle.com/c/nlp-getting-started) dataset

# ### 1. Data Cleaning

# #### Imports

import matplotlib.pyplot as plt
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
import seaborn as sns
from collections import Counter
import re
import pandas as pd

# > **Loading the data:**

disaster = pd.read_csv('train.csv')
disaster.head()

# > **We are interested in the** text and target.

tweets = disaster["text"].values
targets =  disaster["target"].values

print(tweets[:1])
targets[:50]

# > **Data Cleaning** - We want to remove:
#     
#     
# * urls
# * non word chars
# * stopwords - A stop word is a commonly used word (such as “the”, “a”, “an”, “in”) that a search engine has been programmed to ignore, both when indexing entries for searching and when retrieving them as the result of a search query.

test = "<NAME> '@Alexis_Sanchez: happy to see my teammates and training hard ?? goodnight gunners.?????? http://t.co/uc4j4jHvGR'"

stopwords = stopwords.words('english')

a = re.sub(r'https?://(\S+|www)\.\S+', ' ',test) # reomove urls
b = re.sub(r'\W|_', ' ', a) # remove punctuactions and numbers and underscores
c = re.sub(r'\s+', ' ', b).strip() # remove double spacing
c.lower()

# > **Now lets do it for all the tweets**

tweets_cleaned1 = []
for tweet in tweets:
    a = re.sub(r'https?://(\S+|www)\.\S+', ' ',tweet)
    b = re.sub(r'\W|_', ' ', a)
    c = re.sub(r'\d', " ", b)
    d = re.sub(r'[^a-zA-Z]', " ", c)
    e = re.sub(r'\s+', ' ', d).strip()
    tweets_cleaned1.append(e.lower())

tweets_cleaned1

tweets_cleaned2 = []
for sent in tweets_cleaned1:
    words = word_tokenize(sent)
    tweets_cleaned2.append(" ".join([word for word in words if word not in stopwords]))

tweets_cleaned2[:10]

# ### Balancing the tweets

conter = Counter(targets)
conter

# +

tweets_balanced = []
targets_balanced = []

c0, c1 = 0, 0

for i in range(len(targets)):
    if targets[i] == 1 and c1 <= conter[1]:
        targets_balanced.append(targets[i])
        tweets_balanced.append(tweets_cleaned2[i])
        c1 += 1
    elif targets[i] == 0 and c0 < conter[1]:
        targets_balanced.append(targets[i])
        tweets_balanced.append(tweets_cleaned2[i])
        c0 += 1
    else:
        continue
    
# -

Counter(targets_balanced), tweets_balanced[:2]

# ### Visualisation

sns.countplot(x=targets_balanced); plt.show()

# ### Counting unique words

# +
count = Counter()

for sent in tweets_balanced:
    words = word_tokenize(sent)
    for word in words:
        count[word] +=1
# -

count

count.most_common(3)

unique_words = len(count)
unique_words

# #### Spliting the Data

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(tweets_balanced, targets_balanced, test_size =.2, random_state=43)

# #### Create vectors of words

from tensorflow.keras.preprocessing.text import Tokenizer

tokenizer = Tokenizer(num_words=unique_words)
tokenizer.fit_on_texts(X_train)

word_index = tokenizer.word_index

word_index

X_train_tokens = tokenizer.texts_to_sequences(X_train)
X_test_tokens = tokenizer.texts_to_sequences(X_test)

X_train_tokens[:2]

# > **We need to apply the  ``pad_sequence`` to each sentence to balance the lenght of each sentence**. Thi is because our NN accepts the data with the same shape.

from tensorflow.keras.preprocessing.sequence import pad_sequences

max_words = 20 # maximum number of unique words
X_train_tokens_padded = pad_sequences(X_train_tokens, maxlen=max_words, padding="post", truncating="post")
X_test_tokens_padded = pad_sequences(X_test_tokens, maxlen=max_words, padding="post", truncating="post")

X_train_tokens_padded[1], X_train_tokens[1],  X_train[1]

# ####  Fliping the words and it's index

flipped_word_index = dict([(i, word) for (word, i) in word_index.items()])
flipped_word_index

X_train_tokens[1] ## reverse this to a sentence

" ".join([flipped_word_index[i] for i in X_train_tokens[1]])

# ### Creating a ``RNN`` - Tweets Classification

# #### imports

from tensorflow import keras
import numpy as np

model = keras.Sequential([
    keras.layers.Embedding(unique_words, 32, input_length=max_words),
    keras.layers.LSTM(64, dropout=.2),
    keras.layers.Dense(64, activation='relu'),
    keras.layers.Dense(16, activation='relu'),
    keras.layers.Dense(1, activation='sigmoid')
])



model.compile(
    loss = keras.losses.BinaryCrossentropy(from_logits=False),
    metrics= ['accuracy'],
    optimizer = keras.optimizers.Adam(lr=1e-3)
)

# +
EPOCHS = 5
VERBOSE = 1
VALIDATION_DATA = (X_test_tokens_padded,np.array(y_test, dtype="float32"))
BATCH_SIZE = 128

history = model.fit(X_train_tokens_padded, np.array(y_train, dtype="float32"), batch_size=BATCH_SIZE, epochs=EPOCHS, verbose = VERBOSE, validation_data = VALIDATION_DATA)
# -

print(np.round(model.predict(X_test_tokens_padded[:5]))), y_test[:5], X_test[:5], X_test_tokens_padded[5]

# +
cate = ["Not Disatser", "Disater"]

def makePrediction(sent):
    tokens = tokenizer.texts_to_sequences([sent])
    padded = np.array(pad_sequences(tokens, maxlen=20, padding="post", truncating="post"))
    print(cate[np.round(model.predict(padded))[0][0].astype('int32')])
    
makePrediction('we are in floods')
# -

# ### Saving the Model.

model.save('models/disaster_tweets.h5')
print('Saved')

clf = keras.models.load_model('models/disaster_tweets.h5')
clf.predict(X_test_tokens_padded[:5])

# ### Labling Tweets in the test.csv

disaster = pd.read_csv('test.csv')
disaster.head()

tweets = disaster["text"].values

tweets_cleaned1 = []
for tweet in tweets:
    a = re.sub(r'https?://(\S+|www)\.\S+', ' ',tweet)
    b = re.sub(r'\W|_', ' ', a)
    c = re.sub(r'\d', " ", b)
    d = re.sub(r'[^a-zA-Z]', " ", c)
    e = re.sub(r'\s+', ' ', d).strip()
    tweets_cleaned1.append(e.lower())

tweets_cleaned2 = []
for sent in tweets_cleaned1:
    words = word_tokenize(sent)
    tweets_cleaned2.append(" ".join([word for word in words if word not in stopwords]))

tweets_cleaned2[:5]

tokens = tokenizer.texts_to_sequences(tweets_cleaned2)

tokens_padded = pad_sequences(tokens, maxlen=max_words, padding="post", truncating="post")

tokens_padded[0]

# ### Making Predictions

predictions = np.round(model.predict(tokens_padded)).astype('int32')
predictions

predictions_targets = [pred[0] for pred in predictions]
predictions_targets[:10]

# ### Creating a new pandas csv with labeled tweets



columns_names = ["text", "target"]
rows = np.column_stack([tweets_cleaned2,predictions_targets ])

dataframe = pd.DataFrame(rows, columns=columns_names)
dataframe.to_csv('tweets_labeled.csv')
print("Saved")



tweets = pd.read_csv('tweets_labeled.csv')
tweets.head()



StarcoderdataJupyterScriptsDedupFiltered