# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# Twitter Sentiment Analysis Data PreProcessing (Test data)

# import the dependencies
import keras
from keras import backend as k
import numpy as np
import pandas as pd
import re
import nltk
from nltk import PorterStemmer
from nltk.tokenize import word_tokenize
import time

# reading the data
train_data = pd.read_csv('data/test.csv', encoding = 'ISO-8859-1')

# showing the structure of the data
train_data.head(10)

# no of rows in the dataset
rows = train_data.count()
print('no of rows in the data set : ', rows[0])

# +
# first of all we will remove @ from the text like this

text = "@aditya you are a very good boy"
print('remove @ from the tweet')
print('original text : ', text)
print('processed text : ',re.sub(r'@[A-Za-z0-9]+','',text))

print('\n')
# removing links from the text
text= "http://aditya.com is a very good site"
print('remove links from the tweets')
print('original text: ', text)
print('precessed text: ',re.sub('https?://[A-Za-z0-9./]+','',text))

print('\n')
# removing hash tags and numbers from the text
text = "we won #INvsAUS #cricket #bleedblue"
print('remove hastags from the text')
print('original text: ',text)
print('processed text: ',re.sub("[^a-zA-Z]", " ", text))

print('\n')
# remove emoji from the text
print('to remove emoji from the text')
text = " hello boy 🙅 "
emoji_pattern = re.compile("["
        u"\U0001F600-\U0001F64F"  # emoticons
        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
        u"\U0001F680-\U0001F6FF"  # transport & map symbols
        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           "]+", flags=re.UNICODE)

print('original text: ',text)
print('processed text : ',emoji_pattern.sub(r'', text)) # no emoji

print('\n')
# to convert the text to lower case
print('to convert uppercase letters to lower case letters')
text = "This Is "
print('original text: ',text)
print('processed text : ', text.lower())

print('\n')
# remove punctuations from the text
print('to remove punctuations from the text')
text = "string. With. Punctuation?"
print('original text: ',text)
print('processed text: ',re.sub(r'[^\w\s]','',text))

print('\n')
# to remove repeating characters 
text = "I’m in a hurrryyyyyyyyyyyyyyyyy"
print('to remove repeating characters from the text')
print('original text: ', text)
print('processed text: ',re.sub(r'(.)\1+', r'\1\1', text))

print('\n')
# to deal with stemming words
text = "running"
print('to deal with stemming words')
print('original text: ', text)
print('processed text: ',PorterStemmer().stem(text))

# +

text1 = "worry"
tokens = word_tokenize(text1)
print(tokens)
# -

SentimentText = []
def preprocess(train_data):
    i = 0
    while i <rows[0]:
        text =train_data['SentimentText'][i]
        text =text.lower()
        text = re.sub(r'@[A-Za-z0-9]+','',text)
        text = re.sub("[^a-zA-Z]", " ", text)
        text =emoji_pattern.sub(r'', text)
        text = re.sub(r'[^\w\s]','',text)
        text = re.sub(r'(.)\1+', r'\1\1', text)
        tokens = word_tokenize(text)
        new_text =""
        for token in tokens:
            new_text += PorterStemmer().stem(token)
            new_text +=' ' 
        SentimentText.append(new_text)
        
        if i%10000 == 0:
            print('processed :' , i , 'tweets')
        i = i + 1
    print('Done PreProcessing the text' )


# +
curr_time = time.time()
preprocess(train_data)
new_time = time.time()

time_taken = (new_time - curr_time)/60
print('time taken : ',time_taken, ' mins')
# -

import pickle
with open("SentimentText_test.txt", "wb") as fp:
    pickle.dump(SentimentText, fp)



StarcoderdataJupyterScriptsDedupFiltered