# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: metis
#     language: python
#     name: metis
# ---

# ##### Import necessary ibraries

from utils import pickle_to, pickle_from, ignore_warnings
import pandas as pd
import pickle
import seaborn as sns
import re
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
pd.set_option('display.max_colwidth', -1)


# # Scrub data
# ###### Encoding newly created variable names into 0 or 1
#
# Encoding | Label
# -- | --
# 0 | FAKE
# 1 | REAL
#
#

# + code_folding=[]
def encode_label(label):
    """Encode real as 1, fake as 0, everything else as None"""
    if label == "REAL":
        return 1
    elif label == "FAKE":
        return 0
    else:
        return None


# +
# Load raw data
raw = pickle_from('../data/raw/raw.pkl')



# +
# Apply label encoding function to raw data

raw['numeric_label'] = raw['label'].apply(encode_label)
raw.head()
# -

raw[raw.text.str.contains('turtle')]

# +
# Removing rows that have the string 'sweeping consequences'
# These have been observed to be duplicates of a single line

raw = raw[~raw['text'].str.contains('sweeping consequence')]




# +
# Removing rows that have the string 'automatically' 
# as this is part of html code

raw = raw[~raw['text'].str.contains('automatically')]
raw = raw.drop(220).reset_index()
raw = raw.drop('index',axis = 1)




# -

# Dropping null values
data = raw[['title','text','numeric_label']].dropna()


data.info()


# ##### Combining text and title data into a single column by name 'news'

data['news'] = data['title'] + '. '  + data['text']

data.head()

# ## Cleaning text data
#
# Involves the following <br>
#
# * converting everyting to lowercase
# * removing punctuations
# * removing numbers
# * removing non english words
#

# +
# Converting to lowercase
data.text = data.text.apply(lambda x:x.lower())
data['text'] = data['text'].str.replace("’","'")


#Removing all punctuations
data.text = data.text.str.replace('[^\w\s]','')

# Removing numbers
data.text = data.text.str.replace('\d+', ' ')

# Making sure any double-spaces are single
data.text = data.text.str.replace('  ',' ')


# +
data.sample(5)



# +
text_data = data[['numeric_label','text']].copy()


# -

text_data.info()

text_data.head(1)

# ### Tokenize & Lemmatize

# +
w_tokenizer = nltk.tokenize.WhitespaceTokenizer()
lemmatizer = nltk.stem.WordNetLemmatizer()

def lemmatize_text(text):
    return [lemmatizer.lemmatize(w) for w in w_tokenizer.tokenize(text)]


# -

text_data['tokenized'] = text_data.text.apply(lemmatize_text)


text_data.head(1)


# ### Remove english stopwords

stop_words = stopwords.words('english')


# +
stop_words.append('tweet')
stop_words.append('home')
stop_words.append('headlines')
stop_words.append('finance')
stop_words.append('news')
stop_words.append('was')
stop_words.append('has')
stop_words.append('said')
stop_words.append('wa')
stop_words.append('ha')
stop_words.append('leave')
stop_words.append('comment')
stop_words.append('loading')
stop_words.append('eaten')
stop_words.append('matrix')
stop_words.append('extraterrestrial')
stop_words.append('wi')
stop_words.append('ivt')


# -

pickle_to(stop_words,'stop_words.pkl')

# +
text_data['tokenized'] = text_data['tokenized'].apply(lambda x: [item for item in x if item not in stop_words])

# Remove words that have less than 3 characters
text_data['text'] = text_data['text'].str.replace(r'\b(\w{1,2})\b', ' ')

# +
# Create a new column that has the length of each text

text_data['token_length'] = text_data.apply(lambda row: len(row['tokenized']), axis=1)

# -

text_data[text_data['text'].str.contains('phrase block')]


text_data[text_data.text.str.contains('automatically')].head()


text_data.head()


# ### Pickling

# +
# Pickling

pickle_to(text_data,'text_data.pkl')
pickle_to(data,'data.pkl')


# -

raw[raw['label'] == 'REAL']

raw = pickle_from('raw.pkl')

raw[raw['label'] == 'FAKE']



StarcoderdataJupyterScriptsDedupFiltered