"""
<div style="width:100%;text-align: center;"> <img align=middle src="https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQfrEUPl6MA-wBBVSQr86WCOrG1uNtqn_NEKQ&usqp=CAU" alt="Heat beating" style="margin-top:3rem;"> </div>
"""
"""
--------------------------------------
With the growth of speach freedom and various forms of communacation certain groups of people/agenceis want to promote thier own perspective of the world. Unfortunetley, in doing so sometimes they are trying to manipulate people opinions by generating fake news. It is done to make people believe in what can promote interest of news generating agencies.

From the perspective of the users it is important to understand wether they can trust the news or they should be suspicous of it to avoid becoming a victim of some sort of scum or disinformation.
"""
"""
--------------------------------------
**Dataset**: In this project we deal with fake and real news dataset.   
**Dataset
Research question**: The main question we wanted to answer is: wheather the given news is a true news or a fake one?  
**Methods and Findings**:  To answer to this question we have created a base model using logistic regression method. The base model serves as a reference for the further improvement by the LSTM-model. However, the results of the base model were surprisingly good. Hence, we initiated a detailed exploratory analysis of the data which revealed that there are some problems in the way the data has been formed.
"""
"""
# Reading the data
"""
"""
Our dataset consists of two parts: fake and true news datasets. The respective number of news in each dataset is $23481, 21417$.

In the concatinated dataset there are $2$ categories of labels to classify the news as either true or false based on the news content.
"""
import pandas as pd

data_true = pd.read_csv('../input/fake-and-real-news-dataset/True.csv')
data_fake = pd.read_csv('../input/fake-and-real-news-dataset/Fake.csv')

data_true['label'] = 1
data_fake['label'] = 0
data = pd.concat([data_true, data_fake], axis = 0)
data = data.sample(frac=1).reset_index(drop=True)

print(data_true.shape), print(data_fake.shape), print(data.shape)
data_true.head()
data_fake.head()
print("Total number of news: ", len(data))
print("Number of true news: ", len(data_true))
print("Number of fake news: ", len(data_fake))
"""
There are almost an equal quantity of fake vs. true news.
"""
X_text = data['text']
y_label = data['label']
"""
# Base model
"""
from sklearn.model_selection import train_test_split

X, X_test, y, y_test = train_test_split(X_text, y_label, test_size=0.2, random_state=13) # split to train and test data
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=13) # split to train and validation data
y_train.mean()
"""
The mean of the labels in the train data shows that if we use a model that naively assigns to each observation $1$ i.e. classifies every news as 'fake' we would classify $48$ percent of the news correctly.

Let's try to improve this accuracy.
"""
from sklearn.feature_extraction.text import TfidfVectorizer

tfv = TfidfVectorizer()
tfv.fit(X_train)

X_train_tfv = tfv.transform(X_train)
X_valid_tfv = tfv.transform(X_val)
import numpy as np
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import log_loss

lr = LogisticRegression()
lr.fit(X_train_tfv, y_train)
lr_preds = lr.predict(X_valid_tfv)
lr_preds_proba = lr.predict_proba(X_valid_tfv)

print("The prediction accuracy is {0}".format(np.mean(y_val==lr_preds)))
print("The loss is {0}".format(round(log_loss(y_val, lr_preds_proba), 2)))
"""
# Data exploration
"""
"""
The high prediction accuracy for the base model surprised me and made me suspicous about the data. Let's investigate it to see wheather there are any peculiarities in the data.

In particular at this moment we should find out if there is some feature that helps to easily differentiate between true and fake data. Now we will try to see if:
- There is a significant difference between the words usage in true vs fake news.
- There is a significant difference between the lenghts of the text in true vs fake news.
- There is a significant difference of topics of the true vs fake news.
"""
from collections import defaultdict
from nltk.corpus import stopwords

stopwords = stopwords.words('english')


# create a corpuse of words in the data.
def create_corpuse(data, column_name):
    words_by_frequency = defaultdict(int)

    for title in data['{0}'.format(column_name)]:
        for word in title.lower().split(' '):
            if word not in stopwords:
                words_by_frequency[word] += 1

    return sorted(words_by_frequency.items(), key=lambda x: x[1], reverse=True)

counter_words_title_true = create_corpuse(data_true, 'title')
counter_words_title_fake = create_corpuse(data_fake, 'title')

counter_words_text_true = create_corpuse(data_true, 'text')
counter_words_text_fake = create_corpuse(data_fake, 'text')
%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns


# plot the most frequent words in the data.
def most_frequent_words_barplot(true_words, fake_words, part = 'titles'):
    fig, ax = plt.subplots(1, 2, figsize = (16, 8)) 
    sns.barplot(x=[x[1] for x in true_words[:30]], y =[x[0] for x in true_words[:30]], palette='Blues', ax = ax[0])
    sns.barplot(x=[x[1] for x in fake_words[:30]], y =[x[0] for x in fake_words[:30]], palette='Reds', ax = ax[1])

    ax[0].title.set_text("30 most common words in true news articles {0}".format(part))
    ax[1].title.set_text("30 most common words in fake news articles {0}".format(part))
# Most frequent words in titles of true vs fake news.
most_frequent_words_barplot(counter_words_title_true, counter_words_title_fake)
# Most frequent words in texts of true vs fake news.
most_frequent_words_barplot(counter_words_text_true, counter_words_text_fake)
"""
The two plots above don't show any apparent difference between the titles of the news. The same can be said for the text. Though I noticed that the true news text mentiones the 'reuters'-as a source for the news quite frequentley.
"""
"""
Let's see if there is a difference between the lengths of the text in the true/fake news.
"""
# get number of words in the data
def text_length(text):
    word_count = 0
    for word in text.split(" "):
        word_count += 1
    return word_count

true_words_count = data_true['text'].apply(text_length)
fake_words_count = data_fake['text'].apply(text_length)

words_count = pd.DataFrame({'True': true_words_count, 'Fake': true_words_count})

plt.figure(figsize=(10, 6))
words_count.boxplot()
plt.xlabel('News category')
plt.ylabel('Distribution of words in the news text')
"""
Surprisingly fake news have more words than true ones. Hypothetically, this can be explained by the desire of fake news to convince the audience in their reliablity and hence, they use more words and tricks to do it. However, at the same the true news should give an extensive explanation of their articles to the audience. Hence, some mystery occures here.
"""
true_duplicated = data_true['text'].duplicated()
fake_duplicated = data_fake['text'].duplicated()

sum(true_duplicated), sum(fake_duplicated)
"""
To partly resolve this phenomenon we looked at the presence of the duplicated texts and found out that there are over $6000$ dupliacte text in the fake news. 
"""
"""
Number of unique words can be investigated too.
"""
num_unique_words_true = len(counter_words_text_true)
num_unique_words_fake = len(counter_words_text_fake)

plt.figure(figsize=(10, 6))
sns.barplot(y=[num_unique_words_true, num_unique_words_fake], x=['True', 'Fake'], palette='Reds')
plt.xlabel('News category')
plt.ylabel('Number of unique words in news text')
"""
Again fake news use more unique tokens than the true news. This can be connected with the fact that fake news use lots of external links to twitter users and web-sites (as we found out below).
"""
"""
A good idea is to investigate to which sources news refer to. In particular we will look at the usage of twitter or some external web-site links. In addition to this we can look if some other news agencies names are used apart of 'reuters'.
"""
import re

# get the number of 'element' in the data
def find_text_elements(data, column_name, element):
    count_element = 0
    re_element = re.compile(element)
    for text in data['{0}'.format(column_name)]:
        count_element += len(re.findall(re_element, text.lower()))
    return count_element
true_twitter_user_names = find_text_elements(data_true, 'text', element = '@[0-9A-Za-z]*')
fake_twitter_user_names = find_text_elements(data_fake, 'text', element = '@[0-9A-Za-z]*')

plt.figure(figsize=(10, 6))
sns.barplot(y=[true_twitter_user_names, fake_twitter_user_names], x=['True', 'Fake'], palette='Reds')
plt.xlabel('News category')
plt.ylabel('Number of twitter users in news text')
"""
It seems to be the case that the fake news are using/citing twitter far more extensively than the true news. So, there seems to be a bias in the data.
"""
true_sites_web_links = find_text_elements(data_true, 'text', element = 'http|www\\.|\\.com')
fake_sites_web_links = find_text_elements(data_fake, 'text', element = 'http|www\\.|\\.com')


plt.figure(figsize=(10, 6))
sns.barplot(y=[true_sites_web_links, fake_sites_web_links], x=['True', 'Fake'], palette='Reds')
plt.xlabel('News category')
plt.ylabel('Number of web-link references in news text')
"""
Again the fake news refer to different web-sites far more often than the true news.
"""
"""
What popular news agencies the news use?
"""
# some popular news agencies names randomly chosen by me
articles = {'the washington post': [0, 0], 'cnn': [0, 0], 'bbc': [0, 0], 'reuters': [0, 0], 
            'fox news': [0, 0], 'the new york times': [0, 0], 'nbc': [0, 0]}
for key, _ in articles.items():
    true_value = find_text_elements(data_true, 'text', element=key)
    fake_value = find_text_elements(data_fake, 'text', element=key)
    articles[key] = true_value, fake_value
    
articles
articles_count = pd.DataFrame(articles)


plt.figure(figsize=(10, 6))
articles_count.T.plot(kind='barh', colormap='autumn')
plt.xlabel('Number of times the articles are mentioned');
plt.legend(['True', 'Fake']);
"""
We see that fake news reffer to wildly known news aggencies more frequently than the true news. I think, this can be partly explained by the desire of those fake news to be seem more reliable by mentioning the famous agencies names even if they are connected to the essence of the article only remotley. 
From another point I suppose, that the true news are represnted by those famous agencies and they rarely want to share the spotlight by mentioning a competitive agency as their source. Instead they use their reporters to get the reliable news.

At the same time the things are not so clear with 'reuters'. So, an assumption is that either 'reuters' is very reliable news agency and the others are not or the data should be biased. The latter will partly explain the high prediction results for the base model.
"""
"""
# Model evaluation
"""
"""
Though during the exploratory analysis we saw that the data seems to be biased in the way it is formed we decided to create an LSTM-model too to see if it captures the underlying features better than the simple logistic regression.

Note that we are not doing any data cleaning in this case becuase already by the nature of the data peculiarities we would expect to get a model with quite high prediction accuracy.
"""
from tensorflow import keras

num_words = 10000

tokenizer = keras.preprocessing.text.Tokenizer(num_words=num_words+1)
tokenizer.fit_on_texts(X_train)

X_train_seq = tokenizer.texts_to_sequences(X_train)
X_val_seq = tokenizer.texts_to_sequences(X_val)
X_train_padded = keras.preprocessing.sequence.pad_sequences(X_train_seq,
                                                    maxlen=200,
                                                    padding='post',
                                                    truncating='post')
X_val_padded = keras.preprocessing.sequence.pad_sequences(X_val_seq,
                                                    maxlen=200,
                                                    padding='post',
                                                    truncating='post')
X_train_padded.shape, len(y_train)
X_val_padded.shape, len(y_val)
model = keras.Sequential([
    keras.layers.Embedding(num_words+1, 16, input_length=200),
    keras.layers.Bidirectional(keras.layers.LSTM(64)),
    keras.layers.Dense(16, activation='relu'),
    keras.layers.Dense(1, activation='sigmoid')
])

model.compile(loss='binary_crossentropy', metrics=['accuracy'],
             optimizer='adam')

history = model.fit(X_train_padded, y_train, 
                    validation_data=(X_val_padded, y_val), epochs = 10)
fig, ax = plt.subplots(1, 2, figsize=(10, 4))
ax[0].plot(history.history['accuracy'])
ax[0].plot(history.history['val_accuracy'])

ax[1].plot(history.history['loss'])
ax[1].plot(history.history['val_loss'])

ax[0].set_title('Model accuracy')
ax[0].set_ylabel('accuracy')
ax[0].set_xlabel('epoch')
ax[0].legend(['train', 'val'])

ax[1].set_title('Model loss')
ax[1].set_ylabel('loss')
ax[1].set_xlabel('epoch')
ax[1].legend(['train', 'val'])
"""
The model starts to slightly overfit at the $5$th epoch.
"""