# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # BanFakeNews

# ## Setup and import libraries

# +
# Automatically reloading imported modules
# %load_ext autoreload
# %autoreload 2

import sys
sys.path.append('../..')

# +
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

from src.helpers import *

pd.set_option('display.max_columns', None)

# +
# Change design of plots
sns.set(style="whitegrid")

# Change sizes and resolution of plots
plt.rcParams['figure.figsize'] = (10, 6)
# %config InlineBackend.figure_format='retina'
plt.rcParams.update({'font.size': 15})

# Hide warnings
import warnings
warnings.filterwarnings('ignore')
# -

# ## Load the data

# +
df_labeled_authentic_7k = pd.read_csv('data/LabeledAuthentic-7K.csv')
df_labeled_fake_1k = pd.read_csv('data/LabeledFake-1K.csv')

df = pd.concat([df_labeled_authentic_7k, df_labeled_fake_1k], ignore_index=True)
# -

# ## General descriptive analysis

# Let's check shape of the data - number of rows and attributes:

df.shape

# Overview of the data:

df.head()

# ### Datatypes
#
# **Note:** Be careful, attributes with only NaN values are considered as `float64` type by default.

df.dtypes

# ### Basic characteristics

df.describe()

df.describe(exclude=[np.number])

# ### One-value columns
#
# Which attributes contain only one value?

one_value_attributes_analysis(df)

# ### Missing values
#
# Analysis of missing values in attributes:

missing_values_analysis(df)

# ### Duplicates
#
# Are there any duplicates?

df.duplicated().any()

# ## Attributes analysis
#
# Analysis of all attributes:

# +
skip_attributes = [
    'articleID', 'domain', 'date',
    'headline', 'content'  # also skipping textual attributes, because of Bengali language (not supported)
] # attributes to skip in analysis (e.g. id)

textual_attributes = [
] # attributes with text values (e.g. content of article)
textual_attributes = list(filter(lambda value: value not in skip_attributes, textual_attributes))

numerical_attributes = list(df.select_dtypes([np.number]).columns)
numerical_attributes = list(filter(lambda value: value not in textual_attributes + skip_attributes, numerical_attributes))

categorical_attributes = list(df.select_dtypes(['object', 'category', 'bool']).columns)
categorical_attributes = list(filter(lambda value: value not in textual_attributes + skip_attributes, categorical_attributes))

label_column = 'label' # attribute considered as "label"
# -

# ### Label attribute distribution

df[label_column].value_counts().plot(kind='pie', title='Distribution of predicted classes');

df[label_column].value_counts().plot(kind='bar', title='Distribution of predicted classes');

# ### Numerical attributes
#
# Analysis of numerical attributes:

analyse_numerical_attributes(df, label_column, numerical_attributes)

# ### Categorical attributes
#
# Analysis of categorical attributes:

analyse_categorical_attributes(df, label_column, categorical_attributes)

# ### Textual attributes
#
# Some parts of analysis include preprocessing text. In this case, the following operations are performed:
# * removing special characters (only letters are preserved),
# * removing tokens shorter than 3 characters,
# * removing tokens that are in english stop-words defined by NLTK library,
# * removing accent marks from tokens.
#
# Analysis of textual attributes:

analyse_textual_attributes(df, textual_attributes)

# ## Pairwise analysis
#
# Pairwise analysis of attributes (numerical attributes):

# ### Pair analysis

if numerical_attributes and len(numerical_attributes) > 1:
    sns.pairplot(df, vars=numerical_attributes, hue=label_column);

# ### Correlations
#
# Correlation matrix:

if numerical_attributes and len(numerical_attributes) > 1:
    check_correlations(df, numerical_attributes)

StarcoderdataJupyterScriptsDedupFiltered