# -*- coding: utf-8 -*-
"""TestData_DC_EDA.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1m3CirRTAUEKu3puUhB1wi85utStFj-ho

# Data Cleaning and Exploratory Data Analysis of Test Data
The unseen data recieved from Riposte and  saved as a CSV file named - Riposte_Posts_1_25_2021, 12_42_08 PM For testing mental health model.csv

### Step 0: Load the essential Libraries and Riposte Data
"""

# Commented out IPython magic to ensure Python compatibility.
import re
import numpy as np
import pandas as pd
import seaborn as sns
import missingno as msno
import matplotlib.pyplot as plt
# %matplotlib inline
import nltk
from nltk.tokenize import RegexpTokenizer
from nltk.stem import WordNetLemmatizer
!pip install texthero  
import texthero as hero
from texthero import preprocessing
from texthero import stopwords
import plotly.express as px
from PIL import Image

##Load the file
from google.colab import files
data_to_load = files.upload()

"""###Step 1: Text Cleaning"""

##Test Data cleaning 
!pip install wordninja
import wordninja

# Import the Mental Wellbeing Test Data
Riposte_MWB_Data = pd.read_csv('Riposte_Posts_1_25_2021, 12_42_08 PM For testing mental health model.csv')
# Viewing a glimpse of the Data
Riposte_MWB_Data.head()

# Using Texthero to do the following changes in one go and in consistent with training data
# 1. fillna(s) Replace not assigned values with empty spaces.
# 2. lowercase(s) Lowercase all text.
# 3. remove_digits() Remove all blocks of digits.
# 4. remove_punctuation() Remove all string.punctuation (!'#$%&'()*+,-./:;<=>?@[\]^_`{|}~).
# 5. remove_diacritics() Remove all accents from strings.
# 6. remove_stopwords() Remove all stop words.
# 7. remove_whitespace() Remove all white space between words.
Stopwords = stopwords.DEFAULT
SAD_Stopwords = Stopwords.union(set(['aot', 'aos', 'aoll',  'aom', 'aove', 'put', 'te', 'r','want','feel','like','know']))
#trials test removed stopwords cant, dont ,didnt , wanna, gonna, much, made, many, day, today, still, and lots
#trials test added stop word like,feel, know
#create a custom cleaning pipeline
DC_Pipeline = [preprocessing.fillna
               , preprocessing.lowercase
               , preprocessing.remove_digits
               , preprocessing.remove_punctuation
               , preprocessing.remove_diacritics
               , preprocessing.remove_whitespace
               , preprocessing.remove_urls
               #, preprocessing.stem
              ]

Riposte_MWB_Data.insert(9, 'Cleaned Topic Hashtags', hero.clean(Riposte_MWB_Data['Topic Hashtags'], pipeline = DC_Pipeline))
Riposte_MWB_Data['Cleaned Topic Hashtags'] = hero.remove_stopwords(Riposte_MWB_Data['Cleaned Topic Hashtags'], stopwords = SAD_Stopwords)

## Split Hashtags into words
Riposte_MWB_Data['Cleaned Topic Hashtags'] = Riposte_MWB_Data['Cleaned Topic Hashtags'].apply(lambda x: ' '.join(wordninja.split(x)))
Riposte_MWB_Data.info()

"""###Step 2: Feature Engineering on Data"""

# Import Required libraries
!pip install -U textblob
from textblob import TextBlob

# Using TextBlob to generate the polarity and subjectivity scores for the test data
polarity = lambda t: TextBlob(t).sentiment.polarity
subjectivity = lambda t: TextBlob(t).sentiment.subjectivity

Riposte_MWB_Data['Polarity'] = Riposte_MWB_Data['Cleaned Topic Hashtags'].apply(polarity)
Riposte_MWB_Data['Subjectivity'] = Riposte_MWB_Data['Cleaned Topic Hashtags'].apply(subjectivity)
Riposte_MWB_Data.head(5)

##Filtering only "Face Palm" to test the classifier
Riposte_MWB_Data = Riposte_MWB_Data.loc[Riposte_MWB_Data['Post Type'] == 'Facepalm']
Riposte_MWB_Data.info()

"""### Step 3: Download Cleaned Text Data"""

Riposte_MWB_Data.to_csv("Riposte_MWB_Data.csv")

# Uncomment the following 2 lines Download the CSV file of training dataset
from google.colab import files
files.download('Riposte_MWB_Data.csv')

