# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Exercise 08
#
# ## Analyze how travelers expressed their feelings on Twitter
#
# A sentiment analysis job about the problems of each major U.S. airline. 
# Twitter data was scraped from February of 2015 and contributors were 
# asked to first classify positive, negative, and neutral tweets, followed
# by categorizing negative reasons (such as "late flight" or "rude service").

# +
import pandas as pd
import numpy as np

# %matplotlib inline
import matplotlib.pyplot as plt

# read the data and set the datetime as the index
import zipfile
with zipfile.ZipFile('../datasets/Tweets.zip', 'r') as z:
    f = z.open('Tweets.csv')
    tweets = pd.read_csv(f, index_col=0)

tweets.head()
# -

tweets.shape

# ### Proportion of tweets with each sentiment

tweets['airline_sentiment'].value_counts()

# ### Proportion of tweets per airline
#

tweets['airline'].value_counts()

pd.Series(tweets["airline"]).value_counts().plot(kind = "bar",figsize=(8,6),rot = 0)

pd.crosstab(index = tweets["airline"],columns = tweets["airline_sentiment"]).plot(kind='bar',figsize=(10, 6),alpha=0.5,rot=0,stacked=True,title="Sentiment by airline")

# # Exercise 8.1 
#
# Predict the sentiment using CountVectorizer, stopwords, n_grams, stemmer, TfidfVectorizer
#
# use Random Forest classifier

from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.ensemble import RandomForestClassifier
from nltk.stem.snowball import SnowballStemmer
from nltk.stem import WordNetLemmatizer

X = tweets['text']
y = tweets['airline_sentiment'].map({'negative':-1,'neutral':0,'positive':1})



# # Exercise 8.2
#
# Train a Deep Neural Network with the following architecture:
#
# - Input = text 
# - Dense(128)
# - Relu Activation
# - BatchNormalization
# - Dropout(0.5)
# - Dense(10, Softmax)
#
# Optimized using rmsprop using as loss categorical_crossentropy
#
# Hints: 
# - test with two iterations then try more. 
# - learning can be ajusted
#
# Evaluate the performance using the testing set (aprox 55% with 50 epochs)

from keras.models import Sequential
from keras.utils import np_utils
from keras.layers import Dense, Dropout, Activation, BatchNormalization
from keras.optimizers import RMSprop
from keras.callbacks import History
from livelossplot import PlotLossesKeras



StarcoderdataJupyterScriptsDedupFiltered