"""
![](https://www.kdnuggets.com/images/sentiment-fig-1-689.jpg)
"""
"""
* Sentiment analysis :- Sentiment analysis help to get wider public opinion that used by businesses to detect sentiment in social data, gauge brand reputation, and understand customers allows brands to learn what makes customers happy or frustrated
"""
import numpy as np 
import pandas as pd 
import matplotlib.pyplot as plt
import seaborn as sns
import sklearn
from sklearn.model_selection import train_test_split
import os
import warnings
warnings.filterwarnings("ignore")
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))
data=pd.read_csv('../input/twitter-airline-sentiment/Tweets.csv')
data.head(10)
"""
# *null value in percentage*
"""
data.isnull().sum() * 100 / len(data)
data.columns
data.shape
"""
# EDA OF THE data sets
"""
"""
HERe check about the airline sentiment about the user 
"""
print(data['airline_sentiment'].value_counts())
sns.countplot(data['airline_sentiment'])
"""
# Total no of tweet for each airline
"""
print(data.airline.value_counts())
sns.countplot(data['airline'])
"""
united airways get more tweet
virgin America get the least tweet
"""
#remove unwanted column from the data 
del data['airline_sentiment_gold']
del data['negativereason_gold']
del data['tweet_coord']
"""
# Here we check airline sentiment for each airline
"""
#calculate no of sentiment for each airline 
plt.figure(1,figsize=(10,10))
airlines= ['US Airways','United','American','Southwest','Delta','Virgin America']
for i in airlines:
    indices=airlines.index(i)
    plt.subplot(2,3,indices+1)
    new_value=data[data['airline']==i]
    print(new_value['airline_sentiment'].value_counts(),i)
    Index = [1,2,3]
    sns.countplot(new_value['airline_sentiment'])
    plt.title('Count of Moods of '+i)
"""
**Data show that the when customer get not good Service from the airline then tweet about it the wrtite less twee when get good service from the airline**
"""
"""
**United, US Airways, American substantially get negative reactions.
Tweets for Virgin America are the most balanced.
**
"""
"""
# Clean the text data and then we check most postive and most negative comment on the text data
"""
#install library of nltk to  clean the text data
import nltk
from wordcloud import WordCloud,STOPWORDS
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
import re

nltk.download('stopwords')
stop_words = stopwords.words('english')
"""
***Word cloud:-visual representation of the text data They are very popular because they help us easily spot the most frequently occurring words. The more frequent the word is used, the larger and bolder it is displayed. That the help to tackel the main problem ***
"""
new_data=data[data['airline_sentiment']=='negative']
words = ' '.join(new_data['text'])
cleaned_word=' '.join([word for word in words.split()
                         if 'http' not in word
                         and not word.startswith('@')
                          and word !='RT'])
wordcloud = WordCloud(stopwords=STOPWORDS,
                      background_color='black',
                      width=3000,
                      height=2500
                     ).generate(cleaned_word)
plt.figure(1,figsize=(10,10))
plt.imshow(wordcloud)
plt.axis('off')
plt.show()
"""
*postive sentiment about the airline* 
"""
new_data=data[data['airline_sentiment']=='positive']
words = ' '.join(new_data['text'])
cleaned_word=' '.join([word for word in words.split()
                         if 'http' not in word
                         and not word.startswith('@')
                          and word !='RT'])
wordcloud = WordCloud(stopwords=STOPWORDS,
                      background_color='black',
                      width=3000,
                      height=2500
                     ).generate(cleaned_word)
plt.figure(1,figsize=(10,10))
plt.imshow(wordcloud)
plt.axis('off')
plt.show()
"""
*Neutral sentiment about the airline*
"""
neutral_text=data[data['airline_sentiment']=='neutral']
words=''.join(neutral_text['text'])
cleaned_word=' '.join([word for word in words.split()
                      if 'http' not in word
                       and not word.startswith('@')
                        and word!='RT'])
wordcloud=WordCloud(stopwords=STOPWORDS,
                    background_color='black',
                      width=3000,
                      height=2500).generate(cleaned_word)
plt.figure(1,figsize=(10,10))
plt.imshow(wordcloud)
plt.axis('off')
plt.show()
"""
***Percentage of the negative reason***
"""
print(data.negativereason.value_counts())
plt.figure(figsize=(25,5))
sns.countplot(data.negativereason)
"""
***Neagative reason by airline that the get type of reviews gave by the customer***
"""
plt.figure(1,figsize=(35,36))
airline=['US Airways','United','American','Southwest','Delta','Virgin America']
for i in airline:
    indices=airline.index(i)
    plt.subplot(3,2,indices+1)
    new_values=data[data['airline']==i]
    Index=[1,2,3,4,5,6,7,8,9,10]
    sns.countplot(new_values['negativereason'])
    plt.title('count of Moods of'+i)
"""
***Data processing technique remove noise from the text data ***
**Different text processing techniques:**
**Tokenization:-separating a piece of text into smaller units the are bascilly called the tokens Tokenization is essentially splitting a phrase, sentence, paragraph, or an entire text document into smaller units, such as individual words or terms**
"""
text_cleaning_re = "@\S+|https?:\S+|http?:\S|[^A-Za-z0-9]+"
def preprocess(x,stem=False):
    x=re.sub(text_cleaning_re,'  ',str(x).lower()).strip()
    tokens=[]
    for token in x.split('\n'):
        if token not in stop_words:
            if stem:
                tokens.append(stemmer.stem(token))
            else:
                tokens.append(token)
        return '  '.join(tokens)
data.text=data.text.apply(lambda x:preprocess(x))
"""
*postive and neutrl sentiment aree coherent with each other thats why we using label encoding technique 1 denote postive and negative sentiment convert to the 0*
"""
data['sentiment']=data['airline_sentiment'].apply(lambda x: 0 if x=='negative' else 1)
print(data.sentiment.value_counts())
y=data.sentiment
"""
***machine can never stand the word thats why we convert word to vector that help to easy undderstand to the machine differnet type of vectorizaion technique used in ml .
TF-IDF are sparse vectors where the number of non-zero values in the vector is equal to the number of unique words in the document***
![](https://miro.medium.com/max/1838/1*qQgnyPLDIkUmeZKN2_ZWbQ.png)
"""
from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(min_df=10)
X= vectorizer.fit_transform(data.text)
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.2, random_state=0)
"""
**using simple ml model to predict the sentiment of the user**

"""
"""
***Logistic regression :-logistic regression basscially used in the sigmoid function that help to the where the user stasfied or not***
"""
model = sklearn.linear_model.LogisticRegression(penalty="l2", C=0.1)
model.fit(X_train, y_train)
score = model.score(X_test, y_test)
score
"""
***That’s the way to calculate the Shapley value: It is the average of the marginal contributions across all permutations theoretic approach to explain the output of any machine learning model. It connects optimal credit allocation with local explanations using the classic Shapley values from game theory and their related extensions (see papers for details and citations).***
"""
import shap
explainer = shap.LinearExplainer(model, X_train, feature_dependence="independent")
shap_values = explainer.shap_values(X_test)
X_test_array = X_test.toarray() # we need to pass a dense version for the plotting functions
shap.summary_plot(shap_values, X_test_array, feature_names=vectorizer.get_feature_names())
from sklearn.metrics import classification_report, confusion_matrix
sns.heatmap(confusion_matrix(y_test, model.predict(X_test)))
print(classification_report(y_test, model.predict(X_test)))
"""
Random Forest:-The Random Forest  classifiers are suitable for dealing with the high dimensional noisy data in text classification
"""
from sklearn.ensemble import RandomForestClassifier
models=RandomForestClassifier()
model.fit(X_train, y_train)
score = model.score(X_test, y_test)
score
from sklearn.metrics import classification_report, confusion_matrix
print(classification_report(y_test, model.predict(X_test)))
sns.heatmap(confusion_matrix(y_test, model.predict(X_test)))
"""
**SVC:-is an algorithm that determines the best decision boundary between vectors that belong to a given group (or category) and vectors that do not belong to it.**
"""
from sklearn.svm import SVC
svm=SVC()
svm=SVC(kernel="rbf", C=0.025, probability=True)
svm.fit(X_train,y_train)
score = model.score(X_test, y_test)
score
from sklearn.metrics import classification_report, confusion_matrix
print(classification_report(y_test, model.predict(X_test)))
sns.heatmap(confusion_matrix(y_test, model.predict(X_test)))
"""
**ANY FEEDBACK IN THE COMMENTS WILL BE HIGHLY APPRECIATED**
"""