"""
## Import Libraries
"""
import pandas as pd
import numpy as np
from nltk.stem import WordNetLemmatizer
from bs4 import BeautifulSoup
from nltk.corpus import stopwords
from wordcloud import WordCloud
import nltk
from textblob import TextBlob
from nltk.tokenize import word_tokenize
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, TfidfTransformer
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report
import re
import string
import glob
import emoji
import warnings
warnings.filterwarnings('ignore')
"""
## Importing Data
"""

# get karachi tweets
karachi = pd.read_csv('../input/tweets/Karachi_travel_tweets.csv')
karachi1 = pd.read_csv('../input/tweets/Karachi_visit_tweets.csv')

# concatenate karachi tweets
karachi_df = pd.concat([karachi, karachi1], ignore_index=True)
karachi_df['city'] = 'karachi'

# get lahore tweets
lahore = pd.read_csv('../input/tweets/visit_lahore_tweets converted.csv')
lahore1 = pd.read_csv('../input/tweets/travel_lahore_tweets converted.csv')

# concatenate lahore tweets
lahore_df = pd.concat([lahore, lahore1], ignore_index=True)
lahore_df['city'] = 'lahore'

# get gilgit tweets
gilgit = pd.read_csv('../input/tweets/Gilgit_tweets.csv')
gilgit['city'] = 'gilgit'

# get neelum valley tweets
neelum_valley = pd.read_csv('../input/tweets/NeelumValley_tweets.csv')
neelum_valley['city'] = 'neelum_valley'

# get northern tweets
northern = pd.read_csv('../input/tweets/Pakistan_north_tweets.csv')
northern1 = pd.read_csv('../input/tweets/Pakistan_northern_tweets.csv')

# concatenate northern tweets
northern_df = pd.concat([northern, northern1], ignore_index=True)
northern_df['city'] = 'northern area'

# # Concatenate all data into one DataFrame
df = pd.concat([karachi_df, lahore_df, gilgit, neelum_valley, northern_df], ignore_index=True)
df.head()
# df = pd.read_csv('../input/Gilgit_tweets.csv')
# df.head()
"""
## Data Preprocessing
"""
def clean_text(text,replace=''):
    lemmatizer = WordNetLemmatizer() 
    stop_words = set(stopwords.words('english'))
    REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;]')
    BAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_]')
    
    text = re.sub(r'\d+', '', str(text))
    text = BeautifulSoup(text, "lxml").text # HTML decoding
    text = text.lower() # lowercase text
    text = BAD_SYMBOLS_RE.sub('', text) # delete symbols which are in BAD_SYMBOLS_RE from text
    text = "".join(i for i in text if ord(i) < 128) # remove non-ascii characters
    text = text.translate(str.maketrans('', '', string.punctuation)) # remove punctuation characters
    text = re.sub('((www\.[^\s]+)|(https?://[^\s]+))', replace, text) # removes URL from string
    text = emoji.demojize(text) # remove emojies
    # remove stop words andapplying Lemmatization
    text = ' '.join(lemmatizer.lemmatize(word, pos ="a") for word in text.split() if word not in stop_words) 
    text = REPLACE_BY_SPACE_RE.sub(' ', text) # replace REPLACE_BY_SPACE_RE symbols by space in text
    return text
df = df.drop(['hashtags', 'Unnamed: 2', 'Unnamed: 3', 'Unnamed: 4'], axis=1)
df['text'] = df['text'].apply(clean_text)
df['text'].replace('nan', np.nan, inplace=True)
df['text'].replace('', np.nan, inplace=True)
df = df.dropna()
df.reset_index(inplace = True, drop = True)
df.head()
"""
## Word Cloud Data
"""
tweet = ' '.join(df["text"])
#function to split text into word
tokens = word_tokenize(tweet)
vocabulary = set(tokens)
frequency_dist = nltk.FreqDist(tokens)

wordcloud = WordCloud(width=1000, height=500).generate_from_frequencies(frequency_dist)
plt.figure( figsize=(20,10), facecolor='k' )
plt.imshow(wordcloud)
plt.axis("off")
plt.tight_layout(pad=0)
plt.show()
"""
## Find Target Class for Tweets
"""
polarity_score = []

for text in df['text']:    
    blob = TextBlob(text)
    for sentence in blob.sentences:
        if sentence.sentiment.polarity > 0.0:
            polarity_score.append('positive')
        elif sentence.sentiment.polarity < 0.0:
            polarity_score.append('negative')
        else:
            polarity_score.append('neutral')

df['target'] = pd.DataFrame(polarity_score)
df.head()
"""
## Classes Frequency Graph
"""
df['target'].value_counts().plot.bar(color=['purple', 'blue', 'red'])
"""
## Positive Tweets Word Cloud
"""
tweet = ' '.join(df['text'][df['target'] == 'positive'] )
#function to split text into word
tokens = word_tokenize(tweet)
vocabulary = set(tokens)
frequency_dist = nltk.FreqDist(tokens)

wordcloud = WordCloud(width=1000, height=500).generate_from_frequencies(frequency_dist)
plt.figure( figsize=(20,10), facecolor='k' )
plt.imshow(wordcloud)
plt.axis("off")
plt.tight_layout(pad=0)
plt.show()
"""
## Negative Tweets Word Cloud
"""
tweet = ' '.join(df['text'][df['target'] == 'negative'] )
#function to split text into word
tokens = word_tokenize(tweet)
vocabulary = set(tokens)
frequency_dist = nltk.FreqDist(tokens)

wordcloud = WordCloud(width=1000, height=500).generate_from_frequencies(frequency_dist)
plt.figure( figsize=(20,10), facecolor='k' )
plt.imshow(wordcloud)
plt.axis("off")
plt.tight_layout(pad=0)
plt.show()
"""
## Class Labels Encoding
"""
label = LabelEncoder()
df_ = df.copy()
df_['target'] = label.fit_transform(df_['target'])
# dummies = pd.get_dummies(df.target)
# df = pd.concat([df, dummies], axis=1)
df_.head()
"""
## Split the Data in train and test
"""
#Seperate data into feature and results
X, y = df_['text'], df_['target']

#Split data in train and test sets.
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=0)
"""
## Uni-Gram TF-IDF Transormation
"""
my_categories=['0','1','2']
tfidf = TfidfVectorizer(min_df=2, max_df=0.5, ngram_range=(1, 1))

# TF-IDF BASED FEATURE REPRESENTATION
tfidf.fit_transform(X_train)
        
train_feature_set=tfidf.transform(X_train)
test_feature_set=tfidf.transform(X_test)
"""
## Logistic Regression 
"""
# instantiate the model (using the default parameters)
logreg = LogisticRegression(random_state=0)

logreg.fit(train_feature_set,y_train)

#
y_pred=logreg.predict(test_feature_set)
lrm1 = accuracy_score(y_pred, y_test)

print('accuracy %s' % lrm1)
print(classification_report(y_test, y_pred,target_names=my_categories))
print('Confusion Matrix \n',confusion_matrix(y_test,y_pred))
"""
## Naive Bayes
"""
nb = MultinomialNB()

nb.fit(train_feature_set, y_train)

y_pred = nb.predict(test_feature_set)
nb1 = accuracy_score(y_pred, y_test)

print('accuracy %s' % nb1)
print(classification_report(y_test, y_pred,target_names=my_categories))
print('Confusion Matrix \n',confusion_matrix(y_test,y_pred))
"""
## Random Forest Classifier
"""
rclf = RandomForestClassifier()
rclf.fit(train_feature_set,y_train)
y_pred = rclf.predict(test_feature_set)
print('accuracy %s' % accuracy_score(y_test,y_pred))
print(classification_report(y_test, y_pred,target_names=my_categories))
print('Confusion Matrix \n',confusion_matrix(y_test,y_pred))
"""
## Bi-Gram TF-IDF Transforamtion 
"""
tfidf = TfidfVectorizer(min_df=2, max_df=0.5, ngram_range=(1, 2))

# TF-IDF BASED FEATURE REPRESENTATION
tfidf.fit_transform(X_train)
        
train_feature_set=tfidf.transform(X_train)
test_feature_set=tfidf.transform(X_test)
"""
## Logistic Regression
"""
# instantiate the model (using the default parameters)
logreg = LogisticRegression(random_state=0)

logreg.fit(train_feature_set,y_train)

#
y_pred=logreg.predict(test_feature_set)
lrm1 = accuracy_score(y_pred, y_test)

print('accuracy %s' % lrm1)
print(classification_report(y_test, y_pred,target_names=my_categories))
print('Confusion Matrix \n',confusion_matrix(y_test,y_pred))
"""
## Naive Bayes 
"""
nb = MultinomialNB()

nb.fit(train_feature_set, y_train)

y_pred = nb.predict(test_feature_set)
nb1 = accuracy_score(y_pred, y_test)

print('accuracy %s' % nb1)
print(classification_report(y_test, y_pred,target_names=my_categories))
print('Confusion Matrix \n',confusion_matrix(y_test,y_pred))
"""
## Random Forest Classifier
"""
rclf = RandomForestClassifier()
rclf.fit(train_feature_set,y_train)
y_pred = rclf.predict(test_feature_set)
print('accuracy %s' % accuracy_score(y_test,y_pred))
print(classification_report(y_test, y_pred,target_names=my_categories))
print('Confusion Matrix \n',confusion_matrix(y_test,y_pred))
print('Proportion of the classes in the data:')
print(df.target.value_counts() / len(df))
"""
## City wise percentage
"""
def city_wise(city):
    print(city, 'sentiemnt percentage pie chart')
    val = df[df['city'] == city]['target']
    (val.value_counts() / len(val)).plot.pie(autopct='%1.1f%%')
city_wise('gilgit') # pass city name to function
"""
## Finding Best City for Visiting
"""
cities = ['lahore', 'karachi', 'gilgit', 'northern area', 'neelum_valley']

percentage = []
for city in cities:
    val = df[df['city'] == city]['target']
    percentage.append({city : dict((val.value_counts() / len(val)))})

percentage
positive_tweet_percentage = [] 
for p in percentage:
    for city in p:
        positive_tweet_percentage.append((city,p[city]['positive']))
cities =  pd.DataFrame(positive_tweet_percentage, columns =['city', 'percentage'])
cities = cities.sort_values('percentage', ascending=False)
cities = cities.set_index('city')
cities
cities.plot.bar(color = 'rgbkymc')
plt.title('Best to worst city for visiting')