# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""
# Importing the libraries Needed.
"""
import pandas as pd
import matplotlib.pyplot as plt
import re
from nltk.stem.porter import PorterStemmer
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score,confusion_matrix
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
import gensim
from sklearn.naive_bayes import MultinomialNB
from xgboost import XGBClassifier
from sklearn.ensemble import RandomForestClassifier
"""
# Reading the files
"""
train_df = pd.read_csv('../input/nlp-getting-started/train.csv')
test_df =  pd.read_csv('../input/nlp-getting-started/test.csv')
submission =pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
train_df.head()
"""
# Info about the Data.
id -> a unique identifier for each tweet.

text -> the text of the tweet.

location -> the location the tweet was sent from (may be blank).

keyword -> a particular keyword from the tweet (may be blank).

target -> in train.csv only, this denotes whether a tweet is about a real disaster (1) or not (0).
"""
train_df.shape, test_df.shape
"""
Since it is a NLP problem we will get rid of Location and keyword Column
"""
labels = train_df['target']
train_df = train_df.drop(['keyword','location','target'],axis = 1)
test_df = test_df.drop(['keyword','location'],axis = 1)


train_df.shape,test_df.shape
labels.head()
labels.value_counts().plot(kind = 'bar')
"""
Train set is not highly imbalanced so we can go on with dataset, Otherwise we would have over
sampled the data
"""
train_df[labels == 0].head()
train_df[labels == 1].head()
test_df.head()
"""
Distribution of lengths of tweets in test and train dataset
"""
train_length = train_df['text'].str.len()
test_length = test_df['text'].str.len()
plt.figure(figsize = (8,3))
plt.hist(train_length,bins = 30, label = "train_texts")
plt.hist(test_length,bins = 30, label = "test_texts")
plt.xlim(0,180)
plt.legend()
plt.show()
"""
Lets combine the train and test dataset and apply cleaning process on texts together.
"""
combined_data = pd.concat([train_df,test_df],ignore_index = True)
combined_data.shape
"""
cleaning text file and appending it to dataframe
"""
#stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()
clean = []
for i in range(0,len(combined_data)):
    clean_text = re.sub('[^a-zA-Z]',' ',combined_data['text'][i])
    clean_text = clean_text.lower()
    clean_text = clean_text.split()
    
    clean_text = [lemmatizer.lemmatize(word) for word in clean_text if word not in stopwords.words('english')]
    #clean_text = [stemmer.stem(word) for word in clean_text if word not in stopwords.words('english')]
    clean_text = ' '.join(clean_text)
    clean.append(clean_text)

combined_data['clean_text']= clean
combined_data.head()
"""
# Creating Bag of Words from the Combined dataset
"""
"""
cv = CountVectorizer(max_features = 3000)
bg = cv.fit_transform(combined_data['clean_text']).toarray()
"""
"""
bg.shape
"""
"""
We can see that there are 24464 different features made using the bag of words.
These 24464 count nothing but the unique words from the combined dataset.
"""
"""
# Creating TFidfVectors

"""
tfidf = TfidfVectorizer(max_features = 2500)
tfidfvectors = tfidf.fit_transform(combined_data['clean_text']).toarray()
tfidfvectors.shape
"""
Lets Split the data Into Orginal train and test shape.
"""
train = tfidfvectors[:7613]
test = tfidfvectors[7613:]
"""
Training the meodel using train dataset
"""
X_train,X_test,y_train,y_test = train_test_split(train,labels,test_size = 0.2,
                                                 random_state = 48)
"""
Training the model using NaiveBayes Classifier
"""
model = MultinomialNB().fit(X_train,y_train)
y_predict = model.predict(X_test)
y_train_predict = model.predict(X_train)
"""
Creating Confusion matrix and analysing our model performance
"""
confusion_matrix(y_test,y_predict)
f1_score(y_test,y_predict)
f1_score(y_train,y_train_predict)
predict = model.predict(test)
submission.head()
submission['target'] = predict
submission.head()
submission.to_csv('submission.csv',index=False)