# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/IanTirok/Climate-change-tweets-analysis/blob/main/Climate_Change_Tweets_Classification_models.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="EbTdWmBR6E4d"
# ##Kneighbors classifier

# + colab={"base_uri": "https://localhost:8080/"} id="Llz_MiVeK-hY" outputId="c3409aa5-e167-4218-df0c-764c74959359"
# !pip install wandb

# + colab={"base_uri": "https://localhost:8080/"} id="XQf-1d5fLXzA" outputId="a8b00b01-6fcc-446d-cff6-de39464265b5"
import wandb
wandb.login()

# + id="7KMs3KKn-EBi"

from wandb.keras import WandbCallback

# + colab={"base_uri": "https://localhost:8080/", "height": 580} id="Ur-fF0pQLlZX" outputId="ef6eb805-ecfa-4eae-a446-9ac6892cdef1"
config = wandb.config
wandb.init(project="Climate change Tweet Classification",
           notes="tweak baseline",
           tags=["baseline", "Remote Learning"],
           config=config)


# + id="hVa2lZgPw3PC"
import pandas as pd
import numpy as np
from google.colab import drive
from sklearn.metrics import confusion_matrix,accuracy_score, classification_report
# drive.mount('/content/drive')

# + colab={"base_uri": "https://localhost:8080/"} id="fPlkS-bCnwkC" outputId="87a8a74b-b671-4873-f085-85c0ae22c7a2"
train_df = pd.read_csv('/content/Clean classified data_kwisha.csv')
test_df = pd.read_csv('/content/Test (2).csv')
train_df['class'].value_counts()

# + id="X7yWNhUNzGFs"
from sklearn import preprocessing
le = preprocessing.LabelEncoder()
train_df['class'] = le.fit_transform(train_df['class'])

# + colab={"base_uri": "https://localhost:8080/"} id="sdwdYU0NxXfb" outputId="7a7c7665-68ac-4580-8e44-c9ac9ecb95e2"
#SPLITTING THE TRAINING DATASET INTO TRAINING AND VALIDATION
 
# Input: "cleaned tweets"
# Target: "class"

from sklearn.model_selection import train_test_split
X_train, X_val, y_train, y_val = train_test_split(train_df["clean_text"],
                                                  train_df["class"],
                                                  test_size=0.05,
                                                  shuffle=True,
                                                  stratify =train_df["class"] )
#import necessary libraries
import nltk
from nltk.tokenize import word_tokenize
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('wordnet')
X_train_tok= [nltk.word_tokenize(i) for i in X_train]  #for word2vec
X_val_tok= [nltk.word_tokenize(i) for i in X_val]      #for word2vec

#TF-IDF
# Convert x_train to vector since model can only run on numbers and not words- Fit and transform
#import TfidfVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vectorizer = TfidfVectorizer(use_idf=True, )
X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train) #tfidf runs on non-tokenized sentences unlike word2vec
# Only transform x_test (not fit and transform)
X_val_vectors_tfidf = tfidf_vectorizer.transform(X_val) #Don't fit() your TfidfVectorizer to your test data: it will 
#change the word-indexes & weights to match test data. Rather, fit on the training data, then use the same train-data-
#fit model on the test data, to reflect the fact you're analyzing the test data only based on what was learned without 
#it, and the have compatible


# + colab={"base_uri": "https://localhost:8080/", "height": 302} id="PJ-FlYSkLtjY" outputId="6a5bba62-4abc-4e72-8bbe-4dc5eee7c966"
test_df.head()


# + id="gIUBcQwsnQXd"
# # Replace classes that are related
# train_df["class"].replace({"famine": "drought", "water": "floods"}, inplace=True)

# + id="R8znjJJrxyky"
from keras.preprocessing.text import Tokenizer
tokenizer = Tokenizer()
tokenizer.fit_on_texts(train_df['clean_text'])
tokenizer.fit_on_texts(test_df['clean_text'])

vocab_size = len(tokenizer.word_index) + 1  # Adding 1 because of reserved 0 index

# + colab={"base_uri": "https://localhost:8080/"} id="thMVCQ4axQcp" outputId="5c2f8386-cc67-4bec-e2dc-14e690ec3ca6"

# split the data into labels and features

X = X_train_vectors_tfidf
y = train_df['class']


#Defining the hyper parameters for the Knearest Neighbors Classifier
leaf_size = list(range(7,20))
n_neighbors = list(range(3,10))
p = [1,2]
metric = ['manhattan', 'euclidean', 'minkowski']
#Creating a dictionary with the hyperparameters
hyperparameters = dict(leaf_size = leaf_size, n_neighbors = n_neighbors, p=p,metric = metric)

#I used GridSearch to look for the best parameters
from sklearn.model_selection import RandomizedSearchCV
from sklearn.neighbors import KNeighborsClassifier
classifier = KNeighborsClassifier()

clf = RandomizedSearchCV(classifier, hyperparameters,cv=10)
clf = clf.fit(X_train_vectors_tfidf, y_train)

print('\n')

print(clf.best_params_)
print('\n')
#Creating a better model based on the parameters given to us by the gridsearch
modelone = KNeighborsClassifier(leaf_size = clf.best_params_['leaf_size'], n_neighbors = clf.best_params_['n_neighbors'], p = clf.best_params_['p'], metric = clf.best_params_['metric']).fit(X_train_vectors_tfidf, y_train)
modelone

# predicting using the model built

y_pred = modelone.predict(X_val_vectors_tfidf)

# comparing the actual and predicted

comparison_frame = pd.DataFrame({'Actual': y_val, 'Predicted': y_pred})
print('\n')
print(comparison_frame)
print('\n')
print(comparison_frame.describe())
# Evaluating the Algorithm
# ---

from sklearn import metrics
print('Mean Absolute Error:', metrics.mean_absolute_error(y_val, y_pred))
print('Mean Squared Error:', metrics.mean_squared_error(y_val, y_pred))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_val, y_pred)))
print('The accuracy of the model is ',metrics.accuracy_score(y_val, y_pred))
print('The F1 score is: ',metrics.f1_score(y_val, y_pred,average='weighted'))
print('The recall score is ',metrics.recall_score(y_val, y_pred,average='weighted'))
print('\n', 'Confusion matrix')
print(confusion_matrix(y_val, y_pred))
print('\n', 'Classification report')
print(classification_report(y_val, y_pred))

# + id="O7uPvymXz0KZ"
#define X variable for fitting of the new model
test_df

X_val = test_df['clean_text']

# + id="5YGCKDya0LB9"
X_val_vectors_tfidf = tfidf_vectorizer.transform(X_val) 

#fit the tfidf model on the test data, to reflect the fact you're analyzing the test data only based on what was learned without 
#it, and the have compatible

# predict using the test dataset
y_pred = modelone.predict(X_val_vectors_tfidf)
y_pred = le.inverse_transform(y_pred)
#Adding the predicted values to our dataset
test_df['class'] = y_pred


# + id="tk0XZDi800IB"
#Lets save our new dataset with class names

# test_df.to_csv('/content/drive/MyDrive/Module 2 groupwork Datasets/Test df with classes KNNclassifier.csv')

# + [markdown] id="9KpZVmzMVt0M"
# # Naive Bayes

# + colab={"base_uri": "https://localhost:8080/"} id="U5QY734eVzRr" outputId="57553b67-ae07-4b3c-83ce-9b35bac66ac9"
from sklearn.naive_bayes import BernoulliNB
from sklearn.naive_bayes import GaussianNB
from sklearn.naive_bayes import MultinomialNB
#SPLITTING THE TRAINING DATASET INTO TRAINING AND VALIDATION
 
# Input: "cleaned tweets"
# Target: "class"

from sklearn.model_selection import train_test_split
X_train, X_val, y_train, y_val = train_test_split(train_df["clean_text"],
                                                  train_df["class"],
                                                  test_size=0.05,
                                                  shuffle=True,
                                                  stratify =train_df["class"] )
#import necessary libraries
import nltk
from nltk.tokenize import word_tokenize
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('wordnet')
X_train_tok= [nltk.word_tokenize(i) for i in X_train]  #for word2vec
X_val_tok= [nltk.word_tokenize(i) for i in X_val]      #for word2vec

#TF-IDF
# Convert x_train to vector since model can only run on numbers and not words- Fit and transform
#import TfidfVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vectorizer = TfidfVectorizer(use_idf=True, )
X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train) #tfidf runs on non-tokenized sentences unlike word2vec
# Only transform x_test (not fit and transform)
X_val_vectors_tfidf = tfidf_vectorizer.transform(X_val) #Don't fit() your TfidfVectorizer to your test data: it will 
#change the word-indexes & weights to match test data. Rather, fit on the training data, then use the same train-data-
#fit model on the test data, to reflect the fact you're analyzing the test data only based on what was learned without 
#it, and the have compatible


# + [markdown] id="t1DjPvdTWB0m"
# Gaussian NB

# + colab={"base_uri": "https://localhost:8080/"} id="tYmx0Pu0V68w" outputId="4a9cb290-c5cd-444e-e15b-faace899814d"
from sklearn.naive_bayes import BernoulliNB
from sklearn.naive_bayes import GaussianNB
from sklearn.naive_bayes import MultinomialNB

# Training and fitting the multinomial model
gausnb = GaussianNB()
gausnb.fit(X_train_vectors_tfidf.toarray(),y_train)
y_pred = gausnb.predict(X_val_vectors_tfidf.toarray())


#Checking performance our model with performance metrics.

comparison_frame = pd.DataFrame({'Actual': y_val, 'Predicted': y_pred})
print('\n')
print(comparison_frame)
print('\n')
print(comparison_frame.describe())
# Evaluating the Algorithm
# ---

from sklearn import metrics
print('Mean Absolute Error:', metrics.mean_absolute_error(y_val, y_pred))
print('Mean Squared Error:', metrics.mean_squared_error(y_val, y_pred))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_val, y_pred)))
print('The accuracy of the model is ',metrics.accuracy_score(y_val, y_pred))
print('The F1 score is: ',metrics.f1_score(y_val, y_pred,average='weighted'))
print('The recall score is ',metrics.recall_score(y_val, y_pred,average='weighted'))
print('\n', 'Confusion matrix')
print(confusion_matrix(y_val, y_pred))
print('\n', 'Classification report')
print(classification_report(y_val, y_pred))

# + [markdown] id="XBjncBXoWNly"
# Bernoulli NB

# + colab={"base_uri": "https://localhost:8080/"} id="4WwuCug1V9Rj" outputId="d0f18d14-c352-4941-a0b9-4c55294c52e4"

# Training and fitting the bernoulli model
bernb = BernoulliNB()
bernb.fit(X_train_vectors_tfidf,y_train)
y_pred = bernb.predict(X_val_vectors_tfidf)


#Checking performance our model with performance metrics.

# comparison_frame = pd.DataFrame({'Actual': y_val, 'Predicted': y_pred})
print('\n')
print(comparison_frame)
print('\n')
print(comparison_frame.describe())
# Evaluating the Algorithm
# ---

from sklearn import metrics
print('Mean Absolute Error:', metrics.mean_absolute_error(y_val, y_pred))
print('Mean Squared Error:', metrics.mean_squared_error(y_val, y_pred))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_val, y_pred)))
print('The accuracy of the model is ',metrics.accuracy_score(y_val, y_pred))
print('The F1 score is: ',metrics.f1_score(y_val, y_pred,average='weighted'))
print('The recall score is ',metrics.recall_score(y_val, y_pred,average='weighted'))
print('\n', 'Confusion matrix')
print(confusion_matrix(y_val, y_pred))
print('\n', 'Classification report')
print(classification_report(y_val, y_pred))

# + [markdown] id="XQwsqPgLWZpF"
# Multinomial NB

# + id="_IJiejZ-WSYo"

from sklearn.pipeline import Pipeline
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.model_selection import train_test_split, GridSearchCV

text_clf = Pipeline([('clf', MultinomialNB())])

tuned_parameters = {
    'vect__ngram_range': [(1, 1), (1, 2), (2, 2)],
    'tfidf__use_idf': (True, False),
    'tfidf__norm': ('l1', 'l2'),
    'clf__alpha': [1, 1e-1, 1e-2]}

# + colab={"base_uri": "https://localhost:8080/"} id="RED2qKmKWWJM" outputId="100d2aef-53f7-4885-849b-aa0cba4ad1c3"
# Training and fitting the multinomial model
mnb = MultinomialNB()
mnb.fit( X_train_vectors_tfidf , y_train)
y_pred = mnb.predict(X_val_vectors_tfidf)

#Checking performance our model with performance metrics.

comparison_frame = pd.DataFrame({'Actual': y_val, 'Predicted': y_pred})
print('\n')
print(comparison_frame)
print('\n')
print(comparison_frame.describe())
# Evaluating the Algorithm
# ---

from sklearn import metrics
print('Mean Absolute Error:', metrics.mean_absolute_error(y_val, y_pred))
print('Mean Squared Error:', metrics.mean_squared_error(y_val, y_pred))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_val, y_pred)))
print('The accuracy of the model is ',metrics.accuracy_score(y_val, y_pred))
print('The F1 score is: ',metrics.f1_score(y_val, y_pred,average='weighted'))
print('The recall score is ',metrics.recall_score(y_val, y_pred,average='weighted'))
print('\n', 'Confusion matrix')
print(confusion_matrix(y_val, y_pred))
print('\n', 'Classification report')
print(classification_report(y_val, y_pred))

# + [markdown] id="DMeFlzNLCLIX"
# Since the accuracy is 60.7%, we will try to improve the model by hyperparameter tuning.

# + [markdown] id="4yMqf6sQWmrw"
# Hyperparameter Tuning

# + colab={"base_uri": "https://localhost:8080/"} id="3v2AZ6GnF_ou" outputId="0c4b69bf-9520-48c9-b52b-d723ce82d2b1"
#Tuning hyperparameters and transforming features to a normal distribution
from sklearn.preprocessing import PowerTransformer
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RepeatedStratifiedKFold

alphas = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
p_grid_NB = {'alpha': alphas}

NB_cls= MultinomialNB()

grid = GridSearchCV(estimator = NB_cls, param_grid = p_grid_NB, cv = 5)
grid.fit(X_train_vectors_tfidf, y_train)
grid.best_params_

# + colab={"base_uri": "https://localhost:8080/"} id="gqo35_H4xkaq" outputId="70ac842a-af8a-433e-b50c-5caf5db78adf"
# Training and fitting the multinomial model with hyperparameter tuned
mnb = MultinomialNB(alpha=0.1)
mnb.fit( X_train_vectors_tfidf , y_train)
y_pred = mnb.predict(X_val_vectors_tfidf)

#Checking performance our model with performance metrics.

comparison_frame = pd.DataFrame({'Actual': y_val, 'Predicted': y_pred})
print('\n')
print(comparison_frame)
print('\n')
print(comparison_frame.describe())
# Evaluating the Algorithm
# ---

from sklearn import metrics
print('Mean Absolute Error:', metrics.mean_absolute_error(y_val, y_pred))
print('Mean Squared Error:', metrics.mean_squared_error(y_val, y_pred))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_val, y_pred)))
print('The accuracy of the model is ',metrics.accuracy_score(y_val, y_pred))
print('The F1 score is: ',metrics.f1_score(y_val, y_pred,average='weighted'))
print('The recall score is ',metrics.recall_score(y_val, y_pred,average='weighted'))
print('\n', 'Confusion matrix')
print(confusion_matrix(y_val, y_pred))
print('\n', 'Classification report')
print(classification_report(y_val, y_pred))

# + [markdown] id="vPfGTOxF6hAr"
# #Random_Forests_classifier_Climate_change_tweet_classification

# + id="XUv_zk9A2MTk"
from sklearn import preprocessing
le = preprocessing.LabelEncoder()
train_df['class'] = le.fit_transform(train_df['class'])

# + colab={"base_uri": "https://localhost:8080/"} id="Nn6ODfb-18z-" outputId="95482a30-5e71-4c62-8ba1-42574902c432"
import pandas as pd
import numpy as np
from google.colab import drive
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import r2_score
#drive.mount('/content/drive')
from sklearn.naive_bayes import BernoulliNB
from sklearn.naive_bayes import GaussianNB
from sklearn.naive_bayes import MultinomialNB
#SPLITTING THE TRAINING DATASET INTO TRAINING AND VALIDATION
 
# Input: "cleaned tweets"
# Target: "class"

from sklearn.model_selection import train_test_split
X_train, X_val, y_train, y_val = train_test_split(train_df["clean_text"],
                                                  train_df["class"],
                                                  test_size=0.05,
                                                  shuffle=True,
                                                  stratify =train_df["class"] )
#import necessary libraries
import nltk
from nltk.tokenize import word_tokenize
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('wordnet')
X_train_tok= [nltk.word_tokenize(i) for i in X_train]  #for word2vec
X_val_tok= [nltk.word_tokenize(i) for i in X_val]      #for word2vec

#TF-IDF
# Convert x_train to vector since model can only run on numbers and not words- Fit and transform
#import TfidfVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vectorizer = TfidfVectorizer(use_idf=True, )
X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train) #tfidf runs on non-tokenized sentences unlike word2vec
# Only transform x_test (not fit and transform)
X_val_vectors_tfidf = tfidf_vectorizer.transform(X_val) #Don't fit() your TfidfVectorizer to your test data: it will 
#change the word-indexes & weights to match test data. Rather, fit on the training data, then use the same train-data-
#fit model on the test data, to reflect the fact you're analyzing the test data only based on what was learned without 
#it, and the have compatible


# + colab={"base_uri": "https://localhost:8080/", "height": 1000} id="M8upO2y-2pRH" outputId="dc48507c-78fd-412c-d611-db256faa2ec5"
# split the data into labels and features

# Creating a dictionary of parameters to tune
#
parameters = {'n_estimators': np.arange(10,13),
         'max_depth': np.arange(7,17)}

# Setting the number of folds to 5 and instantiating the model
# 

grid_search = RandomizedSearchCV(RandomForestClassifier(), parameters, cv = 5, return_train_score = True)

grid_search.fit(X_train_vectors_tfidf, y_train)
print(grid_search.best_params_)
print('\n')

#Lets see how the different max depth values compare to each other
print('how do the different depths compare to each other''\n')
for i in range(len(parameters['max_depth'])):
  print('parameters', grid_search.cv_results_['params'][i])
  print('mean Test scores:', grid_search.cv_results_['mean_test_score'][i])
  print('Rank:', grid_search.cv_results_['rank_test_score'][i])


print('\n')

dtree_model = RandomForestClassifier(n_estimators = grid_search.best_params_['n_estimators'], max_depth = grid_search.best_params_['max_depth']).fit(X_train_vectors_tfidf, y_train)
dtree_model
print('Training score:', dtree_model.score(X_train_vectors_tfidf, y_train))
print('Test score:', r2_score(y_val, y_pred))
y_pred = dtree_model.predict(X_train_vectors_tfidf)
#Random Forests model
comparison_frame = pd.DataFrame({'Actual': y_val, 'Predicted': y_pred})
print('\n')
print(comparison_frame)
print('\n')
print(comparison_frame.describe())

# Evaluating the Algorithm
# ---

from sklearn import metrics
print('Mean Absolute Error:', metrics.mean_absolute_error(y_val, y_pred))
print('Mean Squared Error:', metrics.mean_squared_error(y_val, y_pred))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_val, y_pred)))
print('The accuracy of the model is ',metrics.accuracy_score(y_val, y_pred))
print('\n', 'Confusion matrix')
print(confusion_matrix(y_val, y_pred))
print('\n', 'Classification report')

# + id="Dg6Mh5BT_txE"
from sklearn import metrics
# print('Mean Absolute Error:', metrics.mean_absolute_error(y_val, y_pred))
# print('Mean Squared Error:', metrics.mean_squared_error(y_val, y_pred))
# print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_val, y_pred)))
print('The accuracy of the model is ',metrics.accuracy_score(y_val, y_pred))
print('\n', 'Confusion matrix')
print(confusion_matrix(y_val, y_pred))
print('\n', 'Classification report')

# + id="4dZ13qqh37Ti"
#define X variable for fitting of the new model
test_df

X_val = test_df['clean_text']

# + id="jHQhA5AC3-8y"
X_val_vectors_tfidf = tfidf_vectorizer.transform(X_val) 

#fit the tfidf model on the test data, to reflect the fact you're analyzing the test data only based on what was learned without 
#it, and the have compatible

# predict using the test dataset
y_pred = dtree_model.predict(X_val_vectors_tfidf)
y_pred = le.inverse_transform(y_pred)
#Adding the predicted values to our dataset
test_df['class'] = y_pred


# + id="W_NKVm-w4Bgs"
#Lets save our new dataset with class names

test_df.to_csv('/content/drive/MyDrive/Module 2 groupwork Datasets/Test df with classes RandomForestsclassifier.csv')

# + [markdown] id="ZGJsEVYYVxg-"
# #Logistic Regression

# + id="Ygc_yGVqwEtd"
import pandas as pd
import numpy as np
import seaborn as sns
from sklearn.metrics import r2_score
import sklearn
import matplotlib.pyplot as plt
# %matplotlib inline
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score

# + id="KrXKd8EcxXX-"
train_df = pd.read_csv('/content/Clean classified data_kwisha.csv')
test_df = pd.read_csv('Test (2).csv')

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} id="7Y1bTKeSyeDl" outputId="45c33338-3a09-4140-f8db-eb2d0b4d40d5"
sns.pairplot(train_df, hue='class', size=2.5)

# + id="3usf__B-0nXM"
from sklearn import preprocessing
le = preprocessing.LabelEncoder()
train_df['class'] = le.fit_transform(train_df['class'])

# + id="LqlAFR96555m"
from sklearn.model_selection import train_test_split
X_train, X_val, y_train, y_val = train_test_split(train_df["clean_text"],
                                                  train_df["class"],
                                                  test_size=0.05,
                                                  shuffle=True,
                                                  stratify =train_df["class"] )

# + colab={"base_uri": "https://localhost:8080/"} id="1fYpdO_15_3H" outputId="489cff0c-ba40-4840-d5e2-7aaeb1361925"
import nltk
from nltk.tokenize import word_tokenize
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('wordnet')
X_train_tok= [nltk.word_tokenize(i) for i in X_train]  
X_val_tok= [nltk.word_tokenize(i) for i in X_val]      


# + id="7TWXnPjm6H8E"
# Convert x_train to vector since model can only run on numbers and not words- Fit and transform
#import TfidfVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vectorizer = TfidfVectorizer(use_idf=True, )
X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train) #tfidf runs on non-tokenized sentences unlike word2vec
# Only transform x_test (not fit and transform)
X_val_vectors_tfidf = tfidf_vectorizer.transform(X_val) #Don't fit() your TfidfVectorizer to your test data: it will 
#change the word-indexes & weights to match test data. Rather, fit on the training data, then use the same train-data-
#fit model on the test data, to reflect the fact you're analyzing the test data only based on what was learned without 
#it, and the have compatibl

# + colab={"base_uri": "https://localhost:8080/"} id="AzlsQo7CT5Gm" outputId="e779657d-4ed5-45a0-a5f0-77a4f8a960e3"
from sklearn.metrics import classification_report, f1_score, accuracy_score, confusion_matrix
from sklearn.linear_model import LogisticRegression
lr_tfidf=LogisticRegression(solver = 'liblinear', C=10, penalty = 'l2')
lr_tfidf.fit(X_train_vectors_tfidf, y_train)  #model

#Predict y value for test dataset
y_predict = lr_tfidf.predict(X_val_vectors_tfidf)
y_prob = lr_tfidf.predict_proba(X_val_vectors_tfidf)[:,1]

 
baselog_accuracy = accuracy_score(y_val, y_predict)

from sklearn import metrics
print('Mean Absolute Error:', metrics.mean_absolute_error(y_val, y_predict))
print('Mean Squared Error:', metrics.mean_squared_error(y_val, y_predict))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_val, y_predict)))
print('The accuracy of the model is ',metrics.accuracy_score(y_val, y_predict))
print('The F1 score is: ',metrics.f1_score(y_val, y_predict,average='weighted'))
print('The recall score is ',metrics.recall_score(y_val, y_predict,average='weighted'))
print('\n', 'Confusion matrix')
print(confusion_matrix(y_val, y_predict))
print('\n', 'Classification report')
print(classification_report(y_val, y_predict))

# + id="0PE5Syc7Csh2"
from six import StringIO

# + colab={"base_uri": "https://localhost:8080/"} id="rRCiIfFHUhyG" outputId="e24eeeb8-dc11-4e83-816d-21a67eae43f6"
# Import the libraries
from sklearn.datasets import make_classification
from sklearn.datasets import make_classification
from imblearn.over_sampling import SVMSMOTE 
lr_tfidf = SVMSMOTE(random_state = 101)

# Choosing a sample
X_oversample_svm, y_oversample_svm = make_classification(n_samples=10000, n_features=2,
                                                         n_redundant=0, n_clusters_per_class=1,
                                                         weights=[0.99], flip_y=0, random_state=101)

# Perform Logistic Regression
X_oversample_svm, y_oversample_svm = lr_tfidf.fit_resample(X_train_vectors_tfidf, y_train)
classifier_svm = LogisticRegression(solver = 'liblinear', C=10, penalty = 'l2')
classifier_svm.fit(X_oversample_svm, y_oversample_svm)

#Predict y value for test dataset
y_predict = classifier_svm.predict(X_val_vectors_tfidf)
y_prob = classifier_svm.predict_proba(X_val_vectors_tfidf)[:,1]
 


from sklearn import metrics
print('Mean Absolute Error:', metrics.mean_absolute_error(y_val, y_predict))
print('Mean Squared Error:', metrics.mean_squared_error(y_val, y_predict))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_val, y_predict)))
print('The accuracy of the model is ',metrics.accuracy_score(y_val, y_predict))
print('The F1 score is: ',metrics.f1_score(y_val, y_predict,average='weighted'))
print('The recall score is ',metrics.recall_score(y_val, y_predict,average='weighted'))
print('\n', 'Confusion matrix')
print(confusion_matrix(y_val, y_predict))
print('\n', 'Classification report')
print(classification_report(y_val, y_predict))
#print(classification_report(y_test, classifier_svm.predict(X_test)))
accuracy_score(y_val, y_predict)

# + [markdown] id="CIX5-KBJfA_q"
# ## Transformers

# + [markdown] id="4b0uYrN1WSxd"
# ## Transformers

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} id="m3FDSNzqhG3o" outputId="d45e349f-7a0c-490d-8942-fb80aa64cd1a"
# ! pip install ktrain

# + id="s6UDhSmqhQ8Z"
# %reload_ext autoreload
# %autoreload 2
# %matplotlib inline
import os
os.environ['CUDA_DEVICE_ORDER']='PCI_BUS_ID';
os.environ['CUDA_VISIBLE_DEVICES']='0';

# + id="HD-8mBL_V-Tm"
import pandas as pd
import ktrain
from ktrain import text

df=pd.read_csv('/content/Clean classified data_kwisha.csv')

# + id="FM6K6k4FV-Tn"
from sklearn import preprocessing
le = preprocessing.LabelEncoder()
df['class'] = le.fit_transform(df['class'])

# + id="s_7k5V8dV-To"
data_texts = df["clean_text"].to_list() # Features (not-tokenized yet)
data_labels = df["class"].to_list() # Lables
from sklearn.model_selection import train_test_split

# # Split Train and Validation data
# train_texts, _texts, train_labels, val_labels = train_test_split(data_texts, data_labels, test_size=0.2, random_state=0)
X_train,X_test,y_train,y_test=train_test_split(data_texts, data_labels, test_size=0.2, random_state=0)
# # Keep some data for inference (testing)
# train_texts, test_texts, train_labels, test_labels = train_test_split(train_texts, train_labels, test_size=0.01, random_state=0)

# + colab={"base_uri": "https://localhost:8080/"} id="j6AP5IMofXa-" outputId="5bdc4be7-6193-4173-c75c-38ab981135b7"
len(X_test),len(X_train),len(y_test),len(y_train)

# + [markdown] id="d_t0FTongK9A"
# Build model with Transformer

# + colab={"base_uri": "https://localhost:8080/", "height": 48, "referenced_widgets": ["53c85db8698b4a988155aaf0075f200f", "3e1c435758194733b3f82b574723d212", "ebcdbe00229244c48ab2d5b1be31e030", "a97ded6d7e674d0bac081fceef1f3a84", "7e3b9e7586d141bf8143db2fa901ea0b", "ed2d2b0c74a2435786ed5624740d9594", "d88b524dc5a047fca69ac47b59a8e8b6", "1f2d887e405844be8a4b8da37741b45a", "ce0e8b81725946d7a3e381deb20a354f", "ea7ce50ab6484cb5b155692acf035364", "c893cf781bee487c9dbcfccc7b6d303d"]} id="jpwB6uWwgQ0F" outputId="8553c41d-1a97-47d8-dba0-a34bee2a835c"
# Build the model with pretrained model
model_name='distilbert-base-uncased'
categories=['renewable','drought','floods','air_polutants','temperature','greentalk']
#transformer
trans=text.Transformer(model_name,maxlen=512,class_names=categories)

# + colab={"base_uri": "https://localhost:8080/", "height": 330, "referenced_widgets": ["44990b5bb5c24aa9a11fc07e55e28ba9", "6c6257fd4a0140e7ac7133fabe3fa092", "f3a5ede38ccb4b3b8fb83d30e4acbdd3", "1e293e11f86d40c791fdab7545722bb5", "8d1022bc697749e982b6189d658707a6", "bb6f4945e1844972b22bd05c9334fd64", "3f0bd4cef66f4f2193c23dc6ed312cd9", "19b6e5964a9543e7a2c22ed53f0a826e", "<KEY>", "<KEY>", "<KEY>", "5afcc8964e0045cf8f87ecc53446ac2d", "d001a01282ca4dc9a7cb108f37a7f31b", "38cd23dcdff84c1c9c6ee1c3873a12b0", "<KEY>", "6ce8cd49be7b4e35b12c964c1460795f", "254c4d3fe5fc44eb8699a3c35a1f544e", "<KEY>", "c280131056224e25bb9b3eb0dc7ad447", "0518d45f83af4ab3b49448998465595a", "3867ee7d58364b7e8f2ee0799a5fab90", "0089132b65104968a852d80c0689ae0f"]} id="m4m3AB3oiUWD" outputId="e54a6230-0606-4f4b-b17e-78fe78f029a1"
# Get the train and test data
train_data=trans.preprocess_train(X_train,y_train)
test_data=trans.preprocess_train(X_test,y_test)

# + colab={"base_uri": "https://localhost:8080/", "height": 48, "referenced_widgets": ["5916e97f5b124d0d846fdde042adb433", "4e68b32962f847e79cf9680436da7978", "1392cb14660941b4be727903f8f42a36", "023c56ea7da94c8a824edc5272e95e89", "d9e924a78fec46a79704650615081a33", "<KEY>", "0a86c2cd1bdf461890b0eca8a5294541", "e864ce49e8984b44a7472834f7eb4162", "db6aecc462a14218873a857ac30596ec", "98a1ce45ecc04240aefabf7dce41f84d", "1c2209b1fdeb42e28a5930645a7504b2"]} id="YKuZaixPjGYu" outputId="bab2a43f-ed00-40e2-adf7-1a1c807aa122"
# We retrive the transformer distilbert classifier
model=trans.get_classifier()

# + id="U3v7u2ohjRTk"
# Get the learner
learner=ktrain.get_learner(model,train_data=train_data,val_data=test_data,batch_size=16)

# + id="hDaRBEIJkFkv"
# Train the model
learner.lr_find(show_plot=True,max_epochs=3) # we us ethe code to find the best learning rate

# + colab={"base_uri": "https://localhost:8080/"} id="VVAbXA2xkEJT" outputId="17f293d1-e77a-4945-9ba3-e42b5c564925"
learner.fit_onecycle(0.003,1)

# + colab={"base_uri": "https://localhost:8080/"} id="3jvCSIq7lnfB" outputId="789cdbf2-d2c2-403e-936e-f339f6beafc9"
# Test the metrics
learner.validate(class_names=categories)

# + id="ZB7F__lAmnTD" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="2d949859-d440-4f33-8e31-69f85b805116"
X_test[553]

# + [markdown] id="HBx4IzOKWCvO"
# Making predictions

# + id="ztwcAMyPmIzi" colab={"base_uri": "https://localhost:8080/", "height": 167} outputId="be531be5-af80-4f1b-c27e-f85f18531794"
predictor=ktrain.get_predictor(learner.model,prepoc=trans)

# + id="UVSVr4L9mzZS"
x='Six bodies of the flash flood victims have been recovered, leaving one tourist missing. The search and rescue operation continues as we reach out to next of kin to share details of (this) sad incident and plan together (our) next course of action.”'


# + id="jeTutAThm1_G"
# We predict the random sentence 
predictor.predict(x)

# + id="rCVgw7AyV-T6"
from wandb.keras import WandbCallback

StarcoderdataJupyterScriptsDedupFiltered