# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: 'Python 3.8.8 64-bit (''base'': conda)'
#     language: python
#     name: python3
# ---

# # Import Libraries

import json
import pandas as pd
import emoji
import re
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import classification_report


# # Retrieve Data

f1 = open('tweets.json')
data_tweets = json.load(f1).items()
user_label = pd.read_csv('labeled_users_clean.csv')


# Rename column for merging
user_label.rename(columns={"screen_name": "id"}, inplace=True)
print(user_label.head())
print(len(user_label))


data_tweets = pd.DataFrame.from_dict(data_tweets)
data_tweets.rename(columns={0: "id", 1: "tweets"}, inplace=True)
print(data_tweets[:5])
print(len(data_tweets))


# CSV data after cleaning and selection
alldata = pd.merge(data_tweets, user_label, on='id')
print(len(alldata))

# Extract 3 useful columns ID, tweets, age
idAndTweets = alldata[["id","tweets","age"]]

# # Clean data - remove mentions, urls, hashtags, emoji, punctuations, special chars, stop words

# +
#remove mentions, urls, hashtags.
regexMap = {r"@[\w]+": "", r"http[\S]+": "", r"#[\w]+": ""}

def cleaning(data):
    t = data
    for regx in regexMap.keys():
        t = re.sub(regx, regexMap[regx], str(t))
    return t

#remove emojis
def deEmojify(data):
    t = data
    return emoji.get_emoji_regexp().sub('', str(t))


idAndTweets["tweets"] = idAndTweets["tweets"].apply(cleaning)
idAndTweets["tweets"] = idAndTweets["tweets"].apply(deEmojify)
idAndTweets.head()

# -

# removing punctuations, numbers, and special characters
idAndTweets['tweets'] = idAndTweets['tweets'].str.replace("[^a-zA-Z#]", " ")
idAndTweets.head(10)


#remove stop words
import nltk
from nltk.corpus import stopwords
nltk.download('stopwords')
stop_words = set(stopwords.words('english'))
idAndTweets['tweets'] = idAndTweets['tweets'].apply(
    lambda x: ' '.join([w for w in x.split() if w not in stop_words]))
idAndTweets.head()


print(idAndTweets.head())
print(idAndTweets.size)

# # TF-IDF vectorizer & Logistic Regression, Oversampling, 5-fold cross validation

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from imblearn.over_sampling import SMOTE
tfidf = TfidfVectorizer(max_features=7500, ngram_range=(1, 2))
X = tfidf.fit_transform(idAndTweets['tweets'])
y = idAndTweets['age']
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=1/4.0, random_state=0)
oversample = SMOTE()
X_train, y_train = oversample.fit_resample(X_train, y_train)


# +
# K-fold (5-fold)
from sklearn.model_selection import KFold
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from imblearn.over_sampling import SMOTE
tfidf = TfidfVectorizer(max_features=7500, ngram_range=(1, 2))

X = tfidf.fit_transform(idAndTweets['tweets'])
y = idAndTweets['age']

kf = KFold(n_splits=5)
kf.get_n_splits(X)

for train_index, test_index in kf.split(X):
    #print("TRAIN:", train_index, "TEST:", test_index)
    X_train, X_test = X[train_index], X[test_index]
    y_train, y_test = y[train_index], y[test_index]

# Oversampling
oversample = SMOTE()
X_train, y_train = oversample.fit_resample(X_train, y_train)

# -

# Logistic Regression
from sklearn.linear_model import LogisticRegression
log = LogisticRegression(multi_class='multinomial', solver='lbfgs').fit(X_train, y_train)
y_pred = log.predict(X_test)


print(classification_report(y_test, y_pred))


# # Complement Naive Bayes

# +
import nltk
from nltk.tokenize import RegexpTokenizer

training_dataset = idAndTweets.sample(frac=0.7, ignore_index=True)
training_dataset.head()
test_dataset = idAndTweets.drop(training_dataset.index)

token = RegexpTokenizer(r'[a-zA-Z0-9]+')
vec = CountVectorizer(stop_words='english', ngram_range=(1, 3), tokenizer=token. tokenize)

x_train = vec.fit_transform(training_dataset['tweets'].values)
y_train = training_dataset['age']
x_test = vec.transform(test_dataset['tweets'])
y_test= test_dataset['age']

# -

from sklearn.naive_bayes import ComplementNB
model = ComplementNB()
model.fit(x_train, y_train)


model.score(x_train,y_train)

model.score(x_test,y_test)

y_prediction = model.predict(x_test)
print(classification_report(y_test,y_prediction))

StarcoderdataJupyterScriptsDedupFiltered