from loguru import logger
import pandas as pd
import json
from datetime import datetime
import ast
import numpy as np
from pymongo import MongoClient
from collections import defaultdict
from tqdm import tqdm
import time
import requests
import json
import os
import pandas as pd
import nltk
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
from textblob import TextBlob
import re
from transformers import BertTokenizer, BertModel
from transformers import RobertaTokenizer, RobertaModel
import torch
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np
# Download NLTK resources
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('stopwords')
nltk.download('punkt_tab')
nltk.download('averaged_perceptron_tagger_eng')
class Preprocessor:
def __init__(self,df):
self.df=df
self.tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
self.model = RobertaModel.from_pretrained('roberta-base')
self.stop_words = set(stopwords.words('english'))
self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # Add this line
def get_bert_embedding(self, text):
inputs = self.tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=512)
with torch.no_grad():
outputs = self.model(**inputs)
return outputs.last_hidden_state.mean(dim=1).squeeze().numpy()
def preprocess_text(self,text):
return text if pd.notna(text) else ""
def calculate_duration(self, time_range):
if not isinstance(time_range, str) or "-" not in time_range:
return None
start_str, end_str = time_range.split('-')
start_str = start_str.strip() + ':00' if len(start_str.split(':')) == 1 else start_str.strip()
end_str = end_str.strip() + ':00' if len(end_str.split(':')) == 1 else end_str.strip()
try:
start = datetime.strptime(start_str, '%H:%M')
end = datetime.strptime(end_str, '%H:%M')
duration = (end - start).total_seconds() / 3600
return duration if duration >= 0 else duration + 24
except ValueError:
return None
def calculate_sentiment_severity(self, text):
if pd.isna(text) or not text.strip():
return pd.Series({"good_severity": 0.0, "bad_severity": 0.0})
# Get sentiment polarity (-1 to 1)
blob = TextBlob(text)
polarity = blob.sentiment.polarity
# Define severity weights
good_weight = 0.7
bad_weight = 0.3
if polarity > 0:
good_severity = good_weight * polarity
bad_severity = 0.0
elif polarity < 0:
good_severity = 0.0
bad_severity = bad_weight * abs(polarity)
else: # Neutral (polarity = 0)
good_severity = 0.0
bad_severity = 0.0
return pd.Series({"good_severity": good_severity, "bad_severity": bad_severity})
def get_avg_duration(self, hours_str):
if pd.isna(hours_str) or not isinstance(hours_str, str):
return pd.NA
try:
hours_dict = ast.literal_eval(hours_str)
if not hours_dict:
return pd.NA
durations = [self.calculate_duration(time_range) for time_range in hours_dict.values()]
valid_durations = [d for d in durations if d is not None]
return sum(valid_durations) / len(valid_durations) if valid_durations else pd.NA
except (ValueError, SyntaxError, ZeroDivisionError):
return pd.NA
def calculate_time_since_last_review(self):
present_date = datetime.now()
user_latest_timestamp = {}
# Convert review_date to datetime
self.df["review_date"] = pd.to_datetime(self.df["review_date"])
# Calculate hours difference for each user's latest review
for user_id in self.df["user_id"].unique():
latest_date = self.df[self.df["user_id"] == user_id]["review_date"].max()
if not isinstance(latest_date, datetime):
latest_date = latest_date.to_pydatetime()
hours_difference = (present_date - latest_date).total_seconds() / 3600
user_latest_timestamp[user_id] = hours_difference
# Map the hours difference to a new column
self.df["time_since_last_review_user"] = self.df["user_id"].map(user_latest_timestamp)
def calculate_time_since_last_review_business(self):
present_date = datetime.now()
# Ensure review_date is in datetime format
self.df["review_date"] = pd.to_datetime(self.df["review_date"])
# Initialize dictionary to store hours since last review for each business
business_latest_timestamp = {}
# Iterate over unique business_ids
for business_id in self.df["business_id"].unique():
# Get the latest review date for this business
latest_date = self.df[self.df["business_id"] == business_id]["review_date"].max()
# Convert to datetime object if needed
if not isinstance(latest_date, datetime):
latest_date = latest_date.to_pydatetime()
# Calculate hours difference (already in hours)
hours_difference = (present_date - latest_date).total_seconds() / 3600
business_latest_timestamp[business_id] = hours_difference
# Map the hours difference to the new column
self.df["time_since_last_review_business"] = self.df["business_id"].map(business_latest_timestamp)
def calculate_user_account_age(self):
present_date = datetime.now()
# Convert yelping_since to datetime
self.df["yelping_since"] = pd.to_datetime(self.df["yelping_since"])
# Calculate user account age in days
self.df["user_account_age"] = (present_date - self.df["yelping_since"]).dt.days
def calculate_avg_time_between_reviews(self):
# Ensure review_date is in datetime format
self.df["review_date"] = pd.to_datetime(self.df["review_date"])
# Sort the DataFrame by user_id and review_date to ensure chronological order
self.df = self.df.sort_values(["user_id", "review_date"])
# Define helper function to calculate average time between reviews
def calculate_avg_time(group):
if len(group) == 1:
return 0 # If only one review, assign 0
# Calculate differences in hours between consecutive reviews
diffs = group["review_date"].diff().dt.total_seconds() / 3600
# Drop the first NaN (from diff) and compute the mean
return diffs.dropna().mean()
# Apply the function to each user_id group and create a mapping
avg_time_per_user = self.df.groupby("user_id").apply(calculate_avg_time)
# Map the average time back to the original DataFrame
self.df["average_time_between_reviews"] = self.df["user_id"].map(avg_time_per_user)
def calculate_user_degree(self):
# Calculate the number of unique businesses per user
user_business_counts = self.df.groupby("user_id")["business_id"].nunique()
# Map the counts back to the original DataFrame
self.df["user_degree"] = self.df["user_id"].map(user_business_counts)
def calculate_business_degree(self):
# Calculate the number of unique users per business
business_user_counts = self.df.groupby("business_id")["user_id"].nunique()
# Map the counts back to the original DataFrame
self.df["business_degree"] = self.df["business_id"].map(business_user_counts)
def calculate_rating_variance_user(self):
# Calculate the mode (most frequent rating) per user
user_rating_mode = self.df.groupby("user_id")["review_stars"].agg(lambda x: x.mode()[0])
# Map the most frequent rating back to the original DataFrame
self.df["rating_variance_user"] = self.df["user_id"].map(user_rating_mode)
def calculate_user_review_burst_count(self):
# Ensure review_date is in datetime format
self.df["review_date"] = pd.to_datetime(self.df["review_date"])
# Sort by user_id and review_date for chronological order
self.df = self.df.sort_values(["user_id", "review_date"])
# Function to calculate the max number of reviews in any 20-day window
def calculate_burst_count(group):
if len(group) <= 1:
return 0 # No burst if 1 or fewer reviews
# Convert review_date to a Series for rolling window
dates = group["review_date"]
# Calculate the number of reviews within 20 days of each review
burst_counts = []
for i, date in enumerate(dates):
# Count reviews within 20 days after this date
window_end = date + pd.Timedelta(days=20)
count = ((dates >= date) & (dates <= window_end)).sum()
burst_counts.append(count)
# Return the maximum burst count for this user
return max(burst_counts)
# Calculate the burst count per user
user_burst_counts = self.df.groupby("user_id").apply(calculate_burst_count)
# Map the burst count back to the original DataFrame
self.df["user_review_burst_count"] = self.df["user_id"].map(user_burst_counts)
def calculate_business_review_burst_count(self):
# Ensure review_date is in datetime format
self.df["review_date"] = pd.to_datetime(self.df["review_date"])
# Sort by business_id and review_date for chronological order
self.df = self.df.sort_values(["business_id", "review_date"])
# Function to calculate the max number of reviews in any 10-day window
def calculate_burst_count(group):
if len(group) <= 1:
return 0 # No burst if 1 or fewer reviews
# Convert review_date to a Series for rolling window
dates = group["review_date"]
# Calculate the number of reviews within 10 days of each review
burst_counts = []
for i, date in enumerate(dates):
# Count reviews within 10 days after this date
window_end = date + pd.Timedelta(days=10)
count = ((dates >= date) & (dates <= window_end)).sum()
burst_counts.append(count)
# Return the maximum burst count for this business
return max(burst_counts)
# Calculate the burst count per business
business_burst_counts = self.df.groupby("business_id").apply(calculate_burst_count)
# Map the burst count back to the original DataFrame
self.df["business_review_burst_count"] = self.df["business_id"].map(business_burst_counts)
def calculate_temporal_similarity(self):
self.df["review_date"] = pd.to_datetime(self.df["review_date"])
# Extract the day of the week (0 = Monday, 6 = Sunday)
self.df["day_of_week"] = self.df["review_date"].dt.dayofweek
# Function to calculate avg hours between reviews on frequent days
def calculate_avg_hours_on_frequent_days(group):
frequent_days = group["day_of_week"].mode().tolist()
if len(group) <= 1:
return 0
frequent_reviews = group[group["day_of_week"].isin(frequent_days)]
if len(frequent_reviews) <= 1:
return 0
frequent_reviews = frequent_reviews.sort_values("review_date")
diffs = frequent_reviews["review_date"].diff().dt.total_seconds() / 3600
return diffs.dropna().mean()
# Calculate average hours for each user
avg_hours_per_user = self.df.groupby("user_id").apply(calculate_avg_hours_on_frequent_days)
# Map the average hours to the new column
self.df["temporal_similarity"] = self.df["user_id"].map(avg_hours_per_user)
# Drop temporary column
self.df = self.df.drop(columns=["day_of_week"])
def calculate_rating_deviation_from_business_average(self):
# Calculate the average rating per business
business_avg_rating = self.df.groupby("business_id")["review_stars"].mean()
# Map the average rating to each row
self.df["business_avg_rating"] = self.df["business_id"].map(business_avg_rating)
# Calculate the deviation from the business average
self.df["rating_deviation_from_business_average"] = (
self.df["review_stars"] - self.df["business_avg_rating"]
)
# Drop the temporary column
self.df = self.df.drop(columns=["business_avg_rating"])
def calculate_review_like_ratio(self):
# Create a binary column for liked reviews (stars >= 4)
self.df["is_liked"] = (self.df["review_stars"] >= 4).astype(int)
# Calculate the like ratio per user
user_like_ratio = self.df.groupby("user_id")["is_liked"].mean()
# Map the like ratio back to the DataFrame
self.df["review_like_ratio"] = self.df["user_id"].map(user_like_ratio)
# Drop the temporary column
self.df = self.df.drop(columns=["is_liked"])
def calculate_latest_checkin_hours(self):
self.df["yelping_since"] = pd.to_datetime(self.df["yelping_since"])
# Function to get the latest check-in date from a list of strings
def get_latest_checkin(checkin_list):
if not checkin_list or pd.isna(checkin_list): # Handle empty or NaN
return None
if isinstance(checkin_list, str):
checkin_dates = checkin_list.split(", ")
else:
checkin_dates = checkin_list
return pd.to_datetime(checkin_dates).max()
# Apply the function to get the latest check-in date per row
self.df["latest_checkin_date"] = self.df["checkin_date"].apply(get_latest_checkin)
# Calculate the hours difference between latest check-in and yelping_since
self.df["latest_checkin_hours"] = (
(self.df["latest_checkin_date"] - self.df["yelping_since"])
.dt.total_seconds() / 3600
)
# Drop the temporary column
self.df = self.df.drop(columns=["latest_checkin_date"])
self.df["latest_checkin_hours"].fillna(0,inplace=True)
def compute_pronoun_density(self, text):
text = self.preprocess_text(text)
if not text:
return 0
words = word_tokenize(text.lower())
pos_tags = nltk.pos_tag(words)
pronouns = sum(1 for word, pos in pos_tags if pos in ['PRP', 'PRP$'] and word in ['i', 'we'])
return pronouns / len(words) if words else 0
def compute_avg_sentence_length(self, text):
text = self.preprocess_text(text)
if not text:
return 0
sentences = sent_tokenize(text)
return sum(len(word_tokenize(sent)) for sent in sentences) / len(sentences) if sentences else 0
def compute_excessive_punctuation(self, text):
text = self.preprocess_text(text)
return len(re.findall(r'[!?.]{2,}', text))
def compute_sentiment_polarity(self, text):
text = self.preprocess_text(text)
return TextBlob(text).sentiment.polarity if text else 0
def compute_code_switching_flag(self, text):
text = self.preprocess_text(text)
if not text:
return 0
tokens = self.tokenizer.tokenize(text.lower())
if not tokens:
return 0
english_words = self.stop_words # Use self.stop_words from __init__
token_set = set(tokens)
english_count = sum(1 for token in tokens if token in english_words)
non_english_pattern = re.compile(r'[^\x00-\x7F]')
has_non_ascii = 1 if non_english_pattern.search(text) else 0
english_ratio = english_count / len(tokens) if tokens else 0
non_english_tokens = sum(1 for token in token_set if token not in english_words and "##" in token and has_non_ascii)
# Flag as code-switching if:
# 1. Mixed English presence (ratio between 0.1 and 0.9)
# 2. Non-ASCII characters present OR some non-English subword tokens
if 0.1 < english_ratio < 0.9 and (has_non_ascii or non_english_tokens > 0):
return 1
return 0
def batch_tokenize(self, texts, batch_size=32, max_length=512):
tokenized_outputs = []
for i in tqdm(range(0, len(texts), batch_size), desc="Tokenizing with RoBERTa on GPU"):
batch_texts = texts[i:i + batch_size]
valid_texts = [self.preprocess_text(t) for t in batch_texts]
# Tokenize with fixed max_length to ensure consistent tensor sizes
inputs = self.tokenizer(valid_texts, return_tensors='pt', truncation=True, padding='max_length', max_length=max_length)
tokenized_outputs.append(inputs['input_ids'].to(self.device)) # Move to GPU
# Concatenate on GPU with consistent sizes
return torch.cat(tokenized_outputs, dim=0)
def compute_grammar_error_score(self, texts, tokenized_ids):
print("Computing grammar error scores...")
error_scores = np.zeros(len(texts), dtype=float)
vocab_set = set(self.tokenizer.get_vocab().keys())
for i, input_ids in enumerate(tqdm(tokenized_ids, desc="Processing Grammar Errors")):
if input_ids.sum() == 0: # Empty input
continue
tokens = self.tokenizer.convert_ids_to_tokens(input_ids.cpu().tolist(), skip_special_tokens=True)
unknown_count = sum(1 for token in tokens if token not in vocab_set and token not in self.stop_words)
total_count = len([t for t in tokens if t not in self.stop_words])
error_scores[i] = unknown_count / total_count if total_count > 0 else 0
return error_scores
def compute_repetitive_words_count(self, texts, tokenized_ids):
print("Computing repetitive words counts...")
rep_counts = np.zeros(len(texts), dtype=int)
for i, input_ids in enumerate(tqdm(tokenized_ids, desc="Processing Repetition")):
if input_ids.sum() == 0: # Empty input
continue
tokens = self.tokenizer.convert_ids_to_tokens(input_ids.cpu().tolist(), skip_special_tokens=True)
valid_tokens = [t for t in tokens if t not in self.stop_words and len(t) > 2]
if valid_tokens:
token_counts = {}
for token in valid_tokens:
token_counts[token] = token_counts.get(token, 0) + 1
rep_counts[i] = sum(1 for count in token_counts.values() if count > 1)
return rep_counts
def preprocess_text_for_similarity(self, text):
if pd.isna(text) or not text.strip():
return []
return [w for w in word_tokenize(str(text).lower()) if w not in self.stop_words]
def batch_encode_words(self, texts, batch_size=32, max_length=512):
word_lists = [self.preprocess_text_for_similarity(t) for t in tqdm(texts, desc="Tokenizing Texts")]
vocab = {word: idx + 1 for idx, word in enumerate(set.union(*[set(w) for w in word_lists if w]))}
encoded_batches = []
for i in tqdm(range(0, len(word_lists), batch_size), desc="Encoding Words on GPU"):
batch_words = word_lists[i:i + batch_size]
encoded = np.zeros((len(batch_words), max_length), dtype=np.int64)
for j, words in enumerate(batch_words):
if words:
word_ids = [vocab.get(w, 0) for w in words][:max_length]
encoded[j, :len(word_ids)] = word_ids
encoded_tensor = torch.tensor(encoded, dtype=torch.int64).to(self.device)
encoded_batches.append(encoded_tensor)
return torch.cat(encoded_batches, dim=0), vocab
def compute_similarity_to_other_reviews(self, batch_size=32, max_length=512):
all_texts = self.df["review_text"].tolist()
all_users = self.df["user_id"].tolist()
all_review_ids = self.df["review_id"].tolist()
encoded_words, vocab = self.batch_encode_words(all_texts, batch_size, max_length)
similarity_scores = {rid: 0.0 for rid in all_review_ids} # Default scores
for i, (review_id, user_id) in enumerate(tqdm(zip(all_review_ids, all_users), desc="Computing Similarities on GPU")):
if pd.isna(review_id) or pd.isna(user_id):
continue
current_words = encoded_words[i]
if current_words.sum() == 0:
continue
other_indices = torch.tensor([j for j, u in enumerate(all_users) if u != user_id and pd.notna(u)],
dtype=torch.long).to(self.device)
if not other_indices.numel():
continue
other_words = encoded_words[other_indices]
current_set = torch.unique(current_words[current_words > 0])
other_flat = other_words[other_words > 0]
if other_flat.numel() == 0:
continue
other_set = torch.unique(other_flat)
intersection = torch.sum(torch.isin(current_set, other_set)).float()
union = torch.unique(torch.cat([current_set, other_set])).numel()
similarity = intersection / union if union > 0 else 0.0
similarity_scores[review_id] = similarity.item()
return pd.Series(similarity_scores, index=all_review_ids)
def calculate_friend_count(self):
friends = []
for v in self.df["friends"]:
if isinstance(v, str):
friends.append(len(v.split(",")))
elif type(v)==int or type(v)==float:
friends.append(0)
self.df["friends"] = friends
def count_elite_years(self, elite):
if pd.isna(elite):
return 0
return len(str(elite).split(","))
def transform_elite_status(self):
self.df["elite"] = self.df["elite"].apply(lambda x: True if self.count_elite_years(x) > 1 else False)
self.df["elite"] = self.df["elite"].astype(int)
def calculate_review_useful_funny_cool(self):
self.df["review_useful"] = pd.to_numeric(self.df["review_useful"], errors='coerce').fillna(0)
self.df["review_funny"] = pd.to_numeric(self.df["review_funny"], errors='coerce').fillna(0)
self.df["review_cool"] = pd.to_numeric(self.df["review_cool"], errors='coerce').fillna(0)
self.df["review_useful_funny_cool"] = (
self.df["review_useful"] +
self.df["review_funny"] +
self.df["review_cool"]
)
self.df["review_useful_funny_cool"] = self.df["review_useful_funny_cool"].fillna(0).astype(int)
def calculate_user_useful_funny_cool(self):
self.df["user_useful_funny_cool"] = (
self.df["user_useful"] +
self.df["user_funny"] +
self.df["user_cool"]
)
self.df["user_useful_funny_cool"] = self.df["user_useful_funny_cool"].fillna(0).astype(int)
def compute_fake_score(self, row):
suspicion_points = 0
# Linguistic Features
if row["pronoun_density"] < 0.01: # Low personal engagement
suspicion_points += 1
if row["avg_sentence_length"] < 5 or row["avg_sentence_length"] > 30: # Extreme lengths
suspicion_points += 1
if row["grammar_error_score"] > 5: # Many errors
suspicion_points += 1
if row["repetitive_words_count"] > 5: # High repetition
suspicion_points += 1
if row["code_switching_flag"] == 1: # Language mixing
suspicion_points += 1
if row["excessive_punctuation_count"] > 3: # Overuse of punctuation
suspicion_points += 1
if abs(row["sentiment_polarity"]) > 0.8: # Extreme sentiment
suspicion_points += 1
# Review Patterns
if row["similarity_to_other_reviews"] > 0.8: # High duplication
suspicion_points += 1
if row["user_review_burst_count"] > 5: # Spammy bursts
suspicion_points += 1
if row["business_review_burst_count"] > 5: # Targeted bursts
suspicion_points += 1
if abs(row["rating_deviation_from_business_average"]) > 2: # Large rating deviation
suspicion_points += 1
if row["review_like_ratio"] > 0.9 or row["review_like_ratio"] < 0.1: # Extreme like ratio
suspicion_points += 1
# User Behavior
if row["user_account_age"] < 30: # Very new account (days)
suspicion_points += 1
if row["average_time_between_reviews"] < 24: # Rapid reviews (hours)
suspicion_points += 1
if row["user_degree"] < 2: # Low business interaction
suspicion_points += 1
if row["time_since_last_review_user"] < 24: # Recent burst (hours)
suspicion_points += 1
# Threshold: 3 or more points = fake
return 1 if suspicion_points >= 3 else 0
def dropping_unncessary_columns(self):
self.df.drop("review_text", axis=1, inplace=True)
self.df.drop("review_date", axis=1, inplace=True)
self.df.drop("business_name", axis=1, inplace=True)
self.df.drop("address", axis=1, inplace=True)
self.df.drop("city", axis=1, inplace=True)
self.df.drop("state", axis=1, inplace=True)
self.df.drop("postal_code", axis=1, inplace=True)
self.df.drop("categories", axis=1, inplace=True)
self.df.drop("user_name", axis=1, inplace=True)
self.df.drop("yelping_since", axis=1, inplace=True)
self.df.drop("checkin_date", axis=1, inplace=True)
self.df.drop("review_useful", axis=1, inplace=True)
self.df.drop("review_funny", axis=1, inplace=True)
self.df.drop("review_cool", axis=1, inplace=True)
self.df.drop("user_useful", axis=1, inplace=True)
self.df.drop("user_funny", axis=1, inplace=True)
self.df.drop("user_cool", axis=1, inplace=True)
self.df.drop("is_open", axis=1, inplace=True)
self.df.drop("compliment_hot", axis=1, inplace=True)
self.df.drop("compliment_more", axis=1, inplace=True)
self.df.drop("compliment_profile", axis=1, inplace=True)
self.df.drop("compliment_cute", axis=1, inplace=True)
self.df.drop("compliment_list", axis=1, inplace=True)
self.df.drop("compliment_note", axis=1, inplace=True)
self.df.drop("compliment_plain", axis=1, inplace=True)
self.df.drop("compliment_cool", axis=1, inplace=True)
self.df.drop("compliment_funny", axis=1, inplace=True)
self.df.drop("compliment_writer", axis=1, inplace=True)
self.df.drop("compliment_photos", axis=1, inplace=True)
def run_pipeline(self):
logger.info("FINALYZING HOURS COLUMN ...")
self.df["hours"] = self.df["hours"].apply(self.get_avg_duration)
self.df["hours"] = self.df["hours"].fillna(0)
print(self.df["hours"][:10])
print(self.df["hours"].isnull().sum())
logger.info("FINALYZING ATTRIBUTES COLUMN ...")
self.df.drop("attributes",axis=1,inplace=True)
logger.info("CREATING time_since_last_review_user COLUMN ...")
self.calculate_time_since_last_review()
print(np.unique(self.df["time_since_last_review_user"] ))
logger.info("CREATING time_since_last_review_business COLUMN ...")
self.calculate_time_since_last_review_business()
print(np.unique(self.df["time_since_last_review_business"] ))
logger.info("CREATING user_account_age COLUMN ...")
self.calculate_user_account_age()
print(np.unique(self.df["user_account_age"] ))
logger.info("CREATING average_time_between_reviews COLUMN ...")
self.calculate_avg_time_between_reviews()
print(np.unique(self.df["average_time_between_reviews"] ))
logger.info("CREATING user_degree COLUMN ...")
self.calculate_user_degree()
print(np.unique(self.df["user_degree"] ))
logger.info("CREATING business_degree COLUMN ...")
self.calculate_business_degree()
print(np.unique(self.df["business_degree"] ))
logger.info("CREATING rating_variance_user COLUMN ...")
self.calculate_rating_variance_user()
print(np.unique(self.df["rating_variance_user"] ))
logger.info("CREATING user_review_burst_count COLUMN ...")
self.calculate_user_review_burst_count()
print(np.unique(self.df["user_review_burst_count"] ))
logger.info("CREATING business_review_burst_count COLUMN ...")
self.calculate_business_review_burst_count()
print(np.unique(self.df["business_review_burst_count"] ))
logger.info("CREATING temporal_similarity COLUMN ...")
self.calculate_temporal_similarity()
print(np.unique(self.df["temporal_similarity"] ))
logger.info("CREATING rating_deviation_from_business_average COLUMN ...")
self.calculate_rating_deviation_from_business_average()
print(np.unique(self.df["rating_deviation_from_business_average"] ))
logger.info("CREATING review_like_ratio COLUMN ...")
self.calculate_review_like_ratio()
print(np.unique(self.df["review_like_ratio"] ))
logger.info("CREATING latest_checkin_hours COLUMN ...")
self.calculate_latest_checkin_hours()
print(np.unique(self.df["latest_checkin_hours"] ))
logger.info("CREATING pronoun_density COLUMN ...")
self.df["pronoun_density"] = self.df["review_text"].apply(self.compute_pronoun_density)
print(np.unique(self.df["pronoun_density"] ))
logger.info("CREATING avg_sentence_length COLUMN ...")
self.df["avg_sentence_length"] = self.df["review_text"].apply(self.compute_avg_sentence_length)
print(np.unique(self.df["avg_sentence_length"] ))
logger.info("CREATING excessive_punctuation_count COLUMN ...")
self.df["excessive_punctuation_count"] = self.df["review_text"].apply(self.compute_excessive_punctuation)
print(np.unique(self.df["excessive_punctuation_count"] ))
logger.info("CREATING sentiment_polarity COLUMN ...")
self.df["sentiment_polarity"] = self.df["review_text"].apply(self.compute_sentiment_polarity)
print(np.unique(self.df["sentiment_polarity"] ))
logger.info("CREATING good_severity and bad_severity COLUMNS ...")
severity_scores = self.df["review_text"].apply(self.calculate_sentiment_severity)
self.df[["good_severity", "bad_severity"]] = severity_scores
print(np.unique(self.df["good_severity"] ))
print(np.unique(self.df["bad_severity"] ))
logger.info("CREATING code_switching_flag COLUMN ...")
self.df["code_switching_flag"] = self.df["review_text"].apply(self.compute_code_switching_flag)
print(np.unique(self.df["code_switching_flag"] ))
all_texts = self.df["review_text"].tolist()
tokenized_ids = self.batch_tokenize(all_texts, batch_size=32, max_length=512)
logger.info("CREATING grammar_error_score COLUMN ...")
self.df["grammar_error_score"] = self.compute_grammar_error_score(all_texts, tokenized_ids)
print(np.unique(self.df["grammar_error_score"] ))
logger.info("CREATING repetitive_words_count COLUMN ...")
self.df["repetitive_words_count"] = self.compute_repetitive_words_count(all_texts, tokenized_ids)
print(np.unique(self.df["repetitive_words_count"] ))
logger.info("CREATING similarity_to_other_reviews COLUMN ...")
similarity_scores = self.compute_similarity_to_other_reviews(batch_size=32, max_length=512)
self.df["similarity_to_other_reviews"] = self.df["review_id"].map(similarity_scores)
print(np.unique(self.df["similarity_to_other_reviews"] ))
logger.info("CREATING friends COLUMN ...")
self.calculate_friend_count()
print(self.df["friends"].value_counts())
logger.info("CREATING elite COLUMN ...")
self.transform_elite_status()
print(self.df["elite"].value_counts())
logger.info("CREATING review_useful_funny_cool COLUMN ...")
self.calculate_review_useful_funny_cool()
print(self.df["review_useful_funny_cool"].value_counts())
logger.info("CREATING user_useful_funny_cool COLUMN ...")
self.calculate_user_useful_funny_cool()
print(self.df["user_useful_funny_cool"].value_counts())
# logger.info("CREATING LABEL COLUMN ...")
# self.df["fake"] = self.df.apply(self.compute_fake_score, axis=1)
# print(self.df["fake"].value_counts())
logger.info("DELETING THE UNWANTED COLUMNS ...")
self.dropping_unncessary_columns()
print()
logger.info("SEEING NULL VALUES IN FINAL COLUMNS.....")
print(set(self.df.isnull().sum().values))
return self.df