# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda env:jcop_usl]
#     language: python
#     name: conda-env-jcop_usl-py
# ---

# Dataset yang digunakan dapat didownload di: https://github.com/rizalespe/Dataset-Sentimen-Analisis-Bahasa-Indonesia atau menggunakan ***git clone*** seperti contoh dibawah ini. Folder yang di _clone_ tersimpan ke dalam folder tempat file project ini disimpan.

# + colab={"base_uri": "https://localhost:8080/"} id="uQhkN4DG2gXb" outputId="120f7340-a07e-4a82-b0a2-7b7de9bc2337"
# #!git clone https://github.com/rizalespe/Dataset-Sentimen-Analisis-Bahasa-Indonesia

# + [markdown] id="GQLN-l93iF2B"
# ## Install Package
# -

# **Requirement Package**:
#
# ```
# 1. nltk : https://www.nltk.org/
# 2. Sastrawi: https://github.com/sastrawi/sastrawi
# 3. numpy: https://numpy.org/
# 4. pandas: https://pandas.pydata.org/
# 5. sklearn: https://scikit-learn.org/stable/
#
# ```

# + [markdown] id="LZMhPthAicoE"
# # Import Package

# +
# #!pip install Sastrawi
#nltk.download('stopwords')
#nltk.download('punkt')

# + id="uq4zAXNPih6-"
import numpy as np
import pandas as pd
import re
import pickle
from string import punctuation
import os
import json

from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory

from sklearn.model_selection import train_test_split


# + [markdown] id="Nh5funiyhB5G"
# # ```{Utils}```

# + id="UW6_M8jhgyy-"
def process_tweet(tweet):
    
    # kumpulan stemming
    factory_stem = StemmerFactory()
    stemmer = factory_stem.create_stemmer()

    # kumpulan stopwords
    factory_stopwords = StopWordRemoverFactory()
    stopword = factory_stopwords.get_stop_words() + stopwords.words('indonesian')
  
    # menghapus kata-kata yang tidak penting seperti @, #
    tweet = re.sub(r'\$\w*', '', tweet)
    tweet = re.sub(r'^RT[\s]+', '', tweet)
    tweet = re.sub(r'https?:\/\/.*[\r\n]*', '', tweet)
    tweet = re.sub(r'#', '', tweet)
    tweet = re.sub(r'<USERNAME>', '', tweet)
    
    # tokenizer word
    tweet_tokens = word_tokenize(tweet)
    
    # membersihkan word
    tweets_clean = [stemmer.stem(word) for word in tweet_tokens if (word not in stopword and word not in punctuation)]
  
    return tweets_clean


# -

def lookup(freqs, word, label):
    
    n = 0
    
    pair = (word, label)
    if (pair in freqs):
        n = freqs[pair]
        
    return n


# + id="cKzrfhkFqV3K"
def count_tweets(tweets, ys):
    
    yslist = np.squeeze(ys).tolist()
    
    freqs = {}
    for y, tweet in zip(yslist, tweets):
        for word in process_tweet(tweet):
            pair = (word, y)
            if pair in freqs:
                freqs[pair] += 1
            else:
                freqs[pair] = 1
    return freqs


# + [markdown] id="Hmi7dW_Ttakm"
# # Processing data

# + [markdown] id="wLVQNDWptjJF"
# ### Import data

# + id="OnmXDdv7ia_u"
df = pd.read_csv("data/dataset_komentar_instagram_cyberbullying.csv")

# + colab={"base_uri": "https://localhost:8080/", "height": 204} id="upLfzDUrihHG" outputId="bdc4f034-0896-4e6d-9375-fd848593a09d"
df.head()

# + colab={"base_uri": "https://localhost:8080/"} id="eIOQcJYdodwa" outputId="ad5180ad-9eee-4e09-c645-9cc84d24a9fa"
df.Sentiment.value_counts()

# + id="pCoOr7tSzPNC"
df.loc[(df.Sentiment == 'negative'),'Sentiment']=0
df.loc[(df.Sentiment == 'positive'),'Sentiment']=1

# + id="fjogwFsJv887"
X = pd.DataFrame(df['Instagram Comment Text'])
y = df.Sentiment

# + id="yn0a5Qw1uuiB"
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# + id="9O8VIbfl0inp"
X_train = X_train.values.squeeze().tolist()
X_test = X_test.values.squeeze().tolist()
y_train = np.array([y_train.values.squeeze().tolist()])
y_test = np.array([y_test.values.squeeze().tolist()])
# -

print(process_tweet(X_train[0]))

# + [markdown] id="hEOOD0DD0HN9"
# ### Build Freqs
# -

# Cell bisa dijalankan atau langsung saja import file `freqs.json`

# + colab={"base_uri": "https://localhost:8080/"} id="mFFFA72t0GJa" outputId="c733f0af-f623-4fb2-ff84-c20ac78dfc72"
# freqs = count_tweets(X_train, y_train)
# -

# check output
print("type(freqs) = " + str(type(freqs)))
print("len(freqs) = " + str(len(freqs.keys())))

# +
# save data
## os.makedirs(name="data", exist_ok=True)

## with open('data/freqs.json', 'wb') as fp:
    ##pickle.dump(freqs, fp)
# -

with open('data/freqs_instag.json', 'rb') as f:
    freqs = pickle.load(f)


# + [markdown] id="3nODmHg3LWsh"
# # Naive Bayes Algorithm
# -

def train_naive_bayes(freqs, X_train, y_train):
    
    loglikelihood = {}
    logprior = 0
    
    # menghitung v, jumlah dari unik word di dalam vocabulary
    vocab = set([pair[0] for pair in freqs.keys()]) # pakai set untuk eleminasi word yang sama
    V = len(vocab)
    
    # hitung N_pos and N_neg
    N_pos = N_neg = 0
    for pair in freqs.keys():# bentuk ('benci', 0)
        # jika label positif atau lebih dari 0
        if pair[1] > 0:
            
            N_pos += freqs[pair]
        
        # else, maka label negatif
        else:
            
            N_neg += freqs[pair]
            
    # hitung, jumlah dokumen
    D = len(y_train[-1])
    
    # hitung D_pos atau jumlah dokumen yang positif
    D_pos = sum(pos for pos in y_train[-1] if pos > 0)
    
    # hitung D_neg atau jumlah dokumen yang negatif
    D_neg = D - D_pos
    
    # hitung logprior
    logprior = np.log(D_pos) - np.log(D_neg)
    
    # untu setiap kata di dalam vocabulary
    for word in vocab:
        # dapatkan frekuensi positive dan negatif dalam word
        freq_pos = lookup(freqs, word, 1.0)
        freq_neg = lookup(freqs, word, 0.0)
        
        # hitung probabilitas setip word adalah positif maupun negatif
        p_w_pos = (freq_pos + 1) / (N_pos + V)
        p_w_neg = (freq_neg + 1) / (N_neg + V)
        
        # calculate log likelihood dari kata
        loglikelihood[word] = np.log(p_w_pos/p_w_neg)
        
    
    return logprior, loglikelihood

# tes algoritma
logprior, loglikelihood = train_naive_bayes(freqs, X_train, y_train)
print(logprior)
print(len(loglikelihood))


# # Test Naive Bayes Algorithm

def naive_bayes_predict(tweet, logprior, loglikelihood):
    
    # prosses word
    word_l = process_tweet(tweet)
    
    # inisiasi probabilitas dengan 0
    p = 0
    
    # tambah logprior
    p += logprior
    
    
    for word in word_l:
        
        # cek jika kata ada didalam loglikehood ditionary
        if word in loglikelihood:
            # tambahkan loglikehood dari kata tersebut ke probabilitas
            p += loglikelihood.get(word)
            
    return p

# test dengan tweet sendiri.
my_tweet = 'Ganteng sekali dia.'
p = naive_bayes_predict(my_tweet, logprior, loglikelihood)
print('Output adalah', p)


def test_naive_bayes(X_test, y_test, logprior, loglikelihood):
    
    y_hats = []
    for tweet in X_test:
        # jika prediksi > 0
        if naive_bayes_predict(tweet, logprior, loglikelihood) > 0:
            # prediksi kelas adalah 1
            y_hat_i = 1
        else:
            # else, maka prediksi kelas adalah 0
            y_hat_i = 0
        
        # tambahkan hasil prediksi kelas ke list y_hats
        y_hats.append(y_hat_i)
        
    # error adalah rata-rata nilai absolut dari perbedaan y_hats dan y_test
    error = np.mean(np.absolute(y_hats - y_test[-1]))
    
    # akurasi adalah 1 - erros
    accuracy = 1 - error
    
    return accuracy


print("Naive Bayes accuracy = %0.4f" %
      (test_naive_bayes(X_test, y_test, logprior, loglikelihood)))

X_test[6]

for tweet in ['Cowok mac<NAME> cuihhh', 'Orang kaya malahn berperilaku sebagai mana orang sederhana', 'Romantisme hanya dia dan istri yg tau.']:
    p = naive_bayes_predict(tweet, logprior, loglikelihood)
    print(f'{tweet} -> {p:.2f}')


# # Filter words by ratio dari jumlah positive dan negativ

def get_ratio(freqs, word):
    
    pos_neg_ratio = {'positive':0, 'negative':0, 'ratio':0.0}
    
    # gunakan fungsi lookup() untuk menemukan positive count untuk sebuah kata
    pos_neg_ratio['positive'] = lookup(freqs, word, 1)
    
    # gunakan fungsi lookup() untuk meneukan negativ count untuk sebuah kata 
    pos_neg_ratio['negative'] = lookup(freqs, word, 0)
    
    # hitung rasio positif negativ count untuk word
    pos_neg_ratio['ratio'] = (pos_neg_ratio['positive'] + 1) / (pos_neg_ratio['negative'] + 1)
    
    return pos_neg_ratio


get_ratio(freqs, 'belagu')


def get_words_by_threshold(freqs, label, threshold):
    
    word_list = {}
    
    for key in freqs.keys():
        word, _ = key
        
        # dapatkan positive atau negative ratio untuk sebuah kata
        pos_neg_ratio = get_ratio(freqs, word)
        
        # jika label adalah 1 dan ratio lebih atau sama dengan threshold
        if label == 1 and pos_neg_ratio['ratio'] >= threshold:
            
            # tambahkan pos_neg_ratio ke dictionary
            word_list[word] = pos_neg_ratio
            
        # jika label = 0 dan pos_neg_ratio kurang dari sama dengan threshold
        elif label == 0 and pos_neg_ratio['ratio'] <= threshold:
            
            word_list[word] = pos_neg_ratio
            
    
    return word_list

get_words_by_threshold(freqs, label=0, threshold=0.05)

get_words_by_threshold(freqs, label=1, threshold=10)

# # Error Analysis

print('Truth Predicted Tweet')
for x, y in zip(X_test, y_test[-1]):
    y_hat = naive_bayes_predict(x, logprior, loglikelihood)
    if y != (np.sign(y_hat) > 0):
        print('%d\t%0.2f\t%s' % (y, np.sign(y_hat) > 0, ' '.join(
            process_tweet(x)).encode('ascii', 'ignore')))

# # Predict own tweet

# +
my_tweet = 'Lo mau cari gara-gara'

p = naive_bayes_predict(my_tweet, logprior, loglikelihood)
print(p)

StarcoderdataJupyterScriptsDedupFiltered