"""
# Libraries
"""
# Use the official tokenization script created by the Google team
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
import tensorflow as tf
import timeit

device_name = tf.test.gpu_device_name()
if "GPU" not in device_name:
    print("GPU device not found")
print('Found GPU at: {}'.format(device_name))
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub

import tokenization

# text processing libraries
import re
import string
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords

from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve

# sklearn 
from sklearn import model_selection
from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer
from sklearn.linear_model import LogisticRegression, RidgeClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import f1_score, accuracy_score
from sklearn import preprocessing, decomposition, model_selection, metrics, pipeline
from sklearn.model_selection import GridSearchCV,StratifiedKFold,RandomizedSearchCV
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
from sklearn.svm import SVC

## scrapping
from bs4 import BeautifulSoup
import requests
import os

# matplotlib and seaborn for plotting
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(style="darkgrid")
 
import warnings
warnings.filterwarnings('ignore')
pd.set_option('display.max_rows', 5000)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 10000)
!pip install tokenization
"""
# Load and Prepare Data
"""
#Training data
train = pd.read_excel('../input/fixed-bdc/Data BDC - Satria Data 2020/Data Latih/Data Latih BDC.xlsx')
print('Training data shape: ', train.shape)
train.head()
# Testing data 
test = pd.read_excel('../input/fixed-bdc/Data BDC - Satria Data 2020/Data Uji/Data Uji BDC.xlsx')
print('Testing data shape: ', test.shape)
test.head()
train['text'] = train['judul'] + ' ' + train['narasi']
test['text'] = test['judul'] + ' ' + test['narasi']
train_ = train.drop(columns=['ID', 'tanggal', 'text', 'narasi', 'nama file gambar'])
test_ = test.drop(columns=['ID', 'tanggal', 'text', 'narasi', 'nama file gambar'])

print(train_.head())
print(test_.head())
"""
# EDA
"""
#Missing values in training set
train_.isnull().sum()
#Missing values in test set
test_.isnull().sum()
train_['label'].value_counts()
"""
# Data Preprocessing
"""
# take copies of the data to leave the originals for BERT
train1 = train_.copy()
test1 = test_.copy()
"""
## Scraping
"""
# url = ['https://turnbackhoax.id/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/2/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/3/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/4/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/5/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/6/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/7/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/8/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/9/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/10/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/11/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/12/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/13/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/14/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/15/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/16/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/17/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/18/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/19/?s=%5Bbenar%5D',
#        'https://turnbackhoax.id/page/20/?s=%5Bbenar%5D',
#       ]
# session = requests.Session()
# vectorizer = TfidfVectorizer()
# corpus = []
# term_doc_mtx = None
# sub='read'
# links_url = []
# for u in url:
#     res = session.get(u)
#     soup = BeautifulSoup(res.content, 'html.parser')
#     for a in soup.find_all('a', {"rel": "bookmark"}):
#          links_url.append(a.get_text().strip())
# for x in links_url:
#     x.replace('[BENAR] ','')
# new_set = {x.replace('[BENAR] ','') for x in links_url}
# new_set
# corpus_df = pd.DataFrame(new_set, columns=['judul'])
# corpus_df['label'] = 0
# corpus_df = corpus_df[["label", "judul"]]
# df
"""
### Scrapping's Result
"""
tambah = pd.read_csv('../input/judul-clean/judul_clean.csv', sep=None)
display(tambah.head(10))
tambah.shape
train1.shape
train1 = train1.append(tambah)
train1.shape
train1['label'].value_counts()
"""
**Data cleaning:** In summary, we want to tokenize our text then send it through a round of cleaning where we turn all characters to lower case, remove brackets, URLs, html tags, punctuation, numbers, etc. We'll also remove emojis from the text and remove common stopwords. This is a vital step in the Bag-of-words + linear model
"""
# emoji removal
def remove_emoji(text):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    return emoji_pattern.sub(r'', text)

# Applying the de=emojifying function to both test and training datasets
n = 'judul'
train1[n] = train1[n].apply(lambda x: remove_emoji(x))
test1[n] = test1[n].apply(lambda x: remove_emoji(x))
# Applying a first round of text cleaning techniques

def clean_text(text):
    '''Make text lowercase, remove text in square brackets,remove links,remove punctuation
    and remove words containing numbers.'''
    text = text.lower() # make text lower case
    text = re.sub('\[.*?\]', '', text) # remove text in square brackets
    text = re.sub('https?://\S+|www\.\S+', '', text) # remove URLs
    text = re.sub('<.*?>+', '', text) # remove html tags
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text) # remove punctuation
    text = re.sub('\n', '', text) # remove words conatinaing numbers
    text = re.sub('\w*\d\w*', '', text)
    text = re.sub('[‘’“”…]', '', text)
    text = re.sub(r'(\w)\1{2,}', r'\1', text) # remove repeated char


    return text
train1.head()
# text preprocessing function
def text_preprocessing(text):
    """
    Cleaning and parsing the text.

    """
    tokenizer_reg = nltk.tokenize.RegexpTokenizer(r'\w+')
    
    nopunc = clean_text(text)
    tokenized_text = tokenizer_reg.tokenize(nopunc)
    remove_stopwords = [w for w in tokenized_text if w not in stopwords.words('indonesian')]
    combined_text = ' '.join(remove_stopwords)
    return combined_text

# Applying the cleaning function to both test and training datasets
train1[n] = train1[n].apply(lambda x: text_preprocessing(x))
test1[n] = test1[n].apply(lambda x: text_preprocessing(x))

# Let's take a look at the updated text
train1[n].head()
train1.head()
!pip install sastrawi
from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory

factory = StopWordRemoverFactory()
stopword = factory.get_stop_words()
stopword.append('yg')
stopword.append('dengan')
stopword.append('ia')
stopword.append('bahwa')
stopword.append('oleh')
stopword.append('jadi')
stopword.append('yth')
stopword.append('nya')
stopword.append('dengan')
stopword.append('dgn')
stopword.append('dlm')
#stopword.append('indonesia')
#stopword.append('orang')
#stopword.append('tersebut')
#stopword.append('semua')
print(stopword)
def remove_stopwords(text, stem=False):
    tokens = []
    for token in text.split():
        if token not in stopword:
            tokens.append(token)
    return " ".join(tokens)

train1['stop'] = train1[n].apply(lambda x: remove_stopwords(x))
test1['stop'] = test1[n].apply(lambda x: remove_stopwords(x))
print('Original text:',train.narasi[1399])
print('After Stopwords Removal:',train1.stop[1399])
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
factory = StemmerFactory()
stemmer = factory.create_stemmer()
train1['stop'] = train1['stop'].apply(lambda x: stemmer.stem(x))
test1['stop'] = test1['stop'].apply(lambda x: stemmer.stem(x))

'''for index, row in train['translate'].iteritems():
    stem = stemmer.stem(row) #detecting each row
    train.loc[index, 'stem'] = stem'''
train1.head()
"""
## Remove Duplicates Using Doc2Vec
"""
# gensim modules
from gensim import utils
from gensim.models.doc2vec import LabeledSentence
from gensim.models.doc2vec import Doc2Vec, TaggedDocument
from gensim.models import doc2vec
from gensim.test.utils import common_texts
from nltk.tokenize import word_tokenize
from random import shuffle

%%time

## tagged_data for train set
tagged_data = [TaggedDocument(words=word_tokenize(_d), tags=[str(i)]) for i, _d in enumerate(train1['stop'])]
## tagged_data for test set
tagged_data_test = [TaggedDocument(words=word_tokenize(_d.lower()), tags=[str(i)]) for i, _d in enumerate(test1['stop'])]

## Training Model (PV-DM)
max_epochs = 100
vec_size = 20
alpha = 0.025

## MODEL FOR TRAIN SET
model = Doc2Vec(size=vec_size,
                alpha=alpha, 
                min_alpha=0.00025,
                min_count=1,
                dm =1)
  
model.build_vocab(tagged_data)

for epoch in range(max_epochs):
    model.train(tagged_data,
                total_examples=model.corpus_count,
                epochs=model.iter)
    # decrease the learning rate
    model.alpha -= 0.0002
    # fix the learning rate, no decay
    model.min_alpha = model.alpha
    

## MODEL FOR TEST SET
model_test = Doc2Vec(size=vec_size,
                alpha=alpha, 
                min_alpha=0.00025,
                min_count=1,
                dm =1)
  
model_test.build_vocab(tagged_data_test)

for epoch in range(max_epochs):
    model_test.train(tagged_data_test,
                total_examples=model_test.corpus_count,
                epochs=model_test.iter)
    # decrease the learning rate
    model_test.alpha -= 0.0002
    # fix the learning rate, no decay
    model_test.min_alpha = model_test.alpha

# model.save("d2v.model")
# model_test.save("d2v.model")
# print("Model Saved test")
# model= Doc2Vec.load("./d2v.model")
# model_test = Doc2Vec.load("./d2v.model_test")
"""
### Checking Duplicates using Doc2Vec Similarity for train set
"""
duplicat_index = list()
double = list()
count = 0
for i in range(4231):
    similar_doc = model.docvecs.most_similar(str(i))
    if (similar_doc[0][1] > 0.98):
#         print(tagged_data[i])
#         print(tagged_data[int(similar_doc[0][0])])
#         print(similar_doc[0][1])
        if int(similar_doc[0][0]) not in double: 
            duplicat_index.append(int(similar_doc[0][0]))
            double.append(int(tagged_data[i][1][0]))
#         print()

"""
### Checking Duplicates using Doc2Vec Similarity for test set
"""
duplicat_index_test = list()
double_test = list()
for i in range(len(test1['judul'])):
    similar_doc = model_test.docvecs.most_similar(str(i))
    if (similar_doc[0][1] > 0.98):
#         print(tagged_data[i])
#         print(tagged_data[int(similar_doc[0][0])])
#         print(similar_doc[0][1])
        if int(similar_doc[0][0]) not in double: 
            duplicat_index_test.append(int(similar_doc[0][0]))
            double_test.append(int(tagged_data[i][1][0]))
#         print()
train_ori = train1.copy()
train1 = train1[~train1.index.isin(duplicat_index)]

# test_ori = test1.copy()
# test1 = test1[~test1.index.isin(duplicat_index_test)]
train1.shape, train_ori.shape
dup_index = train_ori.shape[0] - train1.shape[0]
# dup_index_test = test_ori.shape[0] - test1.shape[0]

duplicat_percentage = round( dup_index / train_ori.shape[0] * 100, 2)
print(f'train data has {dup_index} rows that duplicates, that means {duplicat_percentage} percent of the total')

# duplicat_percentage_test = round( dup_index_test / test_ori.shape[0] * 100, 2)
# print(f'test data has {dup_index_test} rows that duplicates, that means {duplicat_percentage_test} percent of the total')
"""
# Counting


We are going to see the diversity of data. How much emoji did train and test set contain? How many symbols & punctuation did train and test set contains? if they have the same proportion that means the train and test set has the same diversity. Also, we use wordcloud to check if the most frequent words are similar between train and test set.
"""
"""
**Note : We use train and test set which have not been preprocessed to calculate the proportions**
"""
"""
## Cek Emoji
"""
column = 'narasi'

have_emoji_train_idx = []
have_emoji_test_idx = []

for idx,text in enumerate(train[column]):
    before = text 
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    after = emoji_pattern.sub(r'', text)
    if before != after:
        have_emoji_train_idx.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()


for idx,text in enumerate(test[column]):
    before = text 
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    after = emoji_pattern.sub(r'', text)
    if before != after:
        have_emoji_test_idx.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()

train_emoji_percentage = round(len(have_emoji_train_idx) / train.shape[0] * 100, 2)
print(f'train data has {len(have_emoji_train_idx)} rows that used emoji, that means {train_emoji_percentage} percent of the total ({column})')

test_emoji_percentage = round(len(have_emoji_test_idx) / test.shape[0] * 100, 2)
print(f'test data has {len(have_emoji_test_idx)} rows that used emoji, that means {test_emoji_percentage} percent of the total ({column})')
"""
## Lower Case
"""
lower_train = list()
lower_test  = list()

for idx,text in enumerate(train[column]):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    before = text
    after = text.lower() 

    if before != after:
        lower_train.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()
for idx,text in enumerate(test[column]):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    before = text
    after = text.lower() 

    if before != after:
        lower_test.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()
train_lower_percentage = round(len(lower_train) / train.shape[0] * 100, 2)
print(f'train data has {len(lower_train)} rows that contains upper case, that means {train_lower_percentage} percent of the total ({column})')

test_lower_percentage = round(len(lower_test) / test.shape[0] * 100, 2)
print(f'test data has {len(lower_test)} rows that contains upper case, that means {test_lower_percentage} percent of the total ({column})')
"""
## Square Brackets
"""
square_brackets_train = list()
square_brackets_test  = list()

for idx,text in enumerate(train[column]):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    text = text.lower() 
    before = text
    after = re.sub('\[.*?\]', '', text) # remove text in square brackets
    if before != after:
        square_brackets_train.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()


for idx,text in enumerate(test[column]):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    text = text.lower() 
    before = text
    after = re.sub('\[.*?\]', '', text) # remove text in square brackets
    if before != after:
        square_brackets_test.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()
square_brackets_percentage = round(len(square_brackets_train) / train.shape[0] * 100, 2)
print(f'train data has {len(square_brackets_train)} rows that used square_brackets, that means {square_brackets_percentage} percent of the total ({column})')

square_brackets_percentage = round(len(square_brackets_test) / test.shape[0] * 100, 2)
print(f'test data has {len(square_brackets_test)} rows that used square_brackets, that means {square_brackets_percentage} percent of the total ({column})')
"""
## URLs
"""
urls_train = list()
urls_test  = list()

for idx,text in enumerate(train[column]):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    text = text.lower() 
    text = re.sub('\[.*?\]', '', text) # remove text in square brackets
    before = text
    after = re.sub('https?://\S+|www\.\S+', '', text) # remove URLs
    if before != after:
        urls_train.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()


for idx,text in enumerate(test[column]):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    text = text.lower() 
    text = re.sub('\[.*?\]', '', text) # remove text in square brackets
    before = text
    after = re.sub('https?://\S+|www\.\S+', '', text) # remove URLs
    if before != after:
        urls_test.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()
urls_train_percentage = round(len(urls_train) / train.shape[0] * 100, 2)
print(f'train data has {len(urls_train)} rows that contains urls, that means {urls_train_percentage} percent of the total ({column})')

urls_test_percentage = round(len(urls_test) / test.shape[0] * 100, 2)
print(f'test data has {len(urls_test)} rows that contains urls, that means {urls_test_percentage} percent of the total ({column})')
"""
## Html tags
"""
html_train = list()
html_test = list()

for idx,text in enumerate(train[column]):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    text = text.lower() 
    text = re.sub('\[.*?\]', '', text) # remove text in square brackets
    text = re.sub('https?://\S+|www\.\S+', '', text) # remove URLs
    before = text
    after = re.sub('<.*?>+', '', text) # remove html tags
    if before != after:
        html_train.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()
for idx,text in enumerate(test[column]):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    text = text.lower() 
    text = re.sub('\[.*?\]', '', text) # remove text in square brackets
    text = re.sub('https?://\S+|www\.\S+', '', text) # remove URLs
    before = text
    after = re.sub('<.*?>+', '', text) # remove html tags
    if before != after:
        html_test.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()
html_train_percentage = round(len(html_train) / train.shape[0] * 100, 2)
print(f'train data has {len(html_train)} rows that contains html tags, that means {html_train_percentage} percent of the total ({column})')

html_test_percentage = round(len(html_test) / test.shape[0] * 100, 2)
print(f'test data has {len(html_test)} rows that contains html tags, that means {html_test_percentage} percent of the total ({column})')
"""
## Punctuation
"""
punc_train = list()
punc_test = list()

for idx,text in enumerate(train[column]):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    text = text.lower() 
    text = re.sub('\[.*?\]', '', text) # remove text in square brackets
    text = re.sub('https?://\S+|www\.\S+', '', text) # remove URLs
    text = re.sub('<.*?>+', '', text) # remove html tags
    before = text
    after = re.sub('[%s]' % re.escape(string.punctuation), '', text) # remove punctuation
    if before != after:
        punc_train.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()
for idx,text in enumerate(test[column]):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    text = text.lower() 
    text = re.sub('\[.*?\]', '', text) # remove text in square brackets
    text = re.sub('https?://\S+|www\.\S+', '', text) # remove URLs
    text = re.sub('<.*?>+', '', text) # remove html tags
    before = text
    after = re.sub('[%s]' % re.escape(string.punctuation), '', text) # remove punctuation
    if before != after:
        punc_test.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()
punc_train_percentage = round(len(punc_train) / train.shape[0] * 100, 2)
print(f'train data has {len(punc_train)} rows that contains punctuation, that means {punc_train_percentage} percent of the total ({column})')

punc_test_percentage = round(len(punc_test) / test.shape[0] * 100, 2)
print(f'test data has {len(punc_test)} rows that contains punctuation, that means {punc_test_percentage} percent of the total ({column})')
"""
## Numbers
"""
num_train = list()
num_test = list()
for idx,text in enumerate(train[column]):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    text = text.lower() 
    text = re.sub('\[.*?\]', '', text) # remove text in square brackets
    text = re.sub('https?://\S+|www\.\S+', '', text) # remove URLs
    text = re.sub('<.*?>+', '', text) # remove html tags
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text) # remove punctuation
    before = text
    text = re.sub('\n', '', text) # remove words conatinaing numbers
    text = re.sub('\w*\d\w*', '', text)
    after = re.sub('[‘’“”…]', '', text)
    if before != after:
        num_train.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()
for idx,text in enumerate(test[column]):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    text = text.lower() 
    text = re.sub('\[.*?\]', '', text) # remove text in square brackets
    text = re.sub('https?://\S+|www\.\S+', '', text) # remove URLs
    text = re.sub('<.*?>+', '', text) # remove html tags
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text) # remove punctuation
    before = text
    text = re.sub('\n', '', text) # remove words conatinaing numbers
    text = re.sub('\w*\d\w*', '', text)
    after = re.sub('[‘’“”…]', '', text)
    if before != after:
        num_test.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()
num_train_percentage = round(len(num_train) / train.shape[0] * 100, 2)
print(f'train data has {len(num_train)} rows that contains numbers, that means {num_train_percentage} percent of the total ({column})')

num_test_percentage = round(len(num_test) / test.shape[0] * 100, 2)
print(f'test data has {len(num_test)} rows that contains numbers, that means {num_test_percentage} percent of the total ({column})')
"""
## Repeated Char
"""
repeat_train = list()
repeat_test = list()

for idx,text in enumerate(train[column]):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    text = text.lower() 
    text = re.sub('\[.*?\]', '', text) # remove text in square brackets
    text = re.sub('https?://\S+|www\.\S+', '', text) # remove URLs
    text = re.sub('<.*?>+', '', text) # remove html tags
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text) # remove punctuation
    text = re.sub('\n', '', text) # remove words conatinaing numbers
    text = re.sub('\w*\d\w*', '', text)
    text = re.sub('[‘’“”…]', '', text)
    before = text
    after = re.sub(r'(\w)\1{2,}', r'\1', text)
    if before != after:
        repeat_train.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()
for idx,text in enumerate(test[column]):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    text = text.lower() 
    text = re.sub('\[.*?\]', '', text) # remove text in square brackets
    text = re.sub('https?://\S+|www\.\S+', '', text) # remove URLs
    text = re.sub('<.*?>+', '', text) # remove html tags
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text) # remove punctuation
    text = re.sub('\n', '', text) # remove words conatinaing numbers
    text = re.sub('\w*\d\w*', '', text)
    text = re.sub('[‘’“”…]', '', text)
    before = text
    after = re.sub(r'(\w)\1{2,}', r'\1', text)
    if before != after:
        repeat_test.append(idx)
#         print(before)
#         print(after)
#         print(idx)
#         print()
repeat_train_percentage = round(len(repeat_train) / train.shape[0] * 100, 2)
print(f'train data has {len(repeat_train)} rows that contains repeated char, that means {repeat_train_percentage} percent of the total ({column})')

repeat_test_percentage = round(len(repeat_test) / test.shape[0] * 100, 2)
print(f'test data has {len(repeat_test)} rows that contains repeated char, that means {repeat_test_percentage} percent of the total ({column})')
'''from wordcloud import WordCloud
wc = WordCloud(
    height=600,repeat=False,width=1400,max_words=1000,stopwords=stopword,
    colormap='terrain',background_color='white',mode='RGBA'
).generate(
    ' '.join(train1['stop'].dropna().astype(str)))
plt.figure(figsize = (16,16))
plt.imshow(wc)
plt.title('Judul Wordcloud')
plt.axis('off')
plt.show()'''
"""
# Models

We try two different approach : bow and tf-idf

Models that we used are :
1. Logistic Regression
2. Random Forest
3. Support Vector
4. Xgboost
5. Adaboost
6. Extratree
"""
"""
## Sparse Matrix
"""
count_vectorizer = CountVectorizer(ngram_range = (1,2), min_df = 1)
train_vectors = count_vectorizer.fit_transform(train1['stop'])
test_vectors = count_vectorizer.transform(test1["stop"])

## Keeping only non-zero elements to preserve space 
train_vectors.shape
tfidf = TfidfVectorizer(ngram_range=(1, 2), min_df = 1)
train_tfidf = tfidf.fit_transform(train1['stop'])
test_tfidf = tfidf.transform(test1["stop"])

train_tfidf.shape
"""
### Logreg
"""
from sklearn.model_selection import StratifiedKFold

# Create a StratifiedKFold object
skf = StratifiedKFold(n_splits=5, shuffle=False, random_state=42)
%%time
logreg_bow = LogisticRegression()

logreg_bow.fit(train_vectors, train1["label"])
scores = model_selection.cross_val_score(logreg_bow, train_vectors, train1["label"], cv=skf, scoring="f1")
scores.mean()
%%time
logreg_tfidf = LogisticRegression()
logreg_tfidf.fit(train_tfidf, train1["label"])
scores = model_selection.cross_val_score(logreg_tfidf, train_tfidf, train1["label"], cv=skf, scoring="f1")
scores.mean()
"""
### RF
"""
'''from scipy.stats import randint
forest_clf = RandomForestClassifier()
param_distribs = {
        'n_estimators': randint(low=100, high=300),
        'max_features': randint(low=8, high=17),
    }

rnd_forest_clf = RandomizedSearchCV(forest_clf, param_distributions=param_distribs,
                                n_iter=10, cv=skfold, scoring='f1', random_state=42)
rnd_forest_clf.fit(train_vectors, train1["label"])

{'max_features': 15, 'n_estimators': 288}''' #malah turun
%%time
# Fitting a simple Random Forest on BoW
RF_bow = RandomForestClassifier(n_estimators=288)
RF_bow.fit(train_vectors, train1["label"])
scores = model_selection.cross_val_score(RF_bow, train_vectors, train1["label"], cv=skf, scoring="f1")
scores.mean()
%%time
# Fitting a simple Random Forest on TFIDF
RF_tfidf = RandomForestClassifier(n_estimators=288)
RF_tfidf.fit(train_tfidf, train1["label"])
scores = model_selection.cross_val_score(RF_tfidf, train_tfidf, train1["label"], cv=skf, scoring="f1")
scores.mean()
"""
### SVC
"""
%%time
from sklearn.svm import SVC
# Fitting a simple SVC on BoW
SVC_bow = SVC(probability=True)
SVC_bow.fit(train_vectors, train1["label"])
scores = model_selection.cross_val_score(SVC_bow, train_vectors, train1["label"], cv=skf, scoring="f1")
scores.mean()
%%time
# Fitting a simple SVC on TFIDF
SVC_tfidf = SVC(probability=True)
SVC_tfidf.fit(train_tfidf, train1["label"])
scores = model_selection.cross_val_score(SVC_tfidf, train_tfidf, train1["label"], cv=skf, scoring="f1")
scores.mean()
"""
### XGB
"""
%%time
XGB_bow = XGBClassifier()

XGB_bow.fit(train_vectors, train1["label"])
scores = model_selection.cross_val_score(XGB_bow, train_vectors, train1["label"], cv=skf, scoring="f1")
scores.mean()
%%time
# Fitting a simple XGB on TFIDF
XGB_tfidf = XGBClassifier()
XGB_tfidf.fit(train_tfidf, train1["label"])
scores = model_selection.cross_val_score(XGB_tfidf, train_tfidf, train1["label"], cv=skf, scoring="f1")
scores.mean()
"""
### Adaboost
"""
%%time
from sklearn.tree import DecisionTreeClassifier
# Fitting a simple AdaBoost on BoW
ada_bow = AdaBoostClassifier(
                        base_estimator=DecisionTreeClassifier(
                                                             max_depth=7,
                                                             min_samples_leaf=5,
                                                             min_samples_split=10,
                                                             ),
                   learning_rate=0.3, n_estimators=2, random_state=42)
ada_bow.fit(train_vectors, train1["label"])
scores = model_selection.cross_val_score(ada_bow, train_vectors, train1["label"], cv=skf, scoring="f1")
scores.mean()
%%time
# Fitting a simple AdaBoost on TFIDF
ada_tfidf = AdaBoostClassifier(
                        base_estimator=DecisionTreeClassifier(
                                                             max_depth=5,
                                                             min_samples_leaf=2,
                                                             min_samples_split=3,
                                                             ),
                            learning_rate=0.3, n_estimators=2, random_state=42)
ada_tfidf.fit(train_tfidf, train1["label"])
scores = model_selection.cross_val_score(ada_tfidf, train_tfidf, train1["label"], cv=skf, scoring="f1")
scores.mean()
"""
### Extra
"""
'''ext_clf = ExtraTreesClassifier()
param_grid = {"max_depth": [None],
              "max_features": [10, 17],
              "min_samples_split": [2, 3, 10],
              "min_samples_leaf": [1, 3, 10],
              "bootstrap": [False, True],
              "n_estimators" :[50,100,200],
              "criterion": ["gini"]}

# Cross validate model with Kfold stratified cross val
skfold = StratifiedKFold(n_splits=5)
grid_ext_clf = GridSearchCV(ext_clf,param_grid, cv=skfold, scoring="f1", n_jobs= 4, verbose = 1)
grid_ext_clf.fit(train_vectors, train1["label"])

grid_ext_clf.best_params_

{'bootstrap': False,
 'criterion': 'gini',
 'max_depth': None,
 'max_features': 17,
 'min_samples_leaf': 1,
 'min_samples_split': 2,
 'n_estimators': 100}
 '''
%%time
# Fitting a simple Extra on BoW
ext_bow = ExtraTreesClassifier(criterion='gini',
                               max_features=17,
                               n_estimators=100,
                              )
ext_bow.fit(train_vectors, train1["label"])
scores = model_selection.cross_val_score(ext_bow, train_vectors, train1["label"], cv=skf, scoring="f1")
scores.mean()
%%time
# Fitting a simple Extra on TFIDF 
ext_tfidf = ExtraTreesClassifier()
ext_tfidf.fit(train_tfidf, train1["label"])
scores = model_selection.cross_val_score(ext_tfidf, train_tfidf, train1["label"], cv=skf, scoring="f1")
scores.mean()
"""
## Cross Validations Analysis
"""
"""
### Confusion Matrix
"""
from sklearn.metrics import confusion_matrix
from sklearn.metrics import f1_score
from sklearn.model_selection import StratifiedKFold
def cm_analysis():
    # splitter
    SEED = 2020
    f1 = []
    tra_fold_df = []
    val_fold_df = []
    cm = []
    skf = StratifiedKFold(n_splits=5, shuffle=False, random_state=SEED)
    # models
    models = [LogisticRegression(), RandomForestClassifier(n_estimators=288), SVC(probability=True),
            XGBClassifier(), 

            AdaBoostClassifier(
                base_estimator=DecisionTreeClassifier(max_depth=7, min_samples_leaf=5,min_samples_split=10,),
                learning_rate=0.3, n_estimators=2, random_state=42),

            ExtraTreesClassifier(criterion='gini',max_features=17,n_estimators=100,)]


    for model in models:
        label = {'mislabeled_1':0, 'true_1':0,'mislabeled_0':0,'true_0':0}
        score = []
        for tra_idx, val_idx in skf.split(train1, train1[['label']]):

            X_tra = train1.iloc[tra_idx]
            X_val = train1.iloc[val_idx]
            tra_fold_df.append(X_tra)
            val_fold_df.append(X_val)
            y_tra = train1.iloc[tra_idx]['label']
            y_val = train1.iloc[val_idx]['label']

            #### IF YOU WANT YOU TO USE BOW UNCOMMENT THE CODE BELOW, AND THEN COMMENT THE TF-IDF CODE SECTION

            ## BOW ##
    #         count_vectorizer = CountVectorizer(ngram_range = (1,2), min_df = 1)
    #         train_vectors = count_vectorizer.fit_transform(X_tra['stop'])
    #         validasi_vectors = count_vectorizer.transform(X_val['stop']) 

            ## TF-IDF ##
            tfidf = TfidfVectorizer(ngram_range=(1, 2), min_df = 1)
            train_vectors = tfidf.fit_transform(X_tra['stop'])
            validasi_vectors = tfidf.transform(X_val["stop"])

            ## TRAINING AND PREDICT
            model.fit(train_vectors,y_tra)
            y_pred = model.predict(validasi_vectors)
    #         print('For zero :',confusion_matrix(y_val, y_pred)[0])
    #         print('For one :',confusion_matrix(y_val,y_pred)[1])


            ## SAVE CM RESULT TO A DICT
            label['true_0'] += confusion_matrix(y_val,y_pred)[0][0]
            label['mislabeled_0'] += confusion_matrix(y_val,y_pred)[0][1]
            label['true_1'] += confusion_matrix(y_val,y_pred)[1][1]
            label['mislabeled_1'] += confusion_matrix(y_val,y_pred)[1][0]

            ## SAVE THE F1_SCORE FOR EACH FOLD
            score.append(f1_score(y_val,y_pred,average=None)[1])
        cm_each = list()
    #     print(f'f1_score_avg_none:',sum(score)/5)
        f1.append(sum(score)/5)
    #     print(label)
        for key,value in label.items():
    #         print(model,key,round(value/5))
            cm_each.append(round(value/5))
        cm.append(cm_each)

    cm = pd.DataFrame(cm, columns = ['Mislabeled_1','True_1','Mislabeled_0','True_0'])
    cm['Model'] = ['logreg','rf','svc','xgb','ada','ext']
    cm.set_index('Model',inplace=True)
    cm = cm.astype('int')
    cm['f1_score'] = f1
    display(cm.style.background_gradient(cmap='Blues'))
    return

cm_analysis()
"""
## Ensemble: Stacking
"""
ext_test_pred = ext_tfidf.predict_proba(test_vectors)[:,1]
forest_test_pred = RF_tfidf.predict_proba(test_vectors)[:,1]
xgb_test_pred = XGB_tfidf.predict_proba(test_vectors)[:,1]
#dt_test_pred = dt_tfidf.predict_proba(test_vectors)[:,1]
ada_test_pred = ada_tfidf.predict_proba(test_vectors)[:,1]
svc_test_pred = SVC_tfidf.predict_proba(test_vectors)[:,1]
#nb_test_pred = NB_tfidf.predict_proba(test_vectors)[:,1]
lg_test_pred = logreg_tfidf.predict_proba(test_vectors)[:,1]

ext_train_pred = ext_tfidf.predict_proba(train_vectors)[:,1]
forest_train_pred = RF_tfidf.predict_proba(train_vectors)[:,1]
xgb_train_pred = XGB_tfidf.predict_proba(train_vectors)[:,1]
#dt_train_pred = dt_tfidf.predict_proba(train_vectors)[:,1]
ada_train_pred = ada_tfidf.predict_proba(train_vectors)[:,1]
svc_train_pred = SVC_tfidf.predict_proba(train_vectors)[:,1]
#nb_train_pred = NB_tfidf.predict_proba(train_vectors)[:,1]
lg_train_pred = logreg_tfidf.predict_proba(train_vectors)[:,1]
ext_test_pred = ext_bow.predict_proba(test_vectors)[:,1]
forest_test_pred = RF_bow.predict_proba(test_vectors)[:,1]
xgb_test_pred = XGB_bow.predict_proba(test_vectors)[:,1]
#dt_test_pred = dt_bow.predict_proba(test_vectors)[:,1]
ada_test_pred = ada_bow.predict_proba(test_vectors)[:,1]
svc_test_pred = SVC_bow.predict_proba(test_vectors)[:,1]
#nb_test_pred = NB_bow.predict_proba(test_vectors)[:,1]
lg_test_pred = logreg_bow.predict_proba(test_vectors)[:,1]

ext_train_pred = ext_bow.predict_proba(train_vectors)[:,1]
forest_train_pred = RF_bow.predict_proba(train_vectors)[:,1]
xgb_train_pred = XGB_bow.predict_proba(train_vectors)[:,1]
#dt_train_pred = dt_bow.predict_proba(train_vectors)[:,1]
ada_train_pred = ada_bow.predict_proba(train_vectors)[:,1]
svc_train_pred = SVC_bow.predict_proba(train_vectors)[:,1]
#nb_train_pred = NB_bow.predict_proba(train_vectors)[:,1]
lg_train_pred = logreg_bow.predict_proba(train_vectors)[:,1]
"""
## SAVING EACH MODEL
"""
import pickle
# save the model to disk
rf_bow_model = 'rf_model.sav'
ext_bow_model = 'ext_model.sav'
XGB_bow_model = 'xgb_model.sav'
ada_bow_model = 'ada_model.sav'
svc_bow_model = 'svc_model.sav'
lg_bow_model = 'lg_model.sav'

pickle.dump(RF_bow, open(rf_bow_model, 'wb'))
pickle.dump(ext_bow, open(ext_bow_model, 'wb'))
pickle.dump(XGB_bow, open(XGB_bow_model, 'wb'))
pickle.dump(ada_bow, open(ada_bow_model, 'wb'))
pickle.dump(SVC_bow, open(svc_bow_model, 'wb'))
pickle.dump(logreg_bow, open(lg_bow_model, 'wb'))
"""
Size for each model :
- ext_model = 36.666 kb
- xgb_model = 503 kb
- lg_model = 220 kb
- ada_model = 11 kb
- rf_model = 37.345 kb
"""
base_pred = pd.DataFrame({
    'ext':ext_train_pred.ravel(),
    'forest':forest_train_pred.ravel(), 
    'xgb':xgb_train_pred.ravel(), 
    'svc':svc_train_pred.ravel(),
    'ada':ada_train_pred.ravel(),
    #'lg':lg_train_pred.ravel(),
    #'nb':nb_train_pred.ravel(),
    #'dt':dt_train_pred.ravel()
    
})

test_pred = pd.DataFrame({
    'ext':ext_test_pred.ravel(),
    'forest':forest_test_pred.ravel(), 
    'xgb':xgb_test_pred.ravel(), 
    'svc':svc_test_pred.ravel(),
    'ada':ada_test_pred.ravel(),
    #'lg':lg_test_pred.ravel(),
    #'nb':nb_test_pred.ravel(),
    #'dt':dt_test_pred.ravel()
})
# Display numerical correlations between features on heatmap
sns.set(font_scale=2.5)
correlation_train = base_pred.corr()
mask = np.triu(correlation_train.corr())
plt.figure(figsize=(10, 10))
sns_plot = sns.heatmap(correlation_train,
            annot=True,
            fmt='.1f',
            cmap='coolwarm',
            square=True,
            mask=mask,
            linewidths=1, alpha=1)

plt.show()
sns_plot.get_figure().savefig('tfidf.png',transparent=True)
#grid={"C":np.logspace(-4,4,20), "penalty":["l1","l2"]}# l1 lasso l2 ridge
final_model = LogisticRegression(C=0.012742749857031334, penalty='l2')
#final_model=GridSearchCV(final_model,grid,cv=10)
final_model.fit(base_pred, train1["label"])
#print(final_model.best_params_)

scores = model_selection.cross_val_score(final_model, base_pred, train1["label"], cv=5, scoring="f1")
print('Cross Validation :', scores.mean())

# make prediction using our test data and model
y_pred = final_model.predict(base_pred)
print('')
print('###### Logreg Classifier ######')

# evaluating the model
print("Testing Accuracy :", accuracy_score(train1["label"], y_pred))
#print("Best Score :", final_model.best_score_)
print('F1-Score :', metrics.f1_score(train1["label"], y_pred))
#print('Best Params :', final_model.best_params_)
#print('Best Estimator', final_model.best_estimator_)
"""
## Prediction
"""
test_read = pd.read_excel('../input/fixed-bdc/Data BDC - Satria Data 2020/Data Uji/Data Uji BDC.xlsx')
template = pd.read_csv('../input/submission-bdc/template jawaban BDC.csv')
final_pred = final_model.predict(test_pred)
submission = pd.DataFrame()
submission["ID"] = test_read["ID"]
submission["prediksi"] = final_pred
submission = pd.merge(template['ID'], submission)
submission.to_csv("submission.csv", index=False)
sub = pd.read_csv('./submission.csv')
sub.prediksi.value_counts()
rf_pred = RF_bow.predict(test_vectors)
rf_submit = pd.DataFrame()
rf_submit["ID"] = test_read["ID"]
rf_submit["prediksi"] = rf_pred
rf_submit = pd.merge(template['ID'], submission)
rf_submit.to_csv("rf_submit.csv", index=False)
rf_sub = pd.read_csv('./rf_submit.csv')
rf_sub.prediksi.value_counts()