# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="t16VscFQJWTM" colab_type="text"
# ## Reading File

# + id="qFFRlxtmssr8" colab_type="code" outputId="693875c2-848b-422f-f067-2b31de8e410f" executionInfo={"status": "ok", "timestamp": 1561804249607, "user_tz": -360, "elapsed": 39904, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10838423032874605851"}} colab={"base_uri": "https://localhost:8080/", "height": 122}
from google.colab import drive
import numpy as np
import pandas as pd
drive.mount('/content/gdrive')
bengali_news_after_preprocessing = pd.read_pickle('/content/gdrive/My Drive/Projects/Bengali Text Classification/Bengali_Text_after_preprocessing.pkl')

# + [markdown] id="qjNZIT_pJbmb" colab_type="text"
# ## X contains the numpy array format of text

# + id="eb9BPsWD3nNU" colab_type="code" colab={}
X = bengali_news_after_preprocessing.iloc[:,:].values

# + [markdown] id="KOs4hECjJhXz" colab_type="text"
# ## Following line removes nonetype values from numpy array

# + id="jhq5JkkF8MP0" colab_type="code" colab={}
X = X[X != np.array(None)]

# + [markdown] id="i4HrtllZJr48" colab_type="text"
# ## Following lines counts the tokens in data and sort them

# + id="NlCYgPQI98QE" colab_type="code" colab={}
unique, idx, cnt = np.unique(X, return_index = True, return_counts=True)

# + id="5iwbhSCc_geR" colab_type="code" colab={}
from operator import itemgetter
cnt = 1 / cnt
sorted_vocab = sorted(zip(unique, cnt, idx), key=itemgetter(1, 2))

# + id="ZmvEk0Fp_hnh" colab_type="code" colab={}
unique, count, index = zip(*sorted_vocab)

# + id="3B9MOKP-ADYf" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="c16a5ae4-0dba-4ad0-ebbc-e07e41eafa85" executionInfo={"status": "ok", "timestamp": 1561810490167, "user_tz": -360, "elapsed": 1201, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10838423032874605851"}}
len(unique)

# + [markdown] id="GDfNitGXkqjA" colab_type="text"
# ## Finally vocabularies are saved in vocab.txt file

# + id="6FuCZNM9H2hx" colab_type="code" colab={}
with open('/content/gdrive/My Drive/Projects/Bengali Text Classification/vocab.txt', 'w') as vocab:
  for i in unique:
    vocab.write(i+'\n')

StarcoderdataJupyterScriptsDedupFiltered