# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + tags=[]
from gluonnlp.data import SentencepieceTokenizer
from kobert.utils import get_tokenizer
import pandas as pd

df = pd.read_csv('./data/ChatBotData.csv', header=0, encoding='utf-8')
sp = SentencepieceTokenizer(get_tokenizer())

# + tags=[]
print(sp('안녕하세요'))
# -

df

sp(df['Q'][10])

Q_len = [len(sp(df['Q'][i])) for i in range(len(df))]
A_len = [len(sp(df['A'][i])) for i in range(len(df))]

import numpy as np
q_len=np.array(Q_len)
a_len=np.array(A_len)

q_len.mean(), q_len.std(), a_len.mean(), a_len.std()

q_len.max(), q_len.min(), a_len.max(), a_len.min()

# + tags=[]
# print longest question
print(sp(df['Q'][Q_len.index(q_len.max())]))
# print longest answer
print(sp(df['A'][A_len.index(a_len.max())]))
# -

max_l = 25
len([l for l in Q_len if l > max_l]), len([l for l in A_len if l > max_l])



StarcoderdataJupyterScriptsDedupFiltered