# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: 'Python 3.7.10 64-bit (''TF-Keras'': conda)'
#     name: python3710jvsc74a57bd06c70104609522f67cb30e8200dfa6f77a5bec0e1b1538a23f62dccdf26f51f37
# ---

import requests
import time
import json
import pandas as pd
import unidecode
import pickle5 as pk

spanish_path = './datasets/es.json'

# +
file = open(spanish_path, 'r')
data = []
for line in file:
    data.append(json.loads(line))

text_raw_df = pd.json_normalize(data)

print(text_raw_df.shape)
text_raw_df['text'].head(10)
# -

start_time = time.time()
r = None
for i in range(1):
    headers = {'Content-Type': 'text/plain'}
    r = requests.post('http://localhost:8080/', headers=headers, data=unidecode.unidecode('#PiñeraCuliao seguro que otro presidente hubiera podido llevar la pandemia, déjense de tirar mierda,          Viajen a cuba total en ese país serán felices , junto con Jadue, CHV, la Red , colegio médico y cuantas personas de izquierda, buenas para el veneno'))
end_time = time.time()
print(end_time-start_time)

b'#Pi\xc3\xb1eraCuliao seguro que otro presidente hubiera podido llevar la pandemia, d\xc3\xa9jense de tirar mierda,          Viajen a cuba total en ese pa\xc3\xads ser\xc3\xa1n felices , junto con Jadue, CHV, la Red , colegio m\xc3\xa9dico y cuantas personas de izquierda, buenas para el veneno'.decode('utf-8')

# print(r.json()['entities']['MisinfoClass'][0]['class'])
r.json()

i = 9
df_slice = text_raw_df.iloc[i*100:(i+1)*100,:]
tags = ['class']*df_slice.shape[0]


index = list(range(i*100,(i+1)*100))
final = pd.concat([pd.DataFrame(df_slice), pd.Series(tags, name='tags', index=index)], axis=1)
final
# pd.DataFrame(df_slice)
# pd.Series(tags, index=list(range(900, 1000)))

def pickleToJson(path):
    df = None
    with open(path, "rb") as output_file:
        df = pk.load(output_file)
    jsonFileName = path.replace(".pkl", ".json")
    df.to_json(jsonFileName, orient="records", lines=True)


pickleToJson('./datasets/all_data_en.pkl')

StarcoderdataJupyterScriptsDedupFiltered