# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# ### Import Libraries

# + colab={"base_uri": "https://localhost:8080/"} id="ap3antBdhxKw" outputId="a9a3f59b-3b5b-4225-9593-8d91fbf8e41e"
import requests
import pandas as pd
import warnings
warnings.filterwarnings('ignore')
# -

# ### Load data

# + colab={"base_uri": "https://localhost:8080/", "height": 224} id="Kj9jc_0I4das" outputId="d2433386-bb44-4928-a66b-7d59f2df59c1"
df = pd.read_csv('data/dialect_dataset.csv')
print(len(df))
display(df.head())
# -

# ### Preprocessing

# + id="DNnUtzShFvvZ"
df['id'] = df['id'].astype(str)
id_list = df['id'].to_list()


# -

# ### Split IDs into batchs of 1000

# + id="Fn_ex4FhyOdZ"
def id_chunks_1000(lst, n):
    for i in range(0, len(lst), n):
        yield lst[i:i + n]


# + colab={"base_uri": "https://localhost:8080/"} id="hQt0CpAM2EKY" outputId="a551f0df-e527-4bbe-db65-f259836d1143"
request_lists = list(id_chunks_1000(id_list,1000))
print(len(request_lists))
print(len(request_lists[-2]))
print(len(request_lists[-1]))
# -

# ### Send POST Request

# + colab={"base_uri": "https://localhost:8080/"} id="FtvOoZ4-ooGx" outputId="6ac59a7c-b5a1-48eb-d06e-8a1a8284e457"
txt_response = []
for lst in request_lists:
    r = requests.post("https://recruitment.aimtechnologies.co/ai-tasks", json= lst)
    print(r.status_code, r.reason)
    json_obj = r.json()
    for id in lst:
        response = json_obj[id]
        txt_response.append(response)
# -

# ### Create Dataframe with response data

# + colab={"base_uri": "https://localhost:8080/", "height": 224} id="_BYS0onRBUXc" outputId="ec0ca8cb-5af5-44e8-96aa-38b40a09e394"
data = pd.DataFrame(list(zip(id_list, txt_response, df['dialect'])), columns =['id', 'txt','dialect'])
print(len(data))
display(data.head())
# -

# ### Save dataframe into Excel

# + colab={"base_uri": "https://localhost:8080/"} id="e8rZVFhb_4u2" outputId="5ce75ce4-c727-4590-b1af-663df3818a9f"
data.to_excel('data/data.xlsx')
print('Done!')

StarcoderdataJupyterScriptsDedupFiltered