# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import pickle

# !pip install html2text

import html2text

pd.set_option('display.max_colwidth', 5000)

df = pd.read_pickle('/Users/hazem/Documents/manzur/data/pickles/enabbaladi/posts/posts_1_0.pkl')

df.columns


# +
def read_tags(tag_file_num=9):
    tags = dict()
    for i in range(tag_file_num):
        file = f'/Users/hazem/Documents/manzur/data/pickles/enabbaladi/tags/tag_{i}.pkl'
        new_tags = read_tags_file(file)
        tags = {**tags, **new_tags}
    return tags

tags = read_tags(8)
len(tags)


# +
def read_tags_file(file):
    cats = dict()
    df = pd.read_pickle(file)
    df['id'] = df['id'].apply(int)
    df['id'] = df['id'].apply(str)
    df = df[['id','name', 'count']]
    df = df.set_index('id')
    names = df.to_dict()['name']
    counts = df.to_dict()['count']
    for k in names:
        if counts.get(k,0) >=1:
            cats[k.strip()] = {'name': names[k], 'count': int(counts.get(k,0))}
    return cats

file = f'/Users/hazem/Documents/manzur/data/pickles/enabbaladi/categories.pkl'
cats = read_tags_file(file)
len(cats)


# -

def clean_post_file(file):
    h = html2text.HTML2Text()
    h.ignore_links = True
    df = pd.read_pickle(file)
    df['title'] = df['title'].apply(lambda x: dict(x)['rendered'].strip())
    df['title'] = df['title'].apply(h.handle)
    df['content'] = df['content'].apply(lambda x: dict(x)['rendered'])
    df['html_content'] = df['content']
    df['content'] = df['content'].apply(h.handle)
    df['content'] = df['content'].str.replace('\n', ' ')
    df['excerpt'] = df['excerpt'].apply(lambda x: dict(x)['rendered'])
    df['excerpt'] = df['excerpt'].apply(h.handle)
    df['tags_str'] = df['tags'].apply(lambda x: '|'.join([tags.get(str(e), dict()).get('name', str(e)) for e in list(x)]))
    df['categories_str'] = df['categories'].apply(lambda x: '|'.join([cats.get(str(e), dict()).get('name', str(e)) for e in list(x)]))
    df['author'] = df['author'].apply(int).apply(str) 
    df['id'] = df['id'].apply(int).apply(str) 
    df['realted_posts'] = df['jetpack-related-posts'].apply(lambda x: '|'.join([str(e['id']) for e in list(x)]))
    
    return df[['id','title','content', 'html_content', 'excerpt','categories','author','date_gmt', 
                'tags','tags_str', 'categories_str', 'realted_posts']]



file = '/Users/hazem/Documents/manzur/data/pickles/enabbaladi/posts/posts_1_0.pkl'
df = clean_post_file(file)

# ! pip install requests

import json
import requests
url = 'https://farasa.qcri.org/webapi/segmentation/'
text = 'يُشار إلى أن اللغة العربية' 
api_key = '<KEY>'
payload = {'text': text, 'api_key': api_key}
headers = {'content-type': 'application/json'} 
data = requests.post(url, data=json.dumps(payload), headers=headers)
result = data.text
print(result) 

StarcoderdataJupyterScriptsDedupFiltered