# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

"""
*******************************************************
Title: Python Notebook to download posts associated with public profiles

Organization: DANE
Author: <NAME>
Version: 3.0
Modification date: 07/21/2021
Descripción:
    [Sec 1] Libraries
    [Sec 2] Data Gathering
    [Sec 3] Dataset preprocessing and building
    
    Returns:
        Posts dataset for each profile
*******************************************************
"""

# # 1. Libraries

# +
#Library to scrape data from Facebook
from facebook_scraper import *

#Library to estimate process progress
from tqdm import tqdm

#Data handling libraries
import pandas as pd
import numpy as np

#Data handling, time and random number generator libraries
import json, time, random
# -

# # 2. Data Gathering

# +
"""
ok:
* Politics
gustavopetrourrego
AlvaroUribeVel
ivanduquemarquez
JuanManSantosC

* News
elespectadorcom
eltiempo
CanalRCN
NoticiasCaracol
Canal1Col
RevistaSemana

* Public order
Policianacionaldeloscolombianos
ejercitocolombia
FiscaliaCol

* Economics
RevistaDineroCol
Bancolombia
BancoDavivienda

* Goverment
MinisterioDeHaciendaYCreditoPublico
DIANCol
DANEColombia
RegistraduriaNacional
departamentonacionaldeplaneacion

* Sports
FCFSeleccionColPage
millosfcoficial
nacionaloficial
independiente.santafe
Americadecalisad
JuniorClubSA
DeportivoCaliOficial
oncecaldasoficial
fedecoltenis

* Artists
AMIGOSDEALEXCAMPOS
juanpisgonzalezWTF

* Other
parquemundoaventura
ParqueSalitreMagico
icetexcolombia
==============
--Noticias
--Finanzas
--Entidades Govierno
--Deportes
--Artistas

==============
error:
shakira
pirryoficial
10Jamesrodriguez
MALUMAMUSIK
GarciaZarateRadamelFalcao
Juanes
JBalvinOficial
susoelpaspioficial
lopezman
marianapajon
"""


"""
New profiles

Mayors
ClaudiaLopezCL
DQuinteroCalle
JorgeIvanOspin
jaimepumarejoheins
JCardenasRey
JoseManuelRiosAlcalde
WilliamDauAlcalde
andreshurtadoalcalde
AlcaldeJorgeMendoza
jaimepumarejoheins
RuizLuisAntonio
quelleguelobueno
carlospenagos.si
WilliamDauAlcalde
MelloCastroG
JoseRamiroBermudezC
JohnsonVirna
gorkymunozc
andreshurtadoalcalde
EdgarTovarPedraza
lecYopal
harmanfelipe
Jair-Esteban-Beltran-Hinojosa-344509085732777
jc.lopezcastrillon
germanchamorrodlr
Martin-Emilio-Sánchez-Valencia-2251716985096279
JorgeIvanOspin
garialcalde2023

Public opinion
Antonio-Caballero-58050573565
GermanVargasLleras
PrensaJuanLozano
#31 perfiles nuevos
---------------------
error
ingjairoyanez
AlcaldeDeManizales
FunemeAlejandro
pablo.yuvabe
carlosandres.guevaragomez.5
MayaAlcaldePereira
ingjairoyanez

Public opinion
DanielCoronellPeriodista
mjduzan
DanielSamperO
saludhernandezmora1
vickydavilah
paginadesilvaromero
"""

listposts = []

#It's called the get_posts function from facebook_scraper over the last 1000 pages
#In between paging, wait 1 to 3 seconds to mitigate blocking by multiple requests.
for post in tqdm(get_posts("garialcalde2023", pages=1000, extra_info=True, timeout=10)):
    try:
        time.sleep(random.randint(0,3))
        listposts.append(post)
    except:
        pass
# -

#Check the number of gathered posts
len(listposts)

#Lets check the data keys
listposts[0].keys()

#Lets check the first post id
listposts[0]['post_id']


# # 3. Dataset preprocessing and building

def check_ei(element, cols):
    '''
    Function to check if the element exists
    
    Get a json dict and a list of interest columns
    
    Iterate over each column, if the data exists, add the value to a list, otherwise add an error message
    
    Args:
        element (dict):   Json dictionary with data
        cols (list):      List with interes columns
    Returns:
        n_row_list (list):  List with pre processed and organized data
    '''
    n_row_list = []
    for x in cols:
        try:
            val = element[x]
        except:
            val = 'void_error'
        n_row_list.append(val)
    return n_row_list


# +
#Interest columns definition
df_cols = ['post_id', 'text', 'post_text', 'shared_text', 'time', 'image',
            'video', 'video_thumbnail', 'video_id', 'likes', 'comments',
            'shares', 'post_url', 'link', 'user_id', 'username', 'is_live',
            'factcheck', 'shared_post_id', 'shared_time', 'shared_user_id',
            'shared_username', 'shared_post_url', 'available', 'images',
            'reactions', 'w3_fb_url', 'fetched_time']
#Post dataframe creation
posts_df = pd.DataFrame(columns=df_cols)

#For used to check elements and construct the posts dataset
for element in tqdm(listposts):
    try:
        element_list = check_ei(element, df_cols)
        n_row = pd.Series(element_list, index=posts_df.columns)
        posts_df = posts_df.append(n_row, ignore_index=True)
    except:
        pass
# -

#Lets check the first 5 rows
posts_df.head()

#Lets check the last 5 rows
posts_df.tail()

#Lets save the posts dataset for each profile
posts_df.to_csv('./bases/popular_posts/garialcalde2023_fb_posts_06_10_2021.csv')



StarcoderdataJupyterScriptsDedupFiltered