# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3.9.4 64-bit
#     name: python3
# ---

# +
from pathlib import Path
import requests
from wikitalk_parser import WikiParserThreads

HERE = Path('.')
# -

# ## Monty Python [talk page](https://en.wikipedia.org/wiki/Talk:Monty_Python) 
#
# Download Cirrus Soc with Wiki markup of the talk page.

resp = requests.get('https://en.wikipedia.org/w/api.php', params=dict(
    action='query',
    format='json',
    prop='cirrusdoc',
    titles='Talk:Monty_Python'
))
data = resp.json()
source = data['query']['pages']['18949']['cirrusdoc'][0]['source']['source_text']

# Using the parser is very easy. First an instance needs to be initialized on
# a string with Wiki markup of a page. And then `.parse()` method can be called
# which returns a generator of subsequent posts. Hierarchy of threads and
# posts within threads is retained by a set of additional index fields.
#
# Each post is a simple `dict` object with the following fields:
#
# 1. `topic`: thread title/topic
# 2. `thread_idx`: thread index according to the actual order on page
#     and counting from `1`.
# 3. `post_idx`: index of a post within a thread according to the order of appearance.
# 4. `parent_idx`: index of the parent post.
# 5. `user_name`: user name of the author.
# 6. `timestamp`: post creation timestamp as `datetime` object.
# 7. `depth`: depth within the discussion tree.
# 8. `content`: raw content of the post as extrected from Wiki markup.
# 9. `content_sanitized`: content cleaned from most of Wiki and HTML markup.
#
# Note that thanks to the fact that the data is returned as a simple flat
# list of homogeneous `dict` objects output of the parser can be easily fit
# into different convenient data structures such as 
# [Pandas](https://pandas.pydata.org/) data frames.

threads = WikiParserThreads(source)
list(threads.parse())

StarcoderdataJupyterScriptsDedupFiltered