# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # PyConversations: A Reddit-based Example
#
# The following is a tutorial notebook that demonstrates how to use `pyconversations` with Reddit data.
#
# The first step will be to obtain some data. In order to do so, you will need to configure a personal application via your Reddit account's [App Preferences](https://www.reddit.com/prefs/apps). You'll want to set up a personal usage script. See the _Getting Access_ portion of [this blog](https://towardsdatascience.com/how-to-use-the-reddit-api-in-python-5e05ddfd1e5c) for additional instructions/visualization.

# +
from pprint import pprint

from pyconversations.convo import Conversation
from pyconversations.message import RedditPost
# -

# ## Data Sample
#
# Before demonstating how we can use `pyconversations` for pre-processing and analysis, we need to obtain a data sample. 
# To do so, we'll be using a package called [praw](https://github.com/praw-dev/praw).

import praw

# Private information
CLIENT_ID = ''  # this should be the 'personal use script' on your App Preferences
SECRET_TOKEN = ''  # this should be the 'secret' on your App Preferences
USER_AGENT = ''  # a custom name for your application for the User-Agent parameter in the request headers; gives a brief app description to Reddit

# +
# configure a read-only praw.Reddit instance
reddit = praw.Reddit(
    client_id=CLIENT_ID,
    client_secret=SECRET_TOKEN,
    user_agent=USER_AGENT,
)

reddit, reddit.read_only

# +
# obtain a sub-reddit of interest
sub_name = 'Drexel'
subreddit = reddit.subreddit(sub_name)

subreddit.title  # PRAW is lazy so won't request till we ask for an attribute

pprint(vars(subreddit))

# +
# get the top submission via 'hot'
top_submission = list(subreddit.hot(limit=1))[0]
top_submission.title

pprint(vars(top_submission))

# +
# get all the comments on this submission
all_comments = top_submission.comments.list()

print(len(all_comments))

all_comments[0].score
pprint(vars(all_comments[0]))
# -

# ## Integration with `pyconversations`
#
# All that's left to do is plug our data directly into `pyconversations`!

# construct a conversation container
conv = Conversation()

# parse our root submission
cons_params = {
    'lang_detect': True,  # whether to enable the language detection module on the text
    'uid': top_submission.id,  # the unique identifier of the post
    'author': top_submission.author.name if top_submission.author is not None else None,  # name of user
    'created_at': RedditPost.parse_datestr(top_submission.created),  # creation timestamp
    'text': (top_submission.title + '\n\n' + top_submission.selftext).strip()  # text of post
}
top_post = RedditPost(**cons_params)
top_post

# add our post to the conversation
conv.add_post(top_post)

# which we can easily access via the .posts property, to verify inclusion
conv.posts

# +
# iterate through comments and add them to the conversation
for com in all_comments:
    conv.add_post(RedditPost(**{
        'lang_detect': True,  # whethher to enable the language detection module on the txt
        'uid': com.id,  # the unique identifier of the post
        'author': com.author.name if com.author is not None else None,  # name of user
        'created_at': RedditPost.parse_datestr(com.created),  # creation timestamp
        'text': com.body.strip(),  # text of post
        'reply_to': {com.parent_id.replace('t1_', '').replace('t3_', '')}  # set of IDs replied to
    }))

len(conv.posts)
# -

# ### Sub-Conversation Segmentation

# +
# seperate disjoint conversations (there is likely just the one with a full query from the site...)
segs = conv.segment()

len(segs)
# -

# ### (Detected) Language Distribution

# +
from collections import Counter

lang_dist = Counter([post.lang for post in conv.posts.values()])
lang_dist
# -

# ### Conversation-Level Redaction
#
# Using `Conversation.redact()` produces a thread that is cleaned of user-specific information. 
# This is conversationally-scoped, so all usernames are first enumerated (either from author names or from in-text reference for Reddit and Twitter) and then user mentions (and author names) are replaced by `USER{\d}` where `{\d}` is the integer assigned to that username during the enumeration stage.
#
# Here's a demonstration:

# +
# pre-redaction 
names = {post.author for post in conv.posts.values()}

len(names), names
# -

# redaction step
conv.redact()

# post-redaction
names = {post.author for post in conv.posts.values()}
len(names), names

# ### Saving and Loading from the universal format

import json

# saving a conversation to disk
# alternatively: save as a JSONLine file, where each line is a conversation!
json.dump(conv.to_json(), open('reddit_conv.json', 'w+'))

# reloading directly from the JSON
conv_reloaded = Conversation.from_json(json.load(open('reddit_conv.json')))
len(conv_reloaded.posts)

# ### Feature Extraction
#
# The remainder of this notebook exhibits some basic vectorization of features from conversations, posts, and users within this conversation. 
# For more information, see the documentation for PyConversations.

from pyconversations.feature_extraction import ConversationVectorizer
from pyconversations.feature_extraction import PostVectorizer
from pyconversations.feature_extraction import UserVectorizer

# +
convs = True
# convs = False

users = True
# users = False

posts = True
# posts = False

# normalization = None
# normalization = 'minmax'
# normalization = 'mean'
normalization = 'standard'

# cv = ConversationVectorizer(normalization=normalization, agg_user_fts=users, agg_post_fts=posts, include_source_user=True)
pv = PostVectorizer(normalization=normalization, include_conversation=convs, include_user=users)
# uv = UserVectorizer(normalization=normalization, agg_post_fts=posts)
# -

# cv.fit(conv=conv_reloaded)
pv.fit(conv=conv_reloaded)
# uv.fit(conv=conv_reloaded)

# +
# cxs = cv.transform(conv=conv_reloaded)
pxs = pv.transform(conv=conv_reloaded)
# uxs = uv.transform(conv=conv_reloaded)

# pprint(cxs.shape)
pprint(pxs.shape)
# pprint(uxs.shape)

StarcoderdataJupyterScriptsDedupFiltered