# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import glob
import datetime as dt

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
# -

dfs = []
for fname in glob.glob('../data/gamethreads/*.pkl.gz'):
    with open(fname, 'rb') as fid:
        df = pd.read_pickle(fid, compression='gzip')
    dfs.append(df)
df = pd.concat(dfs)
print(df.shape)
df.head()

df["timestamp"] = df.created.apply(lambda x:dt.datetime.fromtimestamp(x)-dt.timedelta(hours=5))
df["is_post"] = df.title.apply(lambda x:"post " in x.lower())
df.sort_values("timestamp", inplace=True)
df[["timestamp","id","title","is_post","ups","num_comments"]].head(20)

df["timestamp"] = df.created.apply(lambda x:dt.datetime.fromtimestamp(x)-dt.timedelta(hours=10))
print(df.shape)
df[["timestamp","id","title","ups","num_comments"]].head()

# +
from sklearn.linear_model import LinearRegression

fig, axs = plt.subplots(1,2,figsize=(15,6))
axs[0].hist(np.log10(df.loc[df.is_post].num_comments.values+1), bins=np.linspace(0,4,51), histtype='step', lw=2, label="Post Game Thread")
axs[0].hist(np.log10(df.loc[~df.is_post].num_comments.values+1), bins=np.linspace(0,4,51), histtype='step', lw=2, label="Game Thread")
axs[0].legend(fontsize='x-large')
axs[0].set_xlabel("log10(#comments + 1)")
axs[1].scatter

StarcoderdataJupyterScriptsDedupFiltered