# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: PySpark - YARN
#     language: python
#     name: spark_yarn_pyspark
# ---

import matplotlib as mpl
import matplotlib.pyplot as plt
import pyspark.sql.functions as funcs
from pyspark.sql.window import Window

# %matplotlib inline
mpl.style.use({
    "font.size": 16,
    "figure.figsize": (14, 7),
    "axes.grid": True,
    "axes.autolimit_mode": "data",
    "axes.xmargin": 0,
    "axes.ymargin": 0
})

mwh = spark.read.parquet("/wmf/data/wmf/mediawiki/history/snapshot=2019-04")
ns_map_schema = """
    `hostname` string COMMENT 'Canonical URL for the project, for example ja.wikipedia.org', 
    `dbname` string COMMENT 'Database name for the project, for example jawiki', 
    `namespace` int COMMENT 'for example 0, 100, etc.', 
    `namespace_canonical_name` string COMMENT 'the english prefix if exists, otherwise the localized prefix', 
    `namespace_localized_name` string COMMENT 'the localized prefix', 
    `namespace_is_content` int COMMENT 'Whether this namespace is a content namespace'
"""
ns_map = spark.read.csv(
    "/wmf/data/raw/mediawiki/project_namespace_map/snapshot=2019-04", 
    schema=ns_map_schema
)

# +
join_conds = [
    mwh.wiki_db == ns_map.dbname,
    mwh.page_namespace == ns_map.namespace,
    ns_map.namespace_is_content == 1
]

# new editor content edits (nece)
nece = mwh.where("""
    event_entity = "revision" and
    event_type = "create" and
    event_timestamp between "2018-04" and "2019-04" and
    event_user_registration_timestamp between "2018-04" and "2018-10" and
    datediff(event_timestamp, event_user_registration_timestamp) <= 180
""").join(ns_map, join_conds, how="inner")

# +
w = Window.partitionBy([nece.wiki_db, nece.event_user_text]).orderBy(nece.event_timestamp)

fcel = (
    nece
    .select(
        "wiki_db",
        "event_user_text",
        "event_timestamp",
        "event_user_registration_timestamp",
        funcs.row_number().over(w).alias("content_edit_number")
    )
    .where("content_edit_number = 5")
    .selectExpr(
        "wiki_db",
        "event_user_text",
        "datediff(event_timestamp, event_user_registration_timestamp) as days_to_fifth_content_edit"
    )
).toPandas()
# -

fcel.query("wiki_db == 'enwiki'").sample(10)

fcel["days_to_fifth_content_edit"].describe()

plt.hist(
    fcel["days_to_fifth_content_edit"],
    bins=30,
    range=(0, 30)
);




StarcoderdataJupyterScriptsDedupFiltered