# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import string
import numpy as np
import pandas as pd

# +
df = pd.read_json("Sarcasm_Headlines_Dataset.json", lines=True)
display(df.head())
print(df.shape)

df2 = pd.read_json("Sarcasm_Headlines_Dataset_v2.json", lines=True)
display(df2.head())
print(df2.shape)
# -

sum([i in df2.headline for i in df.headline])

df_all = pd.concat([df, df2]).reset_index(drop=True)
df_all.shape

df_clean = df_all.drop(['article_link'], axis=1)
df_clean.head()

df_clean['headline'] = df_clean['headline'].apply(lambda x: x.lower())

df_clean['len'] = df_clean['headline'].apply(lambda x: len(x.split(" ")))
df_clean.head()

for i in string.punctuation:
    df_clean['headline'] = df_clean['headline'].apply(lambda x: x.replace(i, ""))

df_clean

df_clean.groupby('is_sarcastic').describe()

df_clean[df_clean.len==151].iloc[0].headline

df_clean[df_clean.len==2]

sen = pd.read_csv('vader_lexicon.txt', 
                   sep='\t',
                   usecols=[0, 1], 
                   header=None, 
                   names=['token', 'polarity'],
                   index_col='token'
                  )
sen.head()

tidy_format = (
    df_clean['headline']
    .str.split(expand=True)
    .stack()
    .reset_index(level=1)
    .rename(columns={'level_1': 'num', 0: 'word'})
)
tidy_format.head()

df_clean['polarity'] = (
    tidy_format
    .merge(sen, how='left', left_on='word', right_index=True)
    .reset_index()
    .loc[:, ['index', 'polarity']]
    .groupby('index')
    .sum()
    .fillna(0)
)
df_clean.head()

df_clean.groupby('is_sarcastic').describe()



StarcoderdataJupyterScriptsDedupFiltered