# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: venv
#     language: python
#     name: venv
# ---

# + [markdown] id="Sh79750o1d5Q"
# ## Humor detection challenge from Kaggle

# + colab={"base_uri": "https://localhost:8080/"} id="-aPlsbH61d5S" outputId="1d4bc53d-fd05-47e3-9f1a-6e9ef47386a9"
# !wget https://github.com/ravi-ilango/acm-dec-2020-nlp/blob/main/lab5/humor_data.zip?raw=true -O humor_data.zip

# !unzip humor_data.zip

# + colab={"base_uri": "https://localhost:8080/"} id="GXs1m48l1pHo" outputId="6e86a887-3128-4cf8-ba46-ab905dd4cb35"
# !pip install transformers
# !pip install sentencepiece

# + id="NTsRiT811d5S"
import os 

import pandas as pd

import numpy as np

import matplotlib.pyplot as plt

import torch

from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler

from sklearn.model_selection import train_test_split


# + id="c3u7mgq21ojD"


# + [markdown] id="HMv0vO5J1d5T"
# ## LAB
#
#
# - Use the disaster_detection_bert as a reference 
#
# - Load input data, tokenize and prepare tensors for training
#
# - Instantiate a XLNet pretrained model
#
# - Predict humor for few sentences
#
# Note: Use 10K data from dataset, 
#       train/validation split of 70/30,
#       batch_size 32, 
#       train for 3 epochs
# -



# + [markdown] id="GCgP3s971d5T"
# ### Check sample code from huggingface illustrating the use of XLNet
# (https://huggingface.co/transformers/model_doc/xlnet.html#xlnetforsequenceclassification)

# + colab={"base_uri": "https://localhost:8080/", "height": 271, "referenced_widgets": ["e5e3ef156a6141478d2f9efedea115f1", "d5b89b0ce39b43d4a0639e87c7c1a3ec", "<KEY>", "803d56664857403d854549b44dac2015", "<KEY>", "6fc2a4e85d5a4dcfb0035378793ced49", "<KEY>", "35245c79aae7431a9a7d53ce2381c6c9", "<KEY>", "<KEY>", "<KEY>", "23eecc58212845efbe05a92193833a36", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "3a136cdd76a7422db8d7d80266e79dbe", "<KEY>", "921706e29d2740ddab39a9dfeddf4b4e", "57283265cc5a4bd188e9fa57e3d791da", "<KEY>", "<KEY>", "<KEY>"]} id="FpgUKqNS1d5T" outputId="e601ff0d-553b-4225-8de4-b4d768557707"
from transformers import XLNetTokenizer, XLNetForSequenceClassification, AdamW

tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')
model = XLNetForSequenceClassification.from_pretrained('xlnet-base-cased')

inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
outputs = model(**inputs, labels=labels)

loss = outputs.loss
logits = outputs.logits

# + [markdown] id="Aqql63H01d5T"
# ### Load data
#

# + colab={"base_uri": "https://localhost:8080/"} id="AGfSNmYD1d5U" outputId="d28b0ecb-5cf4-4a1b-a1de-26424ee2e7cf"
#
# text are stored in the variable text
# humor or not labels are stored in the variable humor
#
df = pd.read_csv('./humor_data/dataset.csv')

df = df[:10000]

text = df.text.values
labels = df['humor'].apply(lambda x: 1 if x else 0).values

print(text.shape)

# + id="uyWu_q0K1d5U"


# + [markdown] id="rq2joCeN3OF8"
# ### Plot target distribution

# + colab={"base_uri": "https://localhost:8080/", "height": 295} id="ZQA-GxAm1d5U" outputId="9585ed73-6ad9-4b98-9318-8f34ba2f3482"

# %matplotlib inline

plt.hist(labels)
plt.xlabel('humor')
plt.ylabel('count')
plt.title('target distribution')
plt.xticks(np.arange(len(np.unique(labels))));


# + [markdown] id="DRsptOu01d5V"
# ### Prepare Data
#

# + colab={"base_uri": "https://localhost:8080/"} id="P8pTbQ0g1d5V" outputId="4317db7c-1263-4437-82ce-470d66c4086d"
# Tokenize with XLNet tokenizer
tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased', do_lower_case=True)
padded_sequences = tokenizer(list(text), padding=True)
print (f"tokenized inputs {padded_sequences['input_ids'][0]}")


# + colab={"base_uri": "https://localhost:8080/"} id="IWgVrKoj1d5V" outputId="8c4140f5-6abb-421a-92df-27cb4ed69f32"
padded_sequences.keys()

# + id="ZGcBrCF41d5V"


# + [markdown] id="shrjqfS_1d5W"
# ### Split into training and validation datasets

# + id="ufyfnzap1d5W"
# include <your code>



# + [markdown] id="tg0wZslv1d5W"
# ### Convert to tensors

# + id="D0CGOxTS1d5W"
# include <your code>


# + id="S5Yb98991d5W"


# + [markdown] id="fhz6Gt4f1d5X"
# ### Prepare training data generators

# + id="N70qN60y1d5X"
# include <your code>



# + [markdown] id="7qkGn62N1d5X"
# ### Load pretrained XLNet model, setup optimizer
#

# + colab={"base_uri": "https://localhost:8080/"} id="MG3xkIN-1d5X" outputId="1b2b9973-08fb-4be9-de54-e8156615995f"
# specify GPU device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

if device.type == 'cuda':
    print (torch.cuda.get_device_name(0))

# include <your code>


# + id="xzGdNfio1d5X"
model_path = './xlnet_humor_detection_state_dict.pth'

if os.path.exists(model_path):
    print ("Loading weights from saved model...")
    model.load_state_dict(torch.load(model_path, map_location=device))

# + id="jB2YAHk91d5Y"


# + [markdown] id="FNM91i2l1d5Y"
# ### Model Train function 

# + id="cuUBpUTK1d5Y"
# include <your code>


# + id="bQx3rc1b1d5Y"


# + [markdown] id="su3YWus61d5Z"
# ### Model Evaluate function

# + id="u4MMW_Xv1d5Z"
# include <your code>



# + id="5Dxnl02P1d5Z"


# + [markdown] id="I01vBLlY1d5Z"
# ### Train the model

# + colab={"base_uri": "https://localhost:8080/"} id="rxEyOKRv1d5a" outputId="06c64ebb-ad8f-4868-be03-2db3a38ff439"
from tqdm import trange

# include <your code>



# + id="Py7o08SD1d5a"


# + [markdown] id="_-ueN0X-1d5a"
# ### Predict

# + id="mR3Z-oR51d5a"
if os.path.exists(model_path):
    print ("Loading weights from saved model...")
    model.load_state_dict(torch.load(model_path, map_location=device))

model.to('cpu')
model.eval()

def predict(model, sentence):
    inputs = tokenizer(sentence, return_tensors="pt")
    output = model(**inputs)
    logits = output.logits
    logits = logits.detach().cpu().numpy()

    pred = np.argmax(logits, axis=1)[0]
    return 'Humor' if pred == 1 else 'Not a humor'


# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="tQy_9xpJ1d5a" outputId="5fea46d1-56d4-48b0-ac10-5abb30dd184e"
#Humor or not question
predict(model, "Where do eskimos keep their money? in snowbanks.")


# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="rltNoKt-1d5b" outputId="a862c783-030a-43ac-8a62-e51d243e70c0"
#Humor or not question
predict(model, "How many optometrists does it take to screw in a lightbulb? one... or two? one... or two?")


# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="QtFqoAulEEBw" outputId="8e00a330-09ab-4eb2-d737-f0c622f7b658"
#Humor or not question
predict(model, "Mcdonald's will officially kick off all-day breakfast on october 6")

# + colab={"base_uri": "https://localhost:8080/", "height": 35} id="qd2EhqbYEU84" outputId="1812e822-d3c2-480c-932f-99c4152026b9"
#Humor or not question
predict(model, "The journey to labor land: how women can reclaim their birth rights")

# + id="Kr_nTy12DU-u"


StarcoderdataJupyterScriptsDedupFiltered