# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Environment (conda_transformers)
#     language: python
#     name: conda_transformers
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/joshuacalloway/csc575project/blob/main/new_toxic_predict.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="BWyzOw49wNfq" colab={"base_uri": "https://localhost:8080/"} outputId="f30a3209-eba8-493e-a36b-126423e0b424"
# !pip install transformers

# + id="-B72kOoE1_Rk" colab={"base_uri": "https://localhost:8080/"} outputId="f906d482-7566-4006-a0e9-cc2c9fca738e"
# !git clone https://github.com/NVIDIA/apex


# + id="Uu9oirwQ2NU1" colab={"base_uri": "https://localhost:8080/"} outputId="476b0d31-87ef-4408-9dc8-f34f321bea91"
# %cd apex


# + id="DELb49uw2OAn" colab={"base_uri": "https://localhost:8080/"} outputId="b536bb8d-fb7b-4514-e307-810cbc782833"
# !pip install -v --disable-pip-version-check --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./

# + id="vI14dMF30dgv"
# !pip install fast-bert

# + id="U-mM8UK_0c-A"


# + id="wR32Yz4MwbUy"
# !pip install torch

# + id="Etx-NdNeu-mk"
from transformers import BertTokenizer
from pathlib import Path
import torch

from box import Box
import pandas as pd
import collections
import os
from tqdm import tqdm, trange
import sys
import random
import numpy as np
import apex
from sklearn.model_selection import train_test_split

import datetime

from fast_bert.modeling import BertForMultiLabelSequenceClassification
from fast_bert.data_cls import BertDataBunch, InputExample, InputFeatures, MultiLabelTextProcessor, convert_examples_to_features
from fast_bert.learner_cls import BertLearner
from fast_bert.metrics import accuracy_multilabel, accuracy_thresh, fbeta, roc_auc

# + id="A0Iwbxb4u-mm"
torch.cuda.empty_cache()

# + id="ohwgyQ9vu-mm"
pd.set_option('display.max_colwidth', -1)
run_start_time = datetime.datetime.today().strftime('%Y-%m-%d_%H-%M-%S')

# + id="OXFs1y04u-mn"
DATA_PATH = Path('../data/')
LABEL_PATH = Path('../labels/')

AUG_DATA_PATH = Path('../data/data_augmentation/')

MODEL_PATH=Path('../models/')
LOG_PATH=Path('../logs/')
MODEL_PATH.mkdir(exist_ok=True)

model_state_dict = None

# BERT_PRETRAINED_PATH = Path('../../bert_models/pretrained-weights/cased_L-12_H-768_A-12/')
BERT_PRETRAINED_PATH = Path('../../bert_models/pretrained-weights/uncased_L-12_H-768_A-12/')
# BERT_PRETRAINED_PATH = Path('../../bert_fastai/pretrained-weights/uncased_L-24_H-1024_A-16/')
# FINETUNED_PATH = Path('../models/finetuned_model.bin')
FINETUNED_PATH = None
# model_state_dict = torch.load(FINETUNED_PATH)

LOG_PATH.mkdir(exist_ok=True)

OUTPUT_PATH = MODEL_PATH/'output'
OUTPUT_PATH.mkdir(exist_ok=True)

# + id="CrszHIi1u-mn"
args = Box({
    "run_text": "multilabel toxic comments with freezable layers",
    "train_size": -1,
    "val_size": -1,
    "log_path": LOG_PATH,
    "full_data_dir": DATA_PATH,
    "data_dir": DATA_PATH,
    "task_name": "toxic_classification_lib",
    "no_cuda": False,
    "bert_model": BERT_PRETRAINED_PATH,
    "output_dir": OUTPUT_PATH,
    "max_seq_length": 512,
    "do_train": True,
    "do_eval": True,
    "do_lower_case": True,
    "train_batch_size": 8,
    "eval_batch_size": 16,
    "learning_rate": 5e-5,
    "num_train_epochs": 6,
    "warmup_proportion": 0.0,
    "no_cuda": False,
    "local_rank": -1,
    "seed": 42,
    "gradient_accumulation_steps": 1,
    "optimize_on_cpu": False,
    "fp16": True,
    "fp16_opt_level": "O1",
    "weight_decay": 0.0,
    "adam_epsilon": 1e-8,
    "max_grad_norm": 1.0,
    "max_steps": -1,
    "warmup_steps": 500,
    "logging_steps": 50,
    "eval_all_checkpoints": True,
    "overwrite_output_dir": True,
    "overwrite_cache": False,
    "seed": 42,
    "loss_scale": 128,
    "task_name": 'intent',
    "model_name": 'xlnet-base-cased',
    "model_type": 'xlnet'
})

# + id="b4pOPKFRu-mn"
import logging

logfile = str(LOG_PATH/'log-{}-{}.txt'.format(run_start_time, args["run_text"]))

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
    datefmt='%m/%d/%Y %H:%M:%S',
    handlers=[
        logging.FileHandler(logfile),
        logging.StreamHandler(sys.stdout)
    ])

logger = logging.getLogger()

# + id="fagACcSVu-mo"
logger.info(args)

# + id="Mw2ycA-1u-mo"
# tokenizer = BertTokenizer.from_pretrained(BERT_PRETRAINED_PATH, do_lower_case=args['do_lower_case'])

# + id="W9PzyCu8u-mo"
device = torch.device('cuda')
if torch.cuda.device_count() > 1:
    args.multi_gpu = True
else:
    args.multi_gpu = False

# + id="MBYM6cm0u-mp"
label_cols = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]

# + id="AaioWvMfu-mp"
from fast_bert.prediction import BertClassificationPredictor

# + id="FcTs72HMu-mp"
predictor = BertClassificationPredictor(args.output_dir/'model_out', args.output_dir, LABEL_PATH, 
                                        multi_label=True, model_type='xlnet', do_lower_case=False)

# + id="C3mhITGJ6CSH"
predictor = BertClassificationPredictor('../models/output/model_out', args.output_dir, LABEL_PATH, model_type='xlnet', do_lower_case=False)

# + id="zxatam6v6VJR"
args.output_dir

# + id="HPuqq1nxu-mq"
output = predictor.predict_batch(list(pd.read_csv("../data/test.csv")['comment_text'].values))

# + id="OYDUckN5u-mq"
pd.DataFrame(output).to_csv('../data/output_xlnet.csv')

# + id="gc2thPt8u-mq"
results = pd.read_csv('../data/output_xlnet.csv')

# + id="VMZn1p7au-mq"
preds = pd.DataFrame([{item[0]: item[1] for item in pred} for pred in output])

# + id="BpTkZs_Cu-mq"
preds.head()

# + id="WVtt4KQKu-mr"
test_df = pd.read_csv("../data/train.csv")
test_df.head()

# + id="-4fsuam4u-mr"
output_df = pd.merge(test_df, preds, how='left', left_index=True, right_index=True)
del output_df['comment_text']

# + id="qimvQm9uu-mr"
columns = ['id','toxic','severe_toxic','obscene','threat','insult','identity_hate']

# + id="sRTxYkqNu-mr"
output_df = output_df[columns]

# + id="jj2DZkYBu-ms"
output_df.to_csv('../data/output_xlnet.csv', index=None)

# + id="j4oA65xwu-ms"
pd.read_csv('../data/output_xlnet.csv', index_col='id')

# + id="bTBjuCr4u-ms"


StarcoderdataJupyterScriptsDedupFiltered