# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="sv8fiPZsFa5f"
# # Construcción del modelo final

# + id="26RtC6xJFa5g"
# !pip install transformers
# !pip install multimodal-transformers

import json
import logging
import os
import numpy as np
import pandas as pd
from typing import Optional
from dataclasses import dataclass, field
from multimodal_transformers.data import load_data_from_folder
from multimodal_transformers.model import TabularConfig
from multimodal_transformers.model import AutoModelWithTabular
from transformers.training_args import TrainingArguments
from transformers import (AutoTokenizer, AutoConfig, AutoModelForSequenceClassification, Trainer, EvalPrediction, set_seed)

logging.basicConfig(level=logging.INFO)
os.environ['COMET_MODE'] = 'DISABLED'

# + id="ZpTGMUmAFa5h"
data_df = pd.read_csv('df_final_3391.csv')
train_df, val_df, test_df = np.split(data_df.sample(frac=1), [int(.8*len(data_df)), int(.9 * len(data_df))]) #8:1:1, cambiar
print('Num examples train-val-test')
print(len(train_df), len(val_df), len(test_df))
train_df.to_csv('train.csv')
val_df.to_csv('val.csv')
test_df.to_csv('test.csv')


# + id="Lr3yWObWFa5i"
@dataclass
class ModelArguments:
  """
  Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
  """

  model_name_or_path: str = field(
      metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
  )
  config_name: Optional[str] = field(
      default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
  )
  tokenizer_name: Optional[str] = field(
      default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
  )
  cache_dir: Optional[str] = field(
      default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
  )


@dataclass
class MultimodalDataTrainingArguments:
  """
  Arguments pertaining to how we combine tabular features
  Using `HfArgumentParser` we can turn this class
  into argparse arguments to be able to specify them on
  the command line.
  """

  data_path: str = field(metadata={
                            'help': 'the path to the csv file containing the dataset'
                        })
  column_info_path: str = field(
      default=None,
      metadata={
          'help': 'the path to the json file detailing which columns are text, categorical, numerical, and the label'
  })

  column_info: dict = field(
      default=None,
      metadata={
          'help': 'a dict referencing the text, categorical, numerical, and label columns'
                  'its keys are text_cols, num_cols, cat_cols, and label_col'
  })

  categorical_encode_type: str = field(default='ohe',
                                        metadata={
                                            'help': 'sklearn encoder to use for categorical data',
                                            'choices': ['ohe', 'binary', 'label', 'none']
                                        })
  numerical_transformer_method: str = field(default='box_cox',
                                            metadata={
                                                'help': 'sklearn numerical transformer to preprocess numerical data',
                                                'choices': ['yeo_johnson', 'box_cox', 'quantile_normal', 'none']
                                            })
  task: str = field(default="classification",
                    metadata={
                        "help": "The downstream training task",
                        "choices": ["classification", "regression"]
                    })

  mlp_division: int = field(default=4,
                            metadata={
                                'help': 'the ratio of the number of '
                                        'hidden dims in a current layer to the next MLP layer'
                            })
  combine_feat_method: str = field(default='individual_mlps_on_cat_and_numerical_feats_then_concat',
                                    metadata={
                                        'help': 'method to combine categorical and numerical features, '
                                                'see README for all the method'
                                    })
  mlp_dropout: float = field(default=0.1,
                              metadata={
                                'help': 'dropout ratio used for MLP layers'
                              })
  numerical_bn: bool = field(default=True,
                              metadata={
                                  'help': 'whether to use batchnorm on numerical features'
                              })
  use_simple_classifier: str = field(default=True,
                                      metadata={
                                          'help': 'whether to use single layer or MLP as final classifier'
                                      })
  mlp_act: str = field(default='relu',
                        metadata={
                            'help': 'the activation function to use for finetuning layers',
                            'choices': ['relu', 'prelu', 'sigmoid', 'tanh', 'linear']
                        })
  gating_beta: float = field(default=0.2,
                              metadata={
                                  'help': "the beta hyperparameters used for gating tabular data "
                                          "see https://www.aclweb.org/anthology/2020.acl-main.214.pdf"
                              })

  def __post_init__(self):
      assert self.column_info != self.column_info_path
      if self.column_info is None and self.column_info_path:
          with open(self.column_info_path, 'r') as f:
              self.column_info = json.load(f)


# + id="Rd8bejl6Fa5j"
text_cols = ['text']
cat_cols = ['hater', 'vecino_hater', 'profile_changed', 'verificado', 'clase_NER', 'clase_DESCR',
            'clase_FECHA', 'clase_IMG', 'clase_HASHTAGS', 'clase_CATEGORIAS', 'clase_MENCAT', 'clase_RTSCAT','clase_DOMS'] #  ,'clase_LOC', 'clase_LESP', 'clase_LENG', 'clase_LOTR'
numerical_cols = ['negativos', 'positivos', 'neutros', 'n_hate', 'n_nohate', 'n_baddies',
                  #'activity_weekly_0', 'activity_weekly_1', 'activity_weekly_2', 'activity_weekly_3', 
                  #'activity_weekly_4', 'activity_weekly_5', 'activity_weekly_6', 'activity_hourly_00', 
                  #'activity_hourly_01', 'activity_hourly_02', 'activity_hourly_03', 'activity_hourly_04', 
                  #'activity_hourly_05', 'activity_hourly_06', 'activity_hourly_07', 'activity_hourly_08', 
                  #'activity_hourly_09', 'activity_hourly_10', 'activity_hourly_11', 'activity_hourly_12', 
                  #'activity_hourly_13', 'activity_hourly_14', 'activity_hourly_15', 'activity_hourly_16', 
                  #'activity_hourly_17', 'activity_hourly_18', 'activity_hourly_19', 'activity_hourly_20', 
                  #'activity_hourly_21', 'activity_hourly_22', 'activity_hourly_23', 
                  'eigenvector', 'in_degree',
                  'out_degree', 'degree', 'clustering', 'closeness', 
                  'negativos_score', 'positivos_score', 
                  'neutros_score', 'hate_score', 'no_hate_score', 'statuses_count', 'followers_count', 'followees_count', 
                  'listed_count', 'num_hashtags', 'rt_count', 'num_mentions', 'num_urls', 'len_status', 'times_user_quotes', 
                  'num_rts_to_tweets', 'misspelling_counter', 'leet_counter', 'betweenness', 'status_average_tweets_per_day']
                

column_info_dict = {
    'text_cols': text_cols,
    'num_cols': numerical_cols,
    'cat_cols': cat_cols,
    'label_col': 'label',
    'label_list': [0,1]
}


model_args = ModelArguments(
    model_name_or_path='dccuchile/bert-base-spanish-wwm-cased'
)

data_args = MultimodalDataTrainingArguments(
    data_path='.',
    combine_feat_method='gating_on_cat_and_num_feats_then_sum', #
    column_info=column_info_dict,
    task='classification'
)

training_args = TrainingArguments(
    output_dir="./logs/model_name",
    logging_dir="./logs/runs",
    overwrite_output_dir=True,
    do_train=True,
    do_eval=True,
    per_device_train_batch_size=16,
    num_train_epochs=7, #7
    evaluate_during_training=True,
    logging_steps=25,
    eval_steps=250, #250
    dataloader_drop_last=True
)

set_seed(training_args.seed)

# + id="5vK-xIclFa5k"
tokenizer_path_or_name = model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path
print('Specified tokenizer: ', tokenizer_path_or_name)
tokenizer = AutoTokenizer.from_pretrained(
    tokenizer_path_or_name,
    cache_dir=model_args.cache_dir,
)
#inputs = tokenizer(sentences, padding="max_length", truncation=True)

# + id="q8e52A81ZQDY"
data_args.data_path

# + id="FhZkwaJpFa5k"
# Get Datasets
train_dataset, val_dataset, test_dataset = load_data_from_folder(
    data_args.data_path,
    data_args.column_info['text_cols'],
    tokenizer,
    label_col=data_args.column_info['label_col'],
    label_list=data_args.column_info['label_list'],
    categorical_cols=data_args.column_info['cat_cols'],
    numerical_cols=data_args.column_info['num_cols'],
    sep_text_token_str=tokenizer.sep_token
)

# + id="6hKu8HX9Fa5k"
num_labels = len(np.unique(train_dataset.labels))
num_labels

# + id="2xjDv1FcFa5l"
config = AutoConfig.from_pretrained(
        model_args.config_name if model_args.config_name else model_args.model_name_or_path,
        cache_dir=model_args.cache_dir,
    )
tabular_config = TabularConfig(num_labels=num_labels,
                               cat_feat_dim=train_dataset.cat_feats.shape[1],
                               numerical_feat_dim=train_dataset.numerical_feats.shape[1],
                               **vars(data_args))
config.tabular_config = tabular_config

# + id="mokaBSlLJ_Rb"
model = AutoModelWithTabular.from_pretrained(
    model_args.config_name if model_args.config_name else model_args.model_name_or_path,
    config=config,
    cache_dir=model_args.cache_dir
)

# + id="rAkvwjGNvw_y"
print(model)

# + id="A-fbysl1Fa5l"
import numpy as np
from scipy.special import softmax
from sklearn.metrics import (
    auc,
    precision_recall_curve,
    roc_auc_score,
    f1_score,
    confusion_matrix,
    matthews_corrcoef,
    accuracy_score
)

def calc_classification_metrics(p: EvalPrediction):
  pred_labels = np.argmax(p.predictions, axis=1)
  pred_scores = softmax(p.predictions, axis=1)[:, 1]
  labels = p.label_ids
  if len(np.unique(labels)) == 2:  # binary classification
      roc_auc_pred_score = roc_auc_score(labels, pred_scores)
      #accuracy = accuracy_score(labels, pred_scores)
      precisions, recalls, thresholds = precision_recall_curve(labels,
                                                                pred_scores)
      fscore = (2 * precisions * recalls) / (precisions + recalls)
      fscore[np.isnan(fscore)] = 0
      ix = np.argmax(fscore)
      threshold = thresholds[ix].item()
      pr_auc = auc(recalls, precisions)
      tn, fp, fn, tp = confusion_matrix(labels, pred_labels, labels=[0, 1]).ravel()
      result = {#'accuracy' : accuracy_score,
                'roc_auc': roc_auc_pred_score,
                'threshold': threshold,
                'pr_auc': pr_auc,
                'recall': recalls[ix].item(),
                'precision': precisions[ix].item(), 'f1': fscore[ix].item(),
                'tn': tn.item(), 'fp': fp.item(), 'fn': fn.item(), 'tp': tp.item()
                }
  else:
      acc = (pred_labels == labels).mean()
      f1 = f1_score(y_true=labels, y_pred=pred_labels)
      result = {
          "acc": acc,
          "f1": f1,
          "acc_and_f1": (acc + f1) / 2,
          "mcc": matthews_corrcoef(labels, pred_labels)
      }
  print(result)
  return result


# + id="7Lj-1O71Fa5m"
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=val_dataset,
    compute_metrics=calc_classification_metrics,
)

# + id="iLX85q4bnikj"
# %%time
trainer.train()

# + id="bRTsmRZmdWf7"
trainer.evaluate()

# + id="reV-epkPFa5n"
# Load the TensorBoard notebook extension
# %load_ext tensorboard

# + id="qWX5cu_jFa5n"
# %tensorboard --logdir ./logs/runs --port=6006

StarcoderdataJupyterScriptsDedupFiltered