FEAT: new regression model

app.py

@@ -1,6 +1,7 @@
 import os
 import re
 import numpy as np
+import scipy as sp
 import glob
 import json
 from typing import Dict, List, Tuple, Union
@@ -122,9 +123,14 @@ if "01" in option:
         input_tensor = inference_tokenizer.get_item(context=context, actual_sentence=actual_text)
         output_model = model(**input_tensor.data).logits
 
-        output_model = torch.softmax(output_model, dim=-1).detach().numpy()[0]
-        prop_follow = output_model[0]
-        prop_not_follow = output_model[1]
+        output_model = output_model.detach().numpy()[0]
+        if len(output_model) == 2:  # classification
+            output_model = sp.special.softmax(output_model, axis=-1)
+            prop_follow = output_model[0]
+            prop_not_follow = output_model[1]
+        elif len(output_model) == 1:  # regression
+            prop_follow = 1 - output_model[0]
+            prop_not_follow = 1 - prop_follow
 
         submitted = st.form_submit_button("Submit")
         if submitted:
@@ -165,13 +171,18 @@ if "02" in option or "03" in option or "04" in option or "06" in option:
         submitted = st.form_submit_button("Submit")
         if submitted:
             for idx, datapoint in enumerate(data_for_evaluation):
-                progres_bar.progress(idx/len(data_for_evaluation), text="Inference")
+                progres_bar.progress(idx / len(data_for_evaluation), text="Inference")
                 c, s, human_label = datapoint
                 input_tensor = inference_tokenizer.get_item(context=c, actual_sentence=s)
                 output_model = model(**input_tensor.data).logits
-                output_model = torch.softmax(output_model, dim=-1).detach().numpy()[0]
-                prop_follow = output_model[0]
-                prop_not_follow = output_model[1]
+                output_model = output_model.detach().numpy()[0]
+                if len(output_model) == 2:  # classification
+                    output_model = sp.special.softmax(output_model, axis=-1)
+                    prop_follow = output_model[0]
+                    prop_not_follow = output_model[1]
+                elif len(output_model) == 1:  # regression
+                    prop_follow = 1 - output_model[0]
+                    prop_not_follow = 1 - prop_follow
 
                 results.append((c, s, human_label, prop_follow, prop_not_follow))
                 if human_label == "coherent":
@@ -192,15 +203,21 @@ if "05" in option:
             data_for_evaluation = get_evaluation_data_from_dialogue(_clean_conversational_line(context).split("\n"))
             lines = []
             scores = np.zeros(shape=(len(data_for_evaluation), context_size))
-            for datapoint in data_for_evaluation:
+            for idx, datapoint in enumerate(data_for_evaluation):
+                progres_bar.progress(idx / len(data_for_evaluation), text="Inference")
                 for actual_sentence, contexts in datapoint.items():
                     lines.append(actual_sentence)
                     for c in contexts:
                         input_tensor = inference_tokenizer.get_item(context=c, actual_sentence=actual_sentence)
                         output_model = model(**input_tensor.data).logits
-                        output_model = torch.softmax(output_model, dim=-1).detach().numpy()[0]
-                        prop_follow = output_model[0]
-                        prop_not_follow = output_model[1]
+                        output_model = output_model.detach().numpy()[0]
+                        if len(output_model) == 2:  # classification
+                            output_model = sp.special.softmax(output_model, axis=-1)
+                            prop_follow = output_model[0]
+                            prop_not_follow = output_model[1]
+                        elif len(output_model) == 1:  # regression
+                            prop_follow = 1 - output_model[0]
+                            prop_not_follow = 1 - prop_follow
                         scores[len(lines) - 1][len(c) - 1] = prop_follow
 
             aggregated_result = []
@@ -210,8 +227,10 @@ if "05" in option:
 
 if "07" in option:
     from data.example_data import dbc
+
     select_conversation = st.selectbox("Which dialogue to evaluate", list(range(len(dbc))), index=0)
-    context = st.text_area("Insert dialogue here (one turn per line):", value=json.dumps([dbc[int(select_conversation)]]))
+    context = st.text_area("Insert dialogue here (one turn per line):",
+                           value=json.dumps([dbc[int(select_conversation)]]))
     st.markdown("# Formatted form")
     context_json = json.loads(context)
     output = ""
@@ -236,9 +255,14 @@ if "07" in option:
                         for c in contexts:
                             input_tensor = inference_tokenizer.get_item(context=c, actual_sentence=actual_sentence)
                             output_model = model(**input_tensor.data).logits
-                            output_model = torch.softmax(output_model, dim=-1).detach().numpy()[0]
-                            prop_follow = output_model[0]
-                            prop_not_follow = output_model[1]
+                            output_model = output_model.detach().numpy()[0]
+                            if len(output_model) == 2:  # classification
+                                output_model = sp.special.softmax(output_model, axis=-1)
+                                prop_follow = output_model[0]
+                                prop_not_follow = output_model[1]
+                            elif len(output_model) == 1:  # regression
+                                prop_follow = 1 - output_model[0]
+                                prop_not_follow = 1 - prop_follow
                             scores[len(lines) - 1][len(c) - 1] = prop_follow
 
                 for idx, line in enumerate(lines):

model/new_version_v3/config.json

@@ -0,0 +1,33 @@
+{
+  "_name_or_path": "/home/lorenpe2/project/hf_models/bert-base-uncased",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "regression",
+  "torch_dtype": "float32",
+  "transformers_version": "4.30.0.dev0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

model/new_version_v3/info.json

@@ -0,0 +1,4 @@
+{
+  "model": "BERT-REGRESSION",
+  "description": "Model trained on subset of DailyDialogue, CommonDialogues, ChitChatDataset, AirDialogue and SODA. Using [unused1] token to divide sentences in context. More info can be found at https://wandb.ai/alquist/next-sentence-prediction/runs/66pz87ta/overview?workspace=user-petr-lorenc"
+}

model/new_version_v3/meta-info.json

@@ -0,0 +1,28 @@
+{
+  "args": [],
+  "kwargs": {
+    "model_package": "transformers",
+    "model_class": "AutoModelForSequenceClassification",
+    "data_root": "/home/lorenpe2/project/data",
+    "data_sources": [],
+    "pretrained_model": "bert-base-uncased",
+    "tokenizer": "bert-base-uncased",
+    "approach": "IGNORE_DUPLICITIES",
+    "special_token": "[unused1]",
+    "learning_rate": 5e-07,
+    "warmup_ratio": 0.1,
+    "freeze_prefinetuning": true,
+    "prefinenuting_epoch": 10,
+    "finetuning_epochs": 75
+  },
+  "tokenizer_args": {
+    "padding": "max_length",
+    "max_length_ctx": 256,
+    "max_length_res": 40,
+    "truncation": "only_first",
+    "return_tensors": "np",
+    "is_split_into_words": true,
+    "approach": "IGNORE_DUPLICITIES",
+    "special_token": "[unused1]"
+  }
+}

model/new_version_v3/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e3abc0ca002886035ddc3b08b66e8931313e3a0f3dbf9866d3b04c25432100e
+size 438004853

model/new_version_v3/special_tokens_map.json

@@ -0,0 +1,10 @@
+{
+  "additional_special_tokens": [
+    "[unused1]"
+  ],
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

model/new_version_v3/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

model/new_version_v3/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6e790d899c6f03e1990e4d4ff2bd7e63eead63ed41f89122176b62ddefd330e
+size 4091