FEAT: new models, reload model each time when something change (not ideal but it is better than st.cache_resource)

app.py

@@ -8,19 +8,29 @@ import pandas
 import streamlit as st
 import matplotlib.pyplot as plt
 
+
 from inference_tokenizer import NextSentencePredictionTokenizer
+from models import get_class
+from models import OwnBertForNextSentencePrediction
 
+def get_model(_model_path):
+    print(f"Getting model at {_model_path}")
+    if os.path.isfile(os.path.join(_model_path, "meta-info.json")):
+        with open(os.path.join(_model_path, "meta-info.json"), "r") as f:
+            meta_info = json.load(f)
+            _model_package = meta_info["kwargs"].get("model_package", "transformers")
+            _model_class = meta_info["kwargs"].get("model_class", "BertForNextSentencePrediction")
+    else:
+        raise FileNotFoundError("Tokenizer is provided without meta-info.json. Cannot interfere proper configuration!")
 
-@st.cache_resource
-def get_model(model_path):
-    from transformers import BertForNextSentencePrediction
-    _model = BertForNextSentencePrediction.from_pretrained(model_path)
+    model_class = get_class(_model_package, _model_class)
+    _model = model_class.from_pretrained(_model_path)
     _model.eval()
     return _model
 
 
-@st.cache_resource
 def get_tokenizer(tokenizer_path):
+    print(f"Getting tokenizer at {tokenizer_path}")
     from transformers import BertTokenizer
     tokenizer = BertTokenizer.from_pretrained(tokenizer_path)
     if os.path.isfile(os.path.join(tokenizer_path, "meta-info.json")):
@@ -33,8 +43,8 @@ def get_tokenizer(tokenizer_path):
 
     if special_token != " ":
         tokenizer.add_special_tokens({"additional_special_tokens": [special_token]})
-    print(special_token)
-    print(tokenizer_args)
+    # print(special_token)
+    # print(tokenizer_args)
     _inference_tokenizer = NextSentencePredictionTokenizer(tokenizer, **tokenizer_args)
     return _inference_tokenizer
 
@@ -47,11 +57,12 @@ for model_path in models_path:
         model_data["path"] = model_path.replace("info.json", "")
         models[model_data["model"]] = model_data
 
+
 model_name = st.selectbox('Which model do you want to use?',
-                          (x for x in sorted(models.keys())))
+                          (x for x in sorted(models.keys())),
+                          index=0)
 
 model_path = models[model_name]["path"]
-
 model = get_model(model_path)
 inference_tokenizer = get_tokenizer(model_path)
 

inference_tokenizer.py

@@ -13,10 +13,9 @@ class NextSentencePredictionTokenizer:
         self.tokenizer_args["max_length"] = self.max_length_ctx + self.max_length_res
 
         # cleaning
-        del self.tokenizer_args["special_token"]
-        del self.tokenizer_args["naive_approach"]
-        del self.tokenizer_args["max_length_ctx"]
-        del self.tokenizer_args["max_length_res"]
+        for key_to_delete in ["special_token", "naive_approach", "max_length_ctx", "max_length_res", "approach"]:
+            if key_to_delete in self.tokenizer_args:
+                del self.tokenizer_args[key_to_delete]
 
     def get_item(self, context: List[str], actual_sentence: str):
         context_str = f" {self.special_token} ".join(context) if self.special_token != " " else " ".join(context)

model/d1dd8365cbf16ff423f537e2291c61a91c717ed1/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "bert-base-uncased",
+  "architectures": [
+    "OwnBertForNextSentencePrediction"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.17.0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

model/d1dd8365cbf16ff423f537e2291c61a91c717ed1/info.json

@@ -0,0 +1,4 @@
+{
+  "model": "BERT-NSP-v4",
+  "description": "Model trained on DailyDialogue and CommonDialogues. Using [unused1] token to divide sentences in context. Improved training arguments (warmup, smaller learning rate). Using frozen test set to better compare models and therefore trained longer time (about 60 epochs) More info can be found at https://wandb.ai/alquist/next-sentence-prediction/runs/vzpwetvm/overview?workspace=user-petr-lorenc"
+}

model/d1dd8365cbf16ff423f537e2291c61a91c717ed1/meta-info.json

@@ -0,0 +1 @@
+{"args": [], "kwargs": {"model_package": "models", "model_class": "OwnBertForNextSentencePrediction","data_root": "/home/lorenpe2/project/data", "data_sources": [["COMMON_DIALOGUES", "common_dialogues/train.json", "common_dialogues/valid_frozen.json", "common_dialogues/test_frozen.json"], ["DAILY_DIALOGUES", "daily_dialogues/dialogues_text.train.txt", "daily_dialogues/dev_frozen.json", "daily_dialogues/test_frozen.json"]], "pretrained_model": "bert-base-uncased", "tokenizer": "bert-base-uncased", "approach": "IGNORE_DUPLICITIES", "special_token": "[unused1]", "learning_rate": 5e-07, "warmup_ratio": 0.1, "freeze_prefinetuning": true, "prefinenuting_epoch": 10, "finetuning_epochs": 75}, "tokenizer_args": {"padding": "max_length", "max_length_ctx": 256, "max_length_res": 40, "truncation": "only_first", "return_tensors": "np", "is_split_into_words": true, "approach": "IGNORE_DUPLICITIES", "special_token": "[unused1]"}}

model/d1dd8365cbf16ff423f537e2291c61a91c717ed1/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:486f1fc1c4302a5332800463f331651de7735bff5034d1e4c75e97f35412d62e
+size 69074944

model/d1dd8365cbf16ff423f537e2291c61a91c717ed1/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cac4be9fdac90195f60a8ef106ab8f9e382758c3f576a35b52eaea575d4a19a
+size 438871109

model/d1dd8365cbf16ff423f537e2291c61a91c717ed1/special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "additional_special_tokens": ["[unused1]"]}

model/d1dd8365cbf16ff423f537e2291c61a91c717ed1/tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "do_basic_tokenize": true, "never_split": null, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "bert-base-uncased", "tokenizer_class": "BertTokenizer"}

model/d1dd8365cbf16ff423f537e2291c61a91c717ed1/trainer_state.json

@@ -0,0 +1,1060 @@
+{
+  "best_metric": 0.3728518784046173,
+  "best_model_checkpoint": "/home/lorenpe2/scratch/dialogue-quality/d1dd8365cbf16ff423f537e2291c61a91c717ed1/checkpoint-10494",
+  "epoch": 65.99478623566215,
+  "global_step": 10494,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.99,
+      "eval_accuracy": 0.7063609467455622,
+      "eval_f1": 0.6026026026026026,
+      "eval_loss": 0.6779443621635437,
+      "eval_precision_following": 0.6355685131195336,
+      "eval_precision_not_following": 0.9318885448916409,
+      "eval_recall_following": 0.9674556213017751,
+      "eval_recall_not_following": 0.4452662721893491,
+      "eval_runtime": 6.4103,
+      "eval_samples_per_second": 1265.468,
+      "eval_steps_per_second": 4.992,
+      "step": 159
+    },
+    {
+      "epoch": 1.99,
+      "eval_accuracy": 0.76232741617357,
+      "eval_f1": 0.707168894289186,
+      "eval_loss": 0.6727659702301025,
+      "eval_precision_following": 0.6905444126074498,
+      "eval_precision_not_following": 0.9208860759493671,
+      "eval_recall_following": 0.9506903353057199,
+      "eval_recall_not_following": 0.5739644970414202,
+      "eval_runtime": 6.6319,
+      "eval_samples_per_second": 1223.188,
+      "eval_steps_per_second": 4.825,
+      "step": 318
+    },
+    {
+      "epoch": 2.99,
+      "eval_accuracy": 0.8150887573964497,
+      "eval_f1": 0.7912607848594491,
+      "eval_loss": 0.6655098795890808,
+      "eval_precision_following": 0.7565234845443597,
+      "eval_precision_not_following": 0.9083067092651758,
+      "eval_recall_following": 0.9292406311637081,
+      "eval_recall_not_following": 0.7009368836291914,
+      "eval_runtime": 6.7829,
+      "eval_samples_per_second": 1195.945,
+      "eval_steps_per_second": 4.718,
+      "step": 477
+    },
+    {
+      "epoch": 3.14,
+      "learning_rate": 2.0955574182732605e-07,
+      "loss": 0.674,
+      "step": 500
+    },
+    {
+      "epoch": 3.99,
+      "eval_accuracy": 0.8354289940828402,
+      "eval_f1": 0.8217861433720465,
+      "eval_loss": 0.6588814854621887,
+      "eval_precision_following": 0.790891597177678,
+      "eval_precision_not_following": 0.8960698689956332,
+      "eval_recall_following": 0.9119822485207101,
+      "eval_recall_not_following": 0.7588757396449705,
+      "eval_runtime": 6.5136,
+      "eval_samples_per_second": 1245.393,
+      "eval_steps_per_second": 4.913,
+      "step": 636
+    },
+    {
+      "epoch": 4.99,
+      "eval_accuracy": 0.8497287968441815,
+      "eval_f1": 0.8397949796293863,
+      "eval_loss": 0.6521937251091003,
+      "eval_precision_following": 0.8111427944724721,
+      "eval_precision_not_following": 0.8992400788066423,
+      "eval_recall_following": 0.9117357001972387,
+      "eval_recall_not_following": 0.7877218934911243,
+      "eval_runtime": 6.6185,
+      "eval_samples_per_second": 1225.651,
+      "eval_steps_per_second": 4.835,
+      "step": 795
+    },
+    {
+      "epoch": 5.99,
+      "eval_accuracy": 0.8574950690335306,
+      "eval_f1": 0.8495183545951575,
+      "eval_loss": 0.6446988582611084,
+      "eval_precision_following": 0.8232278198840838,
+      "eval_precision_not_following": 0.8998896856039713,
+      "eval_recall_following": 0.9105029585798816,
+      "eval_recall_not_following": 0.8044871794871795,
+      "eval_runtime": 7.6742,
+      "eval_samples_per_second": 1057.054,
+      "eval_steps_per_second": 4.17,
+      "step": 954
+    },
+    {
+      "epoch": 6.29,
+      "learning_rate": 4.191114836546521e-07,
+      "loss": 0.6536,
+      "step": 1000
+    },
+    {
+      "epoch": 6.99,
+      "eval_accuracy": 0.8603303747534516,
+      "eval_f1": 0.8529907875956921,
+      "eval_loss": 0.6363867521286011,
+      "eval_precision_following": 0.8276171262048868,
+      "eval_precision_not_following": 0.9003012873185429,
+      "eval_recall_following": 0.9102564102564102,
+      "eval_recall_not_following": 0.810404339250493,
+      "eval_runtime": 7.6857,
+      "eval_samples_per_second": 1055.464,
+      "eval_steps_per_second": 4.164,
+      "step": 1113
+    },
+    {
+      "epoch": 7.99,
+      "eval_accuracy": 0.8664940828402367,
+      "eval_f1": 0.860456126787785,
+      "eval_loss": 0.6268088817596436,
+      "eval_precision_following": 0.8373042886317222,
+      "eval_precision_not_following": 0.9012145748987854,
+      "eval_recall_following": 0.9097633136094675,
+      "eval_recall_not_following": 0.8232248520710059,
+      "eval_runtime": 7.454,
+      "eval_samples_per_second": 1088.275,
+      "eval_steps_per_second": 4.293,
+      "step": 1272
+    },
+    {
+      "epoch": 8.99,
+      "eval_accuracy": 0.8678500986193294,
+      "eval_f1": 0.8618200567156483,
+      "eval_loss": 0.6177182793617249,
+      "eval_precision_following": 0.8383219954648526,
+      "eval_precision_not_following": 0.9030253916801729,
+      "eval_recall_following": 0.9114891518737672,
+      "eval_recall_not_following": 0.8242110453648915,
+      "eval_runtime": 7.6101,
+      "eval_samples_per_second": 1065.946,
+      "eval_steps_per_second": 4.205,
+      "step": 1431
+    },
+    {
+      "epoch": 9.43,
+      "learning_rate": 4.856969809914275e-07,
+      "loss": 0.6261,
+      "step": 1500
+    },
+    {
+      "epoch": 9.99,
+      "eval_accuracy": 0.8709319526627219,
+      "eval_f1": 0.8653029718255499,
+      "eval_loss": 0.6082141995429993,
+      "eval_precision_following": 0.842320819112628,
+      "eval_precision_not_following": 0.9047619047619048,
+      "eval_recall_following": 0.9127218934911243,
+      "eval_recall_not_following": 0.8291420118343196,
+      "eval_runtime": 6.5573,
+      "eval_samples_per_second": 1237.093,
+      "eval_steps_per_second": 4.88,
+      "step": 1590
+    },
+    {
+      "epoch": 10.99,
+      "eval_accuracy": 0.8730276134122288,
+      "eval_f1": 0.8682190378710338,
+      "eval_loss": 0.5987269282341003,
+      "eval_precision_following": 0.84765625,
+      "eval_precision_not_following": 0.9023936170212766,
+      "eval_recall_following": 0.909516765285996,
+      "eval_recall_not_following": 0.8365384615384616,
+      "eval_runtime": 6.5989,
+      "eval_samples_per_second": 1229.295,
+      "eval_steps_per_second": 4.849,
+      "step": 1749
+    },
+    {
+      "epoch": 11.99,
+      "eval_accuracy": 0.8735207100591716,
+      "eval_f1": 0.8683940482298614,
+      "eval_loss": 0.5899477601051331,
+      "eval_precision_following": 0.846523330283623,
+      "eval_precision_not_following": 0.9050802139037433,
+      "eval_recall_following": 0.9124753451676528,
+      "eval_recall_not_following": 0.8345660749506904,
+      "eval_runtime": 6.3563,
+      "eval_samples_per_second": 1276.22,
+      "eval_steps_per_second": 5.034,
+      "step": 1908
+    },
+    {
+      "epoch": 12.58,
+      "learning_rate": 4.6240216175922476e-07,
+      "loss": 0.5942,
+      "step": 2000
+    },
+    {
+      "epoch": 12.99,
+      "eval_accuracy": 0.8745069033530573,
+      "eval_f1": 0.869218910585817,
+      "eval_loss": 0.581299901008606,
+      "eval_precision_following": 0.8464872262773723,
+      "eval_precision_not_following": 0.9074570815450643,
+      "eval_recall_following": 0.9149408284023669,
+      "eval_recall_not_following": 0.8340729783037475,
+      "eval_runtime": 6.309,
+      "eval_samples_per_second": 1285.773,
+      "eval_steps_per_second": 5.072,
+      "step": 2067
+    },
+    {
+      "epoch": 13.99,
+      "eval_accuracy": 0.8746301775147929,
+      "eval_f1": 0.8694982676761196,
+      "eval_loss": 0.5730020403862,
+      "eval_precision_following": 0.8473142857142857,
+      "eval_precision_not_following": 0.9066095798769066,
+      "eval_recall_following": 0.9139546351084813,
+      "eval_recall_not_following": 0.8353057199211046,
+      "eval_runtime": 6.4991,
+      "eval_samples_per_second": 1248.179,
+      "eval_steps_per_second": 4.924,
+      "step": 2226
+    },
+    {
+      "epoch": 14.99,
+      "eval_accuracy": 0.8747534516765286,
+      "eval_f1": 0.8692743180648481,
+      "eval_loss": 0.5653589963912964,
+      "eval_precision_following": 0.845768880800728,
+      "eval_precision_not_following": 0.9090419806243273,
+      "eval_recall_following": 0.9166666666666666,
+      "eval_recall_not_following": 0.8328402366863905,
+      "eval_runtime": 6.4333,
+      "eval_samples_per_second": 1260.941,
+      "eval_steps_per_second": 4.974,
+      "step": 2385
+    },
+    {
+      "epoch": 15.72,
+      "learning_rate": 4.3910734252702195e-07,
+      "loss": 0.564,
+      "step": 2500
+    },
+    {
+      "epoch": 15.99,
+      "eval_accuracy": 0.8768491124260355,
+      "eval_f1": 0.8722016118715621,
+      "eval_loss": 0.5576348900794983,
+      "eval_precision_following": 0.8512985520569983,
+      "eval_precision_not_following": 0.9064078702472746,
+      "eval_recall_following": 0.9132149901380671,
+      "eval_recall_not_following": 0.840483234714004,
+      "eval_runtime": 6.4791,
+      "eval_samples_per_second": 1252.033,
+      "eval_steps_per_second": 4.939,
+      "step": 2544
+    },
+    {
+      "epoch": 16.99,
+      "eval_accuracy": 0.8764792899408284,
+      "eval_f1": 0.8712413261372397,
+      "eval_loss": 0.5506843328475952,
+      "eval_precision_following": 0.8481532147742818,
+      "eval_precision_not_following": 0.9098228663446055,
+      "eval_recall_following": 0.9171597633136095,
+      "eval_recall_not_following": 0.8357988165680473,
+      "eval_runtime": 7.9718,
+      "eval_samples_per_second": 1017.585,
+      "eval_steps_per_second": 4.014,
+      "step": 2703
+    },
+    {
+      "epoch": 17.99,
+      "eval_accuracy": 0.8764792899408284,
+      "eval_f1": 0.8712744090441932,
+      "eval_loss": 0.5431817173957825,
+      "eval_precision_following": 0.8483120437956204,
+      "eval_precision_not_following": 0.9096030042918455,
+      "eval_recall_following": 0.9169132149901381,
+      "eval_recall_not_following": 0.8360453648915187,
+      "eval_runtime": 7.4474,
+      "eval_samples_per_second": 1089.246,
+      "eval_steps_per_second": 4.297,
+      "step": 2862
+    },
+    {
+      "epoch": 18.86,
+      "learning_rate": 4.158125232948192e-07,
+      "loss": 0.5372,
+      "step": 3000
+    },
+    {
+      "epoch": 18.99,
+      "eval_accuracy": 0.8772189349112426,
+      "eval_f1": 0.8721109399075501,
+      "eval_loss": 0.5359050631523132,
+      "eval_precision_following": 0.8493150684931506,
+      "eval_precision_not_following": 0.909967845659164,
+      "eval_recall_following": 0.9171597633136095,
+      "eval_recall_not_following": 0.8372781065088757,
+      "eval_runtime": 7.5381,
+      "eval_samples_per_second": 1076.128,
+      "eval_steps_per_second": 4.245,
+      "step": 3021
+    },
+    {
+      "epoch": 19.99,
+      "eval_accuracy": 0.8778353057199211,
+      "eval_f1": 0.8725401929260451,
+      "eval_loss": 0.5289158821105957,
+      "eval_precision_following": 0.8488504438880037,
+      "eval_precision_not_following": 0.9120731379403065,
+      "eval_recall_following": 0.9193786982248521,
+      "eval_recall_not_following": 0.8362919132149902,
+      "eval_runtime": 6.6346,
+      "eval_samples_per_second": 1222.686,
+      "eval_steps_per_second": 4.823,
+      "step": 3180
+    },
+    {
+      "epoch": 20.99,
+      "eval_accuracy": 0.8789447731755424,
+      "eval_f1": 0.8737139917695474,
+      "eval_loss": 0.5221996307373047,
+      "eval_precision_following": 0.8499544626593807,
+      "eval_precision_not_following": 0.9131720430107527,
+      "eval_recall_following": 0.9203648915187377,
+      "eval_recall_not_following": 0.8375246548323472,
+      "eval_runtime": 6.5308,
+      "eval_samples_per_second": 1242.11,
+      "eval_steps_per_second": 4.9,
+      "step": 3339
+    },
+    {
+      "epoch": 21.99,
+      "eval_accuracy": 0.8796844181459567,
+      "eval_f1": 0.8742592115434167,
+      "eval_loss": 0.5158143639564514,
+      "eval_precision_following": 0.8495233772128915,
+      "eval_precision_not_following": 0.9155423637344846,
+      "eval_recall_following": 0.9228303747534516,
+      "eval_recall_not_following": 0.8365384615384616,
+      "eval_runtime": 6.3007,
+      "eval_samples_per_second": 1287.474,
+      "eval_steps_per_second": 5.079,
+      "step": 3498
+    },
+    {
+      "epoch": 22.01,
+      "learning_rate": 3.925177040626165e-07,
+      "loss": 0.5126,
+      "step": 3500
+    },
+    {
+      "epoch": 22.99,
+      "eval_accuracy": 0.8798076923076923,
+      "eval_f1": 0.8747269690350764,
+      "eval_loss": 0.5094554424285889,
+      "eval_precision_following": 0.8513112884834664,
+      "eval_precision_not_following": 0.9133351220821035,
+      "eval_recall_following": 0.9203648915187377,
+      "eval_recall_not_following": 0.8392504930966469,
+      "eval_runtime": 6.4725,
+      "eval_samples_per_second": 1253.304,
+      "eval_steps_per_second": 4.944,
+      "step": 3657
+    },
+    {
+      "epoch": 23.99,
+      "eval_accuracy": 0.8793145956607495,
+      "eval_f1": 0.8743744385987424,
+      "eval_loss": 0.5033257603645325,
+      "eval_precision_following": 0.8516571428571429,
+      "eval_precision_not_following": 0.9116938720899117,
+      "eval_recall_following": 0.9186390532544378,
+      "eval_recall_not_following": 0.8399901380670611,
+      "eval_runtime": 6.5235,
+      "eval_samples_per_second": 1243.504,
+      "eval_steps_per_second": 4.905,
+      "step": 3816
+    },
+    {
+      "epoch": 24.99,
+      "eval_accuracy": 0.8806706114398422,
+      "eval_f1": 0.8757062146892656,
+      "eval_loss": 0.49730098247528076,
+      "eval_precision_following": 0.8525114155251141,
+      "eval_precision_not_following": 0.9137191854233655,
+      "eval_recall_following": 0.9206114398422091,
+      "eval_recall_not_following": 0.8407297830374754,
+      "eval_runtime": 6.4549,
+      "eval_samples_per_second": 1256.725,
+      "eval_steps_per_second": 4.957,
+      "step": 3975
+    },
+    {
+      "epoch": 25.16,
+      "learning_rate": 3.692228848304137e-07,
+      "loss": 0.4884,
+      "step": 4000
+    },
+    {
+      "epoch": 25.99,
+      "eval_accuracy": 0.8803007889546351,
+      "eval_f1": 0.8751125401929261,
+      "eval_loss": 0.4917982518672943,
+      "eval_precision_following": 0.8511267926246301,
+      "eval_precision_not_following": 0.9147620328045173,
+      "eval_recall_following": 0.921844181459566,
+      "eval_recall_not_following": 0.8387573964497042,
+      "eval_runtime": 7.4346,
+      "eval_samples_per_second": 1091.112,
+      "eval_steps_per_second": 4.304,
+      "step": 4134
+    },
+    {
+      "epoch": 26.99,
+      "eval_accuracy": 0.8814102564102564,
+      "eval_f1": 0.8765717218373107,
+      "eval_loss": 0.485965758562088,
+      "eval_precision_following": 0.8536808413351623,
+      "eval_precision_not_following": 0.9138576779026217,
+      "eval_recall_following": 0.9206114398422091,
+      "eval_recall_not_following": 0.8422090729783037,
+      "eval_runtime": 7.8472,
+      "eval_samples_per_second": 1033.747,
+      "eval_steps_per_second": 4.078,
+      "step": 4293
+    },
+    {
+      "epoch": 27.99,
+      "eval_accuracy": 0.8815335305719921,
+      "eval_f1": 0.8769053413603177,
+      "eval_loss": 0.48036113381385803,
+      "eval_precision_following": 0.8548498050905755,
+      "eval_precision_not_following": 0.9125566515595841,
+      "eval_recall_following": 0.9191321499013807,
+      "eval_recall_not_following": 0.8439349112426036,
+      "eval_runtime": 6.2961,
+      "eval_samples_per_second": 1288.426,
+      "eval_steps_per_second": 5.083,
+      "step": 4452
+    },
+    {
+      "epoch": 28.3,
+      "learning_rate": 3.4592806559821093e-07,
+      "loss": 0.4673,
+      "step": 4500
+    },
+    {
+      "epoch": 28.99,
+      "eval_accuracy": 0.8814102564102564,
+      "eval_f1": 0.8765083440308087,
+      "eval_loss": 0.47532862424850464,
+      "eval_precision_following": 0.8533576975788031,
+      "eval_precision_not_following": 0.914301017675415,
+      "eval_recall_following": 0.9211045364891519,
+      "eval_recall_not_following": 0.841715976331361,
+      "eval_runtime": 6.4255,
+      "eval_samples_per_second": 1262.466,
+      "eval_steps_per_second": 4.98,
+      "step": 4611
+    },
+    {
+      "epoch": 29.99,
+      "eval_accuracy": 0.8810404339250493,
+      "eval_f1": 0.8754356525106493,
+      "eval_loss": 0.47099417448043823,
+      "eval_precision_following": 0.8495815426374124,
+      "eval_precision_not_following": 0.9187212137632078,
+      "eval_recall_following": 0.9260355029585798,
+      "eval_recall_not_following": 0.8360453648915187,
+      "eval_runtime": 6.5611,
+      "eval_samples_per_second": 1236.373,
+      "eval_steps_per_second": 4.877,
+      "step": 4770
+    },
+    {
+      "epoch": 30.99,
+      "eval_accuracy": 0.8814102564102564,
+      "eval_f1": 0.8758709677419355,
+      "eval_loss": 0.465981125831604,
+      "eval_precision_following": 0.8501584427342689,
+      "eval_precision_not_following": 0.9187872225230103,
+      "eval_recall_following": 0.9260355029585798,
+      "eval_recall_not_following": 0.8367850098619329,
+      "eval_runtime": 6.4473,
+      "eval_samples_per_second": 1258.207,
+      "eval_steps_per_second": 4.963,
+      "step": 4929
+    },
+    {
+      "epoch": 31.44,
+      "learning_rate": 3.2263324636600817e-07,
+      "loss": 0.4474,
+      "step": 5000
+    },
+    {
+      "epoch": 31.99,
+      "eval_accuracy": 0.8817800788954635,
+      "eval_f1": 0.8768460254269936,
+      "eval_loss": 0.4605511724948883,
+      "eval_precision_following": 0.853458114585711,
+      "eval_precision_not_following": 0.9150361833288663,
+      "eval_recall_following": 0.921844181459566,
+      "eval_recall_not_following": 0.841715976331361,
+      "eval_runtime": 7.8836,
+      "eval_samples_per_second": 1028.968,
+      "eval_steps_per_second": 4.059,
+      "step": 5088
+    },
+    {
+      "epoch": 32.99,
+      "eval_accuracy": 0.8826429980276134,
+      "eval_f1": 0.877540519680988,
+      "eval_loss": 0.45592227578163147,
+      "eval_precision_following": 0.8532089212562586,
+      "eval_precision_not_following": 0.9174287251210328,
+      "eval_recall_following": 0.9243096646942801,
+      "eval_recall_not_following": 0.8409763313609467,
+      "eval_runtime": 7.6557,
+      "eval_samples_per_second": 1059.596,
+      "eval_steps_per_second": 4.18,
+      "step": 5247
+    },
+    {
+      "epoch": 33.99,
+      "eval_accuracy": 0.8832593688362919,
+      "eval_f1": 0.8785118665811418,
+      "eval_loss": 0.45115283131599426,
+      "eval_precision_following": 0.8554767893894352,
+      "eval_precision_not_following": 0.9157528751002942,
+      "eval_recall_following": 0.9223372781065089,
+      "eval_recall_not_following": 0.8441814595660749,
+      "eval_runtime": 6.8753,
+      "eval_samples_per_second": 1179.868,
+      "eval_steps_per_second": 4.654,
+      "step": 5406
+    },
+    {
+      "epoch": 34.59,
+      "learning_rate": 2.993384271338054e-07,
+      "loss": 0.4292,
+      "step": 5500
+    },
+    {
+      "epoch": 34.99,
+      "eval_accuracy": 0.8826429980276134,
+      "eval_f1": 0.877351198144808,
+      "eval_loss": 0.4472413659095764,
+      "eval_precision_following": 0.8522469359963686,
+      "eval_precision_not_following": 0.9187803561791689,
+      "eval_recall_following": 0.9257889546351085,
+      "eval_recall_not_following": 0.8394970414201184,
+      "eval_runtime": 7.6512,
+      "eval_samples_per_second": 1060.227,
+      "eval_steps_per_second": 4.182,
+      "step": 5565
+    },
+    {
+      "epoch": 35.99,
+      "eval_accuracy": 0.8825197238658777,
+      "eval_f1": 0.8775221693869684,
+      "eval_loss": 0.44297003746032715,
+      "eval_precision_following": 0.8536585365853658,
+      "eval_precision_not_following": 0.916510067114094,
+      "eval_recall_following": 0.9233234714003945,
+      "eval_recall_not_following": 0.841715976331361,
+      "eval_runtime": 6.5867,
+      "eval_samples_per_second": 1231.58,
+      "eval_steps_per_second": 4.858,
+      "step": 5724
+    },
+    {
+      "epoch": 36.99,
+      "eval_accuracy": 0.8827662721893491,
+      "eval_f1": 0.8777163430628777,
+      "eval_loss": 0.4388849139213562,
+      "eval_precision_following": 0.8535641084035527,
+      "eval_precision_not_following": 0.91722655200215,
+      "eval_recall_following": 0.9240631163708086,
+      "eval_recall_not_following": 0.8414694280078896,
+      "eval_runtime": 7.7033,
+      "eval_samples_per_second": 1053.059,
+      "eval_steps_per_second": 4.154,
+      "step": 5883
+    },
+    {
+      "epoch": 37.73,
+      "learning_rate": 2.760436079016027e-07,
+      "loss": 0.4128,
+      "step": 6000
+    },
+    {
+      "epoch": 37.99,
+      "eval_accuracy": 0.8819033530571992,
+      "eval_f1": 0.8765145656096932,
+      "eval_loss": 0.4353000819683075,
+      "eval_precision_following": 0.8512471655328798,
+      "eval_precision_not_following": 0.9184224743381956,
+      "eval_recall_following": 0.9255424063116371,
+      "eval_recall_not_following": 0.8382642998027613,
+      "eval_runtime": 7.9842,
+      "eval_samples_per_second": 1016.012,
+      "eval_steps_per_second": 4.008,
+      "step": 6042
+    },
+    {
+      "epoch": 38.99,
+      "eval_accuracy": 0.8832593688362919,
+      "eval_f1": 0.878324553514069,
+      "eval_loss": 0.4311205744743347,
+      "eval_precision_following": 0.8545039908779931,
+      "eval_precision_not_following": 0.9170914944995975,
+      "eval_recall_following": 0.9238165680473372,
+      "eval_recall_not_following": 0.8427021696252466,
+      "eval_runtime": 7.7705,
+      "eval_samples_per_second": 1043.95,
+      "eval_steps_per_second": 4.118,
+      "step": 6201
+    },
+    {
+      "epoch": 39.99,
+      "eval_accuracy": 0.8835059171597633,
+      "eval_f1": 0.8785503148695541,
+      "eval_loss": 0.42720890045166016,
+      "eval_precision_following": 0.8545703214041486,
+      "eval_precision_not_following": 0.9175838926174497,
+      "eval_recall_following": 0.9243096646942801,
+      "eval_recall_not_following": 0.8427021696252466,
+      "eval_runtime": 7.4652,
+      "eval_samples_per_second": 1086.638,
+      "eval_steps_per_second": 4.287,
+      "step": 6360
+    },
+    {
+      "epoch": 40.88,
+      "learning_rate": 2.527487886693999e-07,
+      "loss": 0.3974,
+      "step": 6500
+    },
+    {
+      "epoch": 40.99,
+      "eval_accuracy": 0.8841222879684418,
+      "eval_f1": 0.8793324775353017,
+      "eval_loss": 0.4234888553619385,
+      "eval_precision_following": 0.8558702603928735,
+      "eval_precision_not_following": 0.9172469201928227,
+      "eval_recall_following": 0.9238165680473372,
+      "eval_recall_not_following": 0.8444280078895463,
+      "eval_runtime": 7.8319,
+      "eval_samples_per_second": 1035.77,
+      "eval_steps_per_second": 4.086,
+      "step": 6519
+    },
+    {
+      "epoch": 41.99,
+      "eval_accuracy": 0.8828895463510849,
+      "eval_f1": 0.8777977874967842,
+      "eval_loss": 0.42032715678215027,
+      "eval_precision_following": 0.8534365043240782,
+      "eval_precision_not_following": 0.9176976869284562,
+      "eval_recall_following": 0.9245562130177515,
+      "eval_recall_not_following": 0.8412228796844181,
+      "eval_runtime": 6.557,
+      "eval_samples_per_second": 1237.159,
+      "eval_steps_per_second": 4.88,
+      "step": 6678
+    },
+    {
+      "epoch": 42.99,
+      "eval_accuracy": 0.8838757396449705,
+      "eval_f1": 0.8787956767884715,
+      "eval_loss": 0.41689053177833557,
+      "eval_precision_following": 0.8541856232939036,
+      "eval_precision_not_following": 0.9189989235737352,
+      "eval_recall_following": 0.9257889546351085,
+      "eval_recall_not_following": 0.8419625246548323,
+      "eval_runtime": 6.3499,
+      "eval_samples_per_second": 1277.502,
+      "eval_steps_per_second": 5.039,
+      "step": 6837
+    },
+    {
+      "epoch": 43.99,
+      "eval_accuracy": 0.8838757396449705,
+      "eval_f1": 0.8789514263685426,
+      "eval_loss": 0.413469135761261,
+      "eval_precision_following": 0.8549931600547196,
+      "eval_precision_not_following": 0.9178743961352657,
+      "eval_recall_following": 0.9245562130177515,
+      "eval_recall_not_following": 0.8431952662721893,
+      "eval_runtime": 6.2083,
+      "eval_samples_per_second": 1306.64,
+      "eval_steps_per_second": 5.154,
+      "step": 6996
+    },
+    {
+      "epoch": 44.03,
+      "learning_rate": 2.2945396943719717e-07,
+      "loss": 0.3843,
+      "step": 7000
+    },
+    {
+      "epoch": 44.99,
+      "eval_accuracy": 0.883629191321499,
+      "eval_f1": 0.878475798146241,
+      "eval_loss": 0.4109957814216614,
+      "eval_precision_following": 0.8536363636363636,
+      "eval_precision_not_following": 0.9191810344827587,
+      "eval_recall_following": 0.9260355029585798,
+      "eval_recall_not_following": 0.8412228796844181,
+      "eval_runtime": 6.439,
+      "eval_samples_per_second": 1259.826,
+      "eval_steps_per_second": 4.97,
+      "step": 7155
+    },
+    {
+      "epoch": 45.99,
+      "eval_accuracy": 0.8832593688362919,
+      "eval_f1": 0.877916720381591,
+      "eval_loss": 0.40848881006240845,
+      "eval_precision_following": 0.8524144184992065,
+      "eval_precision_not_following": 0.920021615779519,
+      "eval_recall_following": 0.9270216962524654,
+      "eval_recall_not_following": 0.8394970414201184,
+      "eval_runtime": 6.4019,
+      "eval_samples_per_second": 1267.122,
+      "eval_steps_per_second": 4.999,
+      "step": 7314
+    },
+    {
+      "epoch": 46.99,
+      "eval_accuracy": 0.8830128205128205,
+      "eval_f1": 0.877973511636878,
+      "eval_loss": 0.405333548784256,
+      "eval_precision_following": 0.8537918469596902,
+      "eval_precision_not_following": 0.917495296963182,
+      "eval_recall_following": 0.9243096646942801,
+      "eval_recall_not_following": 0.841715976331361,
+      "eval_runtime": 6.585,
+      "eval_samples_per_second": 1231.895,
+      "eval_steps_per_second": 4.86,
+      "step": 7473
+    },
+    {
+      "epoch": 47.17,
+      "learning_rate": 2.061591502049944e-07,
+      "loss": 0.3713,
+      "step": 7500
+    },
+    {
+      "epoch": 47.99,
+      "eval_accuracy": 0.883629191321499,
+      "eval_f1": 0.8785383427689141,
+      "eval_loss": 0.4025915265083313,
+      "eval_precision_following": 0.8539581437670609,
+      "eval_precision_not_following": 0.918729817007535,
+      "eval_recall_following": 0.9255424063116371,
+      "eval_recall_not_following": 0.841715976331361,
+      "eval_runtime": 6.482,
+      "eval_samples_per_second": 1251.458,
+      "eval_steps_per_second": 4.937,
+      "step": 7632
+    },
+    {
+      "epoch": 48.99,
+      "eval_accuracy": 0.8842455621301775,
+      "eval_f1": 0.8792593545068792,
+      "eval_loss": 0.39974769949913025,
+      "eval_precision_following": 0.854930539740378,
+      "eval_precision_not_following": 0.9188390217683419,
+      "eval_recall_following": 0.9255424063116371,
+      "eval_recall_not_following": 0.842948717948718,
+      "eval_runtime": 6.8714,
+      "eval_samples_per_second": 1180.552,
+      "eval_steps_per_second": 4.657,
+      "step": 7791
+    },
+    {
+      "epoch": 49.99,
+      "eval_accuracy": 0.884492110453649,
+      "eval_f1": 0.8796094051137093,
+      "eval_loss": 0.39722564816474915,
+      "eval_precision_following": 0.8556442417331813,
+      "eval_precision_not_following": 0.918433056077274,
+      "eval_recall_following": 0.9250493096646942,
+      "eval_recall_not_following": 0.8439349112426036,
+      "eval_runtime": 6.7422,
+      "eval_samples_per_second": 1203.167,
+      "eval_steps_per_second": 4.746,
+      "step": 7950
+    },
+    {
+      "epoch": 50.31,
+      "learning_rate": 1.8286433097279163e-07,
+      "loss": 0.3602,
+      "step": 8000
+    },
+    {
+      "epoch": 50.99,
+      "eval_accuracy": 0.8839990138067061,
+      "eval_f1": 0.879002185932879,
+      "eval_loss": 0.394969642162323,
+      "eval_precision_following": 0.8547028011842405,
+      "eval_precision_not_following": 0.9185702768073098,
+      "eval_recall_following": 0.9252958579881657,
+      "eval_recall_not_following": 0.8427021696252466,
+      "eval_runtime": 6.3463,
+      "eval_samples_per_second": 1278.229,
+      "eval_steps_per_second": 5.042,
+      "step": 8109
+    },
+    {
+      "epoch": 51.99,
+      "eval_accuracy": 0.8833826429980276,
+      "eval_f1": 0.8781555899021123,
+      "eval_loss": 0.39329206943511963,
+      "eval_precision_following": 0.8530881017257039,
+      "eval_precision_not_following": 0.9193635382955772,
+      "eval_recall_following": 0.9262820512820513,
+      "eval_recall_not_following": 0.840483234714004,
+      "eval_runtime": 6.7431,
+      "eval_samples_per_second": 1203.016,
+      "eval_steps_per_second": 4.746,
+      "step": 8268
+    },
+    {
+      "epoch": 52.99,
+      "eval_accuracy": 0.8842455621301775,
+      "eval_f1": 0.8791039011201237,
+      "eval_loss": 0.3910701274871826,
+      "eval_precision_following": 0.8541240627130198,
+      "eval_precision_not_following": 0.919967663702506,
+      "eval_recall_following": 0.9267751479289941,
+      "eval_recall_not_following": 0.841715976331361,
+      "eval_runtime": 6.5474,
+      "eval_samples_per_second": 1238.969,
+      "eval_steps_per_second": 4.887,
+      "step": 8427
+    },
+    {
+      "epoch": 53.46,
+      "learning_rate": 1.5956951174058888e-07,
+      "loss": 0.3507,
+      "step": 8500
+    },
+    {
+      "epoch": 53.99,
+      "eval_accuracy": 0.8837524654832347,
+      "eval_f1": 0.8784950392990594,
+      "eval_loss": 0.3893094062805176,
+      "eval_precision_following": 0.8531881098252779,
+      "eval_precision_not_following": 0.9201079622132253,
+      "eval_recall_following": 0.9270216962524654,
+      "eval_recall_not_following": 0.840483234714004,
+      "eval_runtime": 6.5965,
+      "eval_samples_per_second": 1229.738,
+      "eval_steps_per_second": 4.851,
+      "step": 8586
+    },
+    {
+      "epoch": 54.99,
+      "eval_accuracy": 0.8839990138067061,
+      "eval_f1": 0.8786589297227595,
+      "eval_loss": 0.3876223862171173,
+      "eval_precision_following": 0.8529345116700657,
+      "eval_precision_not_following": 0.9210597458772641,
+      "eval_recall_following": 0.928007889546351,
+      "eval_recall_not_following": 0.8399901380670611,
+      "eval_runtime": 6.4132,
+      "eval_samples_per_second": 1264.882,
+      "eval_steps_per_second": 4.99,
+      "step": 8745
+    },
+    {
+      "epoch": 55.99,
+      "eval_accuracy": 0.8841222879684418,
+      "eval_f1": 0.8789595673448365,
+      "eval_loss": 0.38539403676986694,
+      "eval_precision_following": 0.8539300318037256,
+      "eval_precision_not_following": 0.9199460916442048,
+      "eval_recall_following": 0.9267751479289941,
+      "eval_recall_not_following": 0.8414694280078896,
+      "eval_runtime": 6.2428,
+      "eval_samples_per_second": 1299.421,
+      "eval_steps_per_second": 5.126,
+      "step": 8904
+    },
+    {
+      "epoch": 56.6,
+      "learning_rate": 1.3627469250838615e-07,
+      "loss": 0.3426,
+      "step": 9000
+    },
+    {
+      "epoch": 56.99,
+      "eval_accuracy": 0.8842455621301775,
+      "eval_f1": 0.8789480469253578,
+      "eval_loss": 0.38400083780288696,
+      "eval_precision_following": 0.8533212423486738,
+      "eval_precision_not_following": 0.9211024047554714,
+      "eval_recall_following": 0.928007889546351,
+      "eval_recall_not_following": 0.840483234714004,
+      "eval_runtime": 6.3044,
+      "eval_samples_per_second": 1286.728,
+      "eval_steps_per_second": 5.076,
+      "step": 9063
+    },
+    {
+      "epoch": 57.99,
+      "eval_accuracy": 0.8847386587771203,
+      "eval_f1": 0.8797118229769716,
+      "eval_loss": 0.38182100653648376,
+      "eval_precision_following": 0.8550625711035267,
+      "eval_precision_not_following": 0.9198278181329029,
+      "eval_recall_following": 0.9265285996055227,
+      "eval_recall_not_following": 0.842948717948718,
+      "eval_runtime": 6.3112,
+      "eval_samples_per_second": 1285.343,
+      "eval_steps_per_second": 5.07,
+      "step": 9222
+    },
+    {
+      "epoch": 58.99,
+      "eval_accuracy": 0.8847386587771203,
+      "eval_f1": 0.8797118229769716,
+      "eval_loss": 0.3803881108760834,
+      "eval_precision_following": 0.8550625711035267,
+      "eval_precision_not_following": 0.9198278181329029,
+      "eval_recall_following": 0.9265285996055227,
+      "eval_recall_not_following": 0.842948717948718,
+      "eval_runtime": 6.8641,
+      "eval_samples_per_second": 1181.797,
+      "eval_steps_per_second": 4.662,
+      "step": 9381
+    },
+    {
+      "epoch": 59.74,
+      "learning_rate": 1.1297987327618338e-07,
+      "loss": 0.3357,
+      "step": 9500
+    },
+    {
+      "epoch": 59.99,
+      "eval_accuracy": 0.884492110453649,
+      "eval_f1": 0.8792370150792628,
+      "eval_loss": 0.37948256731033325,
+      "eval_precision_following": 0.8537083238829667,
+      "eval_precision_not_following": 0.9211450175533351,
+      "eval_recall_following": 0.928007889546351,
+      "eval_recall_not_following": 0.8409763313609467,
+      "eval_runtime": 6.5287,
+      "eval_samples_per_second": 1242.519,
+      "eval_steps_per_second": 4.901,
+      "step": 9540
+    },
+    {
+      "epoch": 60.99,
+      "eval_accuracy": 0.8846153846153846,
+      "eval_f1": 0.8794125225457357,
+      "eval_loss": 0.37801429629325867,
+      "eval_precision_following": 0.85406264185202,
+      "eval_precision_not_following": 0.9209390178089585,
+      "eval_recall_following": 0.9277613412228797,
+      "eval_recall_not_following": 0.8414694280078896,
+      "eval_runtime": 6.9538,
+      "eval_samples_per_second": 1166.564,
+      "eval_steps_per_second": 4.602,
+      "step": 9699
+    },
+    {
+      "epoch": 61.99,
+      "eval_accuracy": 0.8847386587771203,
+      "eval_f1": 0.8796498905908097,
+      "eval_loss": 0.3765193819999695,
+      "eval_precision_following": 0.8547397135712662,
+      "eval_precision_not_following": 0.9202800969566388,
+      "eval_recall_following": 0.9270216962524654,
+      "eval_recall_not_following": 0.8424556213017751,
+      "eval_runtime": 6.5515,
+      "eval_samples_per_second": 1238.193,
+      "eval_steps_per_second": 4.884,
+      "step": 9858
+    },
+    {
+      "epoch": 62.89,
+      "learning_rate": 8.968505404398062e-08,
+      "loss": 0.3296,
+      "step": 10000
+    },
+    {
+      "epoch": 62.99,
+      "eval_accuracy": 0.8848619329388561,
+      "eval_f1": 0.8798250128667009,
+      "eval_loss": 0.375373512506485,
+      "eval_precision_following": 0.8550955414012739,
+      "eval_precision_not_following": 0.920075349838536,
+      "eval_recall_following": 0.9267751479289941,
+      "eval_recall_not_following": 0.842948717948718,
+      "eval_runtime": 6.5325,
+      "eval_samples_per_second": 1241.793,
+      "eval_steps_per_second": 4.899,
+      "step": 10017
+    },
+    {
+      "epoch": 63.99,
+      "eval_accuracy": 0.8846153846153846,
+      "eval_f1": 0.8794435857805255,
+      "eval_loss": 0.37458762526512146,
+      "eval_precision_following": 0.8542234332425068,
+      "eval_precision_not_following": 0.9207119741100324,
+      "eval_recall_following": 0.9275147928994083,
+      "eval_recall_not_following": 0.841715976331361,
+      "eval_runtime": 6.8668,
+      "eval_samples_per_second": 1181.329,
+      "eval_steps_per_second": 4.66,
+      "step": 10176
+    },
+    {
+      "epoch": 64.99,
+      "eval_accuracy": 0.8848619329388561,
+      "eval_f1": 0.8797631307929968,
+      "eval_loss": 0.3735732436180115,
+      "eval_precision_following": 0.8547727272727272,
+      "eval_precision_not_following": 0.9205280172413793,
+      "eval_recall_following": 0.9272682445759369,
+      "eval_recall_not_following": 0.8424556213017751,
+      "eval_runtime": 6.3558,
+      "eval_samples_per_second": 1276.317,
+      "eval_steps_per_second": 5.035,
+      "step": 10335
+    },
+    {
+      "epoch": 65.99,
+      "eval_accuracy": 0.8846153846153846,
+      "eval_f1": 0.8794746330157095,
+      "eval_loss": 0.3728518784046173,
+      "eval_precision_following": 0.8543843707405725,
+      "eval_precision_not_following": 0.9204851752021563,
+      "eval_recall_following": 0.9272682445759369,
+      "eval_recall_not_following": 0.8419625246548323,
+      "eval_runtime": 6.6517,
+      "eval_samples_per_second": 1219.546,
+      "eval_steps_per_second": 4.811,
+      "step": 10494
+    }
+  ],
+  "max_steps": 11925,
+  "num_train_epochs": 75,
+  "total_flos": 2.5032038197959936e+18,
+  "trial_name": null,
+  "trial_params": null
+}

model/d1dd8365cbf16ff423f537e2291c61a91c717ed1/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ebb1ce7c8b03949d067cec4b43721c54661232c7c0fa141a4776f97f18c7e07
+size 3195

model/f1f881389fb38108e623689999ceaaaf398c5e92/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "bert-base-uncased",
+  "architectures": [
+    "OwnBertForNextSentencePrediction"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.17.0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

model/f1f881389fb38108e623689999ceaaaf398c5e92/info.json

@@ -0,0 +1,4 @@
+{
+  "model": "BERT-NSP-v5",
+  "description": "Model trained on DailyDialogue and CommonDialogues. Using [unused1] token to divide sentences in context. Improved training arguments (warmup, smaller learning rate). Using frozen test set to better compare models and therefore trained longer time (about 60 epochs). The model also have bigger classification head (from one layer liner as classical). More info can be found at https://wandb.ai/alquist/next-sentence-prediction/runs/vzpwetvm/overview?workspace=user-petr-lorenc"
+}

model/f1f881389fb38108e623689999ceaaaf398c5e92/meta-info.json

@@ -0,0 +1 @@
+{"args": [], "kwargs": {"model_package": "models", "model_class": "OwnBertForNextSentencePrediction", "data_root": "/home/lorenpe2/project/data", "data_sources": [["COMMON_DIALOGUES", "common_dialogues/train.json", "common_dialogues/valid_frozen.json", "common_dialogues/test_frozen.json"], ["DAILY_DIALOGUES", "daily_dialogues/dialogues_text.train.txt", "daily_dialogues/dev_frozen.json", "daily_dialogues/test_frozen.json"]], "pretrained_model": "bert-base-uncased", "tokenizer": "bert-base-uncased", "approach": "IGNORE_DUPLICITIES", "special_token": "[unused1]", "learning_rate": 5e-07, "warmup_ratio": 0.1, "freeze_prefinetuning": true, "prefinenuting_epoch": 10, "finetuning_epochs": 75}, "tokenizer_args": {"padding": "max_length", "max_length_ctx": 256, "max_length_res": 40, "truncation": "only_first", "return_tensors": "np", "is_split_into_words": true, "approach": "IGNORE_DUPLICITIES", "special_token": "[unused1]"}}

model/f1f881389fb38108e623689999ceaaaf398c5e92/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3126eae7cab98cc9068a81e6384ba77facfd1e8dd3cfecc983e2609744d3539a
+size 438871109

model/f1f881389fb38108e623689999ceaaaf398c5e92/special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "additional_special_tokens": ["[unused1]"]}

model/f1f881389fb38108e623689999ceaaaf398c5e92/tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "do_basic_tokenize": true, "never_split": null, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "bert-base-uncased", "tokenizer_class": "BertTokenizer"}

model/f1f881389fb38108e623689999ceaaaf398c5e92/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49a0183a62c25be44cbf2f333cf9224204e7fbfe84cd9054d94775d61daa9774
+size 3195

models.py

@@ -0,0 +1,40 @@
+from torch import nn
+import transformers
+from typing import List
+
+
+def get_class(_model_package, _model_class):
+    mod = __import__(_model_package, fromlist=[_model_class])
+    return getattr(mod, _model_class)
+
+
+class OwnBertOnlyNSPHead(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.seq_relationship = self._build_layer(config.hidden_size, layer_dimensions=[256, 64])
+
+    def forward(self, pooled_output):
+        seq_relationship_score = self.seq_relationship(pooled_output)
+        return seq_relationship_score
+
+    def _build_layer(self, init_size, layer_dimensions: List, activation=nn.ReLU()):
+        module_list = []
+        _init_size = init_size
+        for layer_dimension in layer_dimensions:
+            module_list.append(nn.Linear(_init_size, layer_dimension))
+            module_list.append(activation)
+            _init_size = layer_dimension
+
+        module_list.append(nn.Linear(_init_size, 2))
+        return nn.Sequential(*module_list)
+
+
+class OwnBertForNextSentencePrediction(transformers.BertForNextSentencePrediction):
+    def __init__(self, config):
+        super().__init__(config)
+
+        # reinit cls layer to be more powerful
+        self.cls = OwnBertOnlyNSPHead(config)
+
+        # Initialize weights and apply final processing
+        self.post_init()