Camera ready

README.md

@@ -12,35 +12,46 @@ language:
 widget:
 - text: "Ha egen brygge og båthus. Hva betyr båthus?"
   example_title: "Definition generation"
+license: cc-by-sa-4.0
 ---
 
+# mT0-Definition-No XL
 
-# mt0-definition-no-xl
+This model is a version of [mT0 XL](https://huggingface.co/bigscience/mt0-xl) finetuned on [Bokmålsordboka](https://ordbokene.no/),
+a dataset of Norwegian definitions and usage examples.
 
-This model is a version of [mt0-xl](https://huggingface.co/bigscience/mt0-xl) fine-tuned on Bokmålsordboka.
-
-It achieves the following results on the evaluation set:
-- Loss: 1.9882
-- Rouge1: 31.4539
-- Rouge2: 16.1017
-- Rougel: 30.6959
-- Rougelsum: 30.6888
-- Gen Len: 8.9348
+It generates definitions of Norwegian words in context.
+Its input is the usage example and the instruction question "Hva betyr TARGET_WORD?"
 
 ## Model description
 
-More information needed
+See details in the paper `Enriching Word Usage Graphs with Cluster Definitions` (LREC-COLING'2024) by 
+Mariia Fedorova, Andrey Kutuzov, Nikolay Arefyev and Dominik Schlechtweg.
 
 ## Intended uses & limitations
 
-More information needed
+The model is intended for research purposes, as a source of contextualized dictionary-like lexical definitions.
+Generated definitions can contain all sorts of biases and stereotypes, stemming from the underlying language model.
 
 ## Training and evaluation data
 
-More information needed
+[Bokmålsordboka](https://ordbokene.no/) by The Norwegian Language Council and the University of Bergen.
+
+## Training results
+
+mT0-Definition-No XL achieves the following results on the evaluation set:
+
+- Loss: 2.0358
+- Rouge1: 28.3491
+- Rouge2: 14.2699
+- Rougel: 27.7602
+- Rougelsum: 27.752
+- Gen Len: 10.0765
 
 ## Training procedure
 
+mT0-Definition-No XL was fine-tuned in a sequence-to-sequence mode on examples of contextualized dictionary definitions.
+
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
@@ -57,25 +68,11 @@ The following hyperparameters were used during training:
 - lr_scheduler_type: linear
 - num_epochs: 20.0
 
-### Training results
-
-| Training Loss | Epoch | Step | Validation Loss | Rouge1  | Rouge2  | Rougel  | Rougelsum | Gen Len |
-|:-------------:|:-----:|:----:|:---------------:|:-------:|:-------:|:-------:|:---------:|:-------:|
-| 2.6123        | 1.0   | 497  | 1.8900          | 28.8529 | 13.6138 | 28.1471 | 28.1199   | 8.9495  |
-| 2.0642        | 2.0   | 994  | 1.8383          | 30.5095 | 15.2505 | 29.8018 | 29.7658   | 8.6406  |
-| 1.7611        | 3.0   | 1491 | 1.8447          | 30.9812 | 15.6932 | 30.2339 | 30.2413   | 8.7151  |
-| 1.5284        | 4.0   | 1989 | 1.8619          | 30.9706 | 15.4516 | 30.1888 | 30.1911   | 9.3787  |
-| 1.3422        | 5.0   | 2486 | 1.8895          | 30.9451 | 15.5242 | 30.1826 | 30.1837   | 9.2026  |
-| 1.1862        | 6.0   | 2983 | 1.9224          | 31.3072 | 15.959  | 30.5538 | 30.5404   | 8.8816  |
-| 1.0526        | 7.0   | 3480 | 1.9882          | 31.4465 | 16.095  | 30.6871 | 30.6739   | 8.7294  |
-| 0.9384        | 8.0   | 3978 | 2.0583          | 31.1434 | 15.7298 | 30.287  | 30.2831   | 9.6134  |
-| 0.8408        | 9.0   | 4475 | 2.1237          | 30.7808 | 15.4943 | 29.9606 | 29.9589   | 9.6527  |
-| 0.7592        | 10.0  | 4972 | 2.1987          | 31.0097 | 15.5823 | 30.1202 | 30.1151   | 9.9255  |
-
-
 ### Framework versions
 
-- Transformers 4.30.2
+- Transformers 4.37.1
 - Pytorch 1.13.1+rocm5.2
-- Datasets 2.12.0
-- Tokenizers 0.12.1
+- Datasets 2.16.1
+- Tokenizers 0.15.1
+
+## Citation

all_results.json

@@ -1,18 +1,18 @@
 {
-    "epoch": 10.0,
-    "eval_gen_len": 8.93481334841629,
-    "eval_loss": 1.9881515502929688,
-    "eval_rouge1": 31.4539,
-    "eval_rouge2": 16.1017,
-    "eval_rougeL": 30.6959,
-    "eval_rougeLsum": 30.6888,
-    "eval_runtime": 176.3688,
+    "epoch": 11.0,
+    "eval_gen_len": 10.07649886877828,
+    "eval_loss": 2.0358376502990723,
+    "eval_rouge1": 28.3491,
+    "eval_rouge2": 14.2699,
+    "eval_rougeL": 27.7602,
+    "eval_rougeLsum": 27.752,
+    "eval_runtime": 189.2199,
     "eval_samples": 7072,
-    "eval_samples_per_second": 40.098,
-    "eval_steps_per_second": 1.253,
-    "train_loss": 1.4084750825324945,
-    "train_runtime": 9144.3174,
+    "eval_samples_per_second": 37.375,
+    "eval_steps_per_second": 1.168,
+    "train_loss": 1.3756333750635685,
+    "train_runtime": 9869.4584,
     "train_samples": 63639,
-    "train_samples_per_second": 139.188,
-    "train_steps_per_second": 1.087
+    "train_samples_per_second": 128.961,
+    "train_steps_per_second": 1.007
 }

config.json

@@ -1,8 +1,9 @@
 {
-  "_name_or_path": "mt0-xl",
+  "_name_or_path": "mt0-xl/",
   "architectures": [
     "MT5ForConditionalGeneration"
   ],
+  "classifier_dropout": 0.0,
   "d_ff": 5120,
   "d_kv": 64,
   "d_model": 2048,
@@ -26,7 +27,7 @@
   "tie_word_embeddings": false,
   "tokenizer_class": "T5Tokenizer",
   "torch_dtype": "float32",
-  "transformers_version": "4.30.2",
+  "transformers_version": "4.37.1",
   "use_cache": true,
   "vocab_size": 250112
 }

eval_results.json

@@ -1,13 +1,13 @@
 {
-    "epoch": 10.0,
-    "eval_gen_len": 8.93481334841629,
-    "eval_loss": 1.9881515502929688,
-    "eval_rouge1": 31.4539,
-    "eval_rouge2": 16.1017,
-    "eval_rougeL": 30.6959,
-    "eval_rougeLsum": 30.6888,
-    "eval_runtime": 176.3688,
+    "epoch": 11.0,
+    "eval_gen_len": 10.07649886877828,
+    "eval_loss": 2.0358376502990723,
+    "eval_rouge1": 28.3491,
+    "eval_rouge2": 14.2699,
+    "eval_rougeL": 27.7602,
+    "eval_rougeLsum": 27.752,
+    "eval_runtime": 189.2199,
     "eval_samples": 7072,
-    "eval_samples_per_second": 40.098,
-    "eval_steps_per_second": 1.253
+    "eval_samples_per_second": 37.375,
+    "eval_steps_per_second": 1.168
 }

generation_config.json

@@ -2,5 +2,5 @@
   "decoder_start_token_id": 0,
   "eos_token_id": 1,
   "pad_token_id": 0,
-  "transformers_version": "4.30.2"
+  "transformers_version": "4.37.1"
 }

pytorch_model-00001-of-00002.bin → pytorch_model-00001-of-00003.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:309ec67c5e94c3ffa70f23e49688256cd4022059e270daed36fa78c0b15ac31f
-size 9977020596
+oid sha256:8a1fe9a02ef2f2f7f2ad22d01aee5265f757e61e622917e74599ffdb0f3a1c67
+size 4993619647

pytorch_model-00002-of-00003.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bf8a6b2f0adc87722dee18bc9e03d784909841108eb101cf5c15172aefd979e
+size 4983398004

pytorch_model-00002-of-00002.bin → pytorch_model-00003-of-00003.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9bc631cc9a7f2880d2096313e2c5bcc77f3a6c853c38998df3ad0ce4e80b2053
+oid sha256:6c4e159dc91bb5a2b3b675915347c543eb153168a7f2b27fc176d1a86f5aad19
 size 4993663292

special_tokens_map.json

@@ -1,5 +1,23 @@
 {
-  "eos_token": "</s>",
-  "pad_token": "<pad>",
-  "unk_token": "<unk>"
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6502d07619068a98aa2d3bb531332a694ffe108ca6c6fe62a467ccfe98d666b9
-size 16315219
+oid sha256:c00dd03b7b29fa0ca79bd6b2ac2a9575b3175486939f4c3429a27812e2830bbb
+size 16315311

tokenizer_config.json

@@ -1,5 +1,31 @@
 {
-  "additional_special_tokens": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
   "clean_up_tokenization_spaces": true,
   "eos_token": "</s>",
   "extra_ids": 0,

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 10.0,
-    "train_loss": 1.4084750825324945,
-    "train_runtime": 9144.3174,
+    "epoch": 11.0,
+    "train_loss": 1.3756333750635685,
+    "train_runtime": 9869.4584,
     "train_samples": 63639,
-    "train_samples_per_second": 139.188,
-    "train_steps_per_second": 1.087
+    "train_samples_per_second": 128.961,
+    "train_steps_per_second": 1.007
 }

trainer_state.json

@@ -1,8 +1,9 @@
 {
-  "best_metric": 31.4465,
-  "best_model_checkpoint": "models/mt0-xl_norwegian_natprompt_adafactor/checkpoint-3480",
-  "epoch": 9.998994469582705,
-  "global_step": 4972,
+  "best_metric": 28.3473,
+  "best_model_checkpoint": "models/mt0-xl_norwegian_natprompt_updated/checkpoint-3978",
+  "epoch": 10.998491704374057,
+  "eval_steps": 500,
+  "global_step": 5469,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -10,206 +11,229 @@
     {
       "epoch": 1.0,
       "learning_rate": 4.75e-05,
-      "loss": 2.6123,
+      "loss": 2.6165,
       "step": 497
     },
     {
       "epoch": 1.0,
-      "eval_gen_len": 8.94951923076923,
-      "eval_loss": 1.8899891376495361,
-      "eval_rouge1": 28.8529,
-      "eval_rouge2": 13.6138,
-      "eval_rougeL": 28.1471,
-      "eval_rougeLsum": 28.1199,
-      "eval_runtime": 143.958,
-      "eval_samples_per_second": 49.125,
-      "eval_steps_per_second": 1.535,
+      "eval_gen_len": 8.414592760180996,
+      "eval_loss": 1.9032089710235596,
+      "eval_rouge1": 26.3511,
+      "eval_rouge2": 12.3506,
+      "eval_rougeL": 25.9106,
+      "eval_rougeLsum": 25.9089,
+      "eval_runtime": 138.9748,
+      "eval_samples_per_second": 50.887,
+      "eval_steps_per_second": 1.59,
       "step": 497
     },
     {
       "epoch": 2.0,
       "learning_rate": 4.5e-05,
-      "loss": 2.0642,
+      "loss": 2.0889,
       "step": 994
     },
     {
       "epoch": 2.0,
-      "eval_gen_len": 8.640554298642535,
-      "eval_loss": 1.8383045196533203,
-      "eval_rouge1": 30.5095,
-      "eval_rouge2": 15.2505,
-      "eval_rougeL": 29.8018,
-      "eval_rougeLsum": 29.7658,
-      "eval_runtime": 141.4332,
-      "eval_samples_per_second": 50.002,
-      "eval_steps_per_second": 1.563,
+      "eval_gen_len": 9.177601809954751,
+      "eval_loss": 1.8498029708862305,
+      "eval_rouge1": 27.424,
+      "eval_rouge2": 13.5368,
+      "eval_rougeL": 26.9384,
+      "eval_rougeLsum": 26.9301,
+      "eval_runtime": 137.6855,
+      "eval_samples_per_second": 51.363,
+      "eval_steps_per_second": 1.605,
       "step": 994
     },
     {
       "epoch": 3.0,
       "learning_rate": 4.25e-05,
-      "loss": 1.7611,
+      "loss": 1.7957,
       "step": 1491
     },
     {
       "epoch": 3.0,
-      "eval_gen_len": 8.715073529411764,
-      "eval_loss": 1.8447171449661255,
-      "eval_rouge1": 30.9812,
-      "eval_rouge2": 15.6932,
-      "eval_rougeL": 30.2339,
-      "eval_rougeLsum": 30.2413,
-      "eval_runtime": 144.3959,
-      "eval_samples_per_second": 48.976,
-      "eval_steps_per_second": 1.531,
+      "eval_gen_len": 8.836962669683258,
+      "eval_loss": 1.8434585332870483,
+      "eval_rouge1": 27.6735,
+      "eval_rouge2": 13.985,
+      "eval_rougeL": 27.1922,
+      "eval_rougeLsum": 27.2052,
+      "eval_runtime": 138.4103,
+      "eval_samples_per_second": 51.094,
+      "eval_steps_per_second": 1.597,
       "step": 1491
     },
     {
       "epoch": 4.0,
       "learning_rate": 3.999496981891348e-05,
-      "loss": 1.5284,
+      "loss": 1.5665,
       "step": 1989
     },
     {
       "epoch": 4.0,
-      "eval_gen_len": 9.378676470588236,
-      "eval_loss": 1.8618718385696411,
-      "eval_rouge1": 30.9706,
-      "eval_rouge2": 15.4516,
-      "eval_rougeL": 30.1888,
-      "eval_rougeLsum": 30.1911,
-      "eval_runtime": 144.7653,
-      "eval_samples_per_second": 48.851,
-      "eval_steps_per_second": 1.527,
+      "eval_gen_len": 8.857466063348417,
+      "eval_loss": 1.850651502609253,
+      "eval_rouge1": 28.2493,
+      "eval_rouge2": 14.358,
+      "eval_rougeL": 27.7524,
+      "eval_rougeLsum": 27.7456,
+      "eval_runtime": 138.4439,
+      "eval_samples_per_second": 51.082,
+      "eval_steps_per_second": 1.596,
       "step": 1989
     },
     {
       "epoch": 5.0,
       "learning_rate": 3.749496981891348e-05,
-      "loss": 1.3422,
+      "loss": 1.3801,
       "step": 2486
     },
     {
       "epoch": 5.0,
-      "eval_gen_len": 9.202630090497738,
-      "eval_loss": 1.889543890953064,
-      "eval_rouge1": 30.9451,
-      "eval_rouge2": 15.5242,
-      "eval_rougeL": 30.1826,
-      "eval_rougeLsum": 30.1837,
-      "eval_runtime": 149.0359,
-      "eval_samples_per_second": 47.452,
-      "eval_steps_per_second": 1.483,
+      "eval_gen_len": 9.235718325791856,
+      "eval_loss": 1.889418125152588,
+      "eval_rouge1": 28.2511,
+      "eval_rouge2": 14.2431,
+      "eval_rougeL": 27.6841,
+      "eval_rougeLsum": 27.6785,
+      "eval_runtime": 141.1671,
+      "eval_samples_per_second": 50.097,
+      "eval_steps_per_second": 1.566,
       "step": 2486
     },
     {
       "epoch": 6.0,
       "learning_rate": 3.499496981891348e-05,
-      "loss": 1.1862,
+      "loss": 1.2233,
       "step": 2983
     },
     {
       "epoch": 6.0,
-      "eval_gen_len": 8.88164592760181,
-      "eval_loss": 1.922377347946167,
-      "eval_rouge1": 31.3072,
-      "eval_rouge2": 15.959,
-      "eval_rougeL": 30.5538,
-      "eval_rougeLsum": 30.5404,
-      "eval_runtime": 145.1437,
-      "eval_samples_per_second": 48.724,
-      "eval_steps_per_second": 1.523,
+      "eval_gen_len": 9.103930995475114,
+      "eval_loss": 1.916949987411499,
+      "eval_rouge1": 28.3057,
+      "eval_rouge2": 14.349,
+      "eval_rougeL": 27.7482,
+      "eval_rougeLsum": 27.7371,
+      "eval_runtime": 138.9786,
+      "eval_samples_per_second": 50.886,
+      "eval_steps_per_second": 1.59,
       "step": 2983
     },
     {
       "epoch": 7.0,
       "learning_rate": 3.249496981891348e-05,
-      "loss": 1.0526,
+      "loss": 1.0877,
       "step": 3480
     },
     {
       "epoch": 7.0,
-      "eval_gen_len": 8.72935520361991,
-      "eval_loss": 1.9881515502929688,
-      "eval_rouge1": 31.4465,
-      "eval_rouge2": 16.095,
-      "eval_rougeL": 30.6871,
-      "eval_rougeLsum": 30.6739,
-      "eval_runtime": 188.8728,
-      "eval_samples_per_second": 37.443,
-      "eval_steps_per_second": 1.17,
+      "eval_gen_len": 9.20475113122172,
+      "eval_loss": 1.9742506742477417,
+      "eval_rouge1": 28.2671,
+      "eval_rouge2": 14.4585,
+      "eval_rougeL": 27.725,
+      "eval_rougeLsum": 27.7475,
+      "eval_runtime": 139.1018,
+      "eval_samples_per_second": 50.84,
+      "eval_steps_per_second": 1.589,
       "step": 3480
     },
     {
       "epoch": 8.0,
       "learning_rate": 2.9989939637826965e-05,
-      "loss": 0.9384,
+      "loss": 0.9717,
       "step": 3978
     },
     {
       "epoch": 8.0,
-      "eval_gen_len": 9.613404977375566,
-      "eval_loss": 2.0582804679870605,
-      "eval_rouge1": 31.1434,
-      "eval_rouge2": 15.7298,
-      "eval_rougeL": 30.287,
-      "eval_rougeLsum": 30.2831,
-      "eval_runtime": 148.3771,
-      "eval_samples_per_second": 47.662,
-      "eval_steps_per_second": 1.489,
+      "eval_gen_len": 9.785633484162895,
+      "eval_loss": 2.0358376502990723,
+      "eval_rouge1": 28.3473,
+      "eval_rouge2": 14.2734,
+      "eval_rougeL": 27.7737,
+      "eval_rougeLsum": 27.7661,
+      "eval_runtime": 142.8331,
+      "eval_samples_per_second": 49.512,
+      "eval_steps_per_second": 1.547,
       "step": 3978
     },
     {
       "epoch": 9.0,
       "learning_rate": 2.7489939637826962e-05,
-      "loss": 0.8408,
+      "loss": 0.8777,
       "step": 4475
     },
     {
       "epoch": 9.0,
-      "eval_gen_len": 9.652714932126697,
-      "eval_loss": 2.1236588954925537,
-      "eval_rouge1": 30.7808,
-      "eval_rouge2": 15.4943,
-      "eval_rougeL": 29.9606,
-      "eval_rougeLsum": 29.9589,
-      "eval_runtime": 147.2785,
-      "eval_samples_per_second": 48.018,
-      "eval_steps_per_second": 1.501,
+      "eval_gen_len": 9.886453619909503,
+      "eval_loss": 2.0969080924987793,
+      "eval_rouge1": 27.7863,
+      "eval_rouge2": 13.8157,
+      "eval_rougeL": 27.1987,
+      "eval_rougeLsum": 27.1859,
+      "eval_runtime": 141.209,
+      "eval_samples_per_second": 50.082,
+      "eval_steps_per_second": 1.565,
       "step": 4475
     },
     {
       "epoch": 10.0,
       "learning_rate": 2.4989939637826962e-05,
-      "loss": 0.7592,
+      "loss": 0.7983,
       "step": 4972
     },
     {
       "epoch": 10.0,
-      "eval_gen_len": 9.92548076923077,
-      "eval_loss": 2.198713779449463,
-      "eval_rouge1": 31.0097,
-      "eval_rouge2": 15.5823,
-      "eval_rougeL": 30.1202,
-      "eval_rougeLsum": 30.1151,
-      "eval_runtime": 147.1485,
-      "eval_samples_per_second": 48.06,
-      "eval_steps_per_second": 1.502,
+      "eval_gen_len": 9.238829185520363,
+      "eval_loss": 2.1536314487457275,
+      "eval_rouge1": 28.2427,
+      "eval_rouge2": 14.3725,
+      "eval_rougeL": 27.6965,
+      "eval_rougeLsum": 27.7019,
+      "eval_runtime": 138.9465,
+      "eval_samples_per_second": 50.897,
+      "eval_steps_per_second": 1.591,
       "step": 4972
     },
     {
-      "epoch": 10.0,
-      "step": 4972,
-      "total_flos": 2.421678655143936e+17,
-      "train_loss": 1.4084750825324945,
-      "train_runtime": 9144.3174,
-      "train_samples_per_second": 139.188,
-      "train_steps_per_second": 1.087
+      "epoch": 11.0,
+      "learning_rate": 2.2489939637826963e-05,
+      "loss": 0.7261,
+      "step": 5469
+    },
+    {
+      "epoch": 11.0,
+      "eval_gen_len": 9.441742081447964,
+      "eval_loss": 2.1868813037872314,
+      "eval_rouge1": 28.0261,
+      "eval_rouge2": 14.1267,
+      "eval_rougeL": 27.4232,
+      "eval_rougeLsum": 27.4261,
+      "eval_runtime": 140.5761,
+      "eval_samples_per_second": 50.307,
+      "eval_steps_per_second": 1.572,
+      "step": 5469
+    },
+    {
+      "epoch": 11.0,
+      "step": 5469,
+      "total_flos": 2.6645663357023027e+17,
+      "train_loss": 1.3756333750635685,
+      "train_runtime": 9869.4584,
+      "train_samples_per_second": 128.961,
+      "train_steps_per_second": 1.007
     }
   ],
+  "logging_steps": 500,
   "max_steps": 9940,
+  "num_input_tokens_seen": 0,
   "num_train_epochs": 20,
-  "total_flos": 2.421678655143936e+17,
+  "save_steps": 500,
+  "total_flos": 2.6645663357023027e+17,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8ea83b67b65f113aa36d935cdb4d3a537ad76372cda288acf58eb6f9c599fac1
-size 4091
+oid sha256:b7a798cecc88bb22c41b9c8ab4c97ef2a2bdbd9d8ec25ade15a04a3f45d8db18
+size 4411

upload.py

@@ -1,11 +0,0 @@
-#!/bin/env python3
-
-import sys
-from huggingface_hub import HfApi
-from huggingface_hub import create_repo
-
-create_repo(sys.argv[1])
-api = HfApi()
-
-api.upload_folder(folder_path=".", repo_id=sys.argv[1], repo_type="model")
-