Update spaCy pipeline

README.md

@@ -13,19 +13,19 @@ model-index:
     metrics:
     - name: NER Precision
       type: precision
-      value: 0.5982281487
+      value: 0.6187035922
     - name: NER Recall
       type: recall
-      value: 0.6152545557
+      value: 0.6062930187
     - name: NER F Score
       type: f_score
-      value: 0.6066219032
+      value: 0.612435439
 ---
 | Feature | Description |
 | --- | --- |
 | **Name** | `ru_patents_ner` |
-| **Version** | `1.0.1` |
-| **spaCy** | `>=3.8.4,<3.9.0` |
+| **Version** | `1.0.0` |
+| **spaCy** | `>=3.8.5,<3.9.0` |
 | **Default Pipeline** | `transformer`, `ner` |
 | **Components** | `transformer`, `ner` |
 | **Vectors** | 500002 keys, 500002 unique vectors (300 dimensions) |
@@ -49,13 +49,14 @@ model-index:
 
 | Type | Score |
 | --- | --- |
-| `F1_MICRO` | 59.95 |
-| `F1_MACRO` | 55.92 |
-| `F1_WEIGHTED` | 59.51 |
-| `F1_COMPONENT` | 66.29 |
-| `F1_SYSTEM` | 66.62 |
-| `F1_ATTRIBUTE` | 34.84 |
-| `ENTS_P` | 60.95 |
-| `ENTS_R` | 58.99 |
-| `TRANSFORMER_LOSS` | 907006.78 |
-| `NER_LOSS` | 1524129.39 |
+| `F1_MICRO` | 61.24 |
+| `F1_MACRO` | 54.82 |
+| `F1_WEIGHTED` | 60.09 |
+| `F1_COMPONENT` | 67.20 |
+| `F1_SYSTEM` | 64.79 |
+| `F1_ATTRIBUTE` | 32.48 |
+| `ENTS_P` | 61.87 |
+| `ENTS_R` | 60.63 |
+| `ENTS_F` | 61.24 |
+| `TRANSFORMER_LOSS` | 144452.32 |
+| `NER_LOSS` | 222665.13 |

config.cfg

@@ -1,8 +1,9 @@
 [paths]
-train = "./Diplom/dataset/all_ner_train.spacy"
-dev = "./Diplom/dataset/all_ner_test.spacy"
+train = "./data/all_ner_train.spacy"
+dev = "./data/all_ner_test.spacy"
 vectors = "ru_core_news_lg"
 init_tok2vec = null
+model = "./model-best"
 
 [system]
 gpu_allocator = "pytorch"
@@ -23,6 +24,7 @@ vectors = {"@vectors":"spacy.Vectors.v1"}
 
 [components.ner]
 factory = "ner_all_metrics"
+eval_frequency = ${training.eval_frequency}
 incorrect_spans_key = null
 moves = null
 scorer = {"@scorers":"spacy.ner_scorer.v1"}
@@ -50,7 +52,7 @@ set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotati
 
 [components.transformer.model]
 @architectures = "spacy-transformers.TransformerModel.v3"
-name = "ai-forever/ruRoberta-large"
+name = "DeepPavlov/rubert-base-cased"
 mixed_precision = false
 
 [components.transformer.model.get_spans]
@@ -84,16 +86,16 @@ limit = 0
 augmenter = null
 
 [training]
-accumulate_gradient = 1
+accumulate_gradient = 3
 dev_corpus = "corpora.dev"
 train_corpus = "corpora.train"
 seed = ${system.seed}
 gpu_allocator = ${system.gpu_allocator}
-dropout = 0.2
+dropout = 0.1
 patience = 1600
 max_epochs = 0
-max_steps = 20000
-eval_frequency = 200
+max_steps = 3000
+eval_frequency = 50
 frozen_components = []
 annotating_components = []
 before_to_disk = null
@@ -122,19 +124,20 @@ eps = 0.00000001
 
 [training.optimizer.learn_rate]
 @schedules = "warmup_linear.v1"
-warmup_steps = 300
-total_steps = 20000
-initial_rate = 0.000005
+warmup_steps = 250
+total_steps = 10000
+initial_rate = 0.00005
 
 [training.score_weights]
-f1_COMPONENT = 0.17
-f1_SYSTEM = 0.17
-f1_ATTRIBUTE = 0.17
+f1_micro = 0.14
+f1_macro = 0.14
+f1_weighted = 0.14
+f1_COMPONENT = 0.14
+f1_SYSTEM = 0.14
+f1_ATTRIBUTE = 0.14
 ents_p = 0.0
 ents_r = 0.0
-f1_macro = 0.17
-f1_weighted = 0.17
-ents_f = 0.17
+ents_f = 0.14
 ents_per_type = null
 
 [pretraining]

custom_factory.py

@@ -1,7 +1,13 @@
 from spacy.pipeline.ner import EntityRecognizer
-from spacy.language import DEFAULT_CONFIG, Language
+from spacy.language import Language
 from thinc.api import Config
 from sklearn.metrics import f1_score, precision_recall_fscore_support
+import plotly.express as px
+import plotly.graph_objects as go
+import time
+import json
+import os
+from pathlib import Path
 
 
 default_model_config = """
@@ -22,35 +28,59 @@ upstream = "*"
 """
 DEFAULT_MODEL = Config().from_str(default_model_config)["model"]
 
+
 @Language.factory("ner_all_metrics",
-  default_config={
-    "model": DEFAULT_MODEL,
-    "moves": None,
-    "scorer": {"@scorers": "spacy.ner_scorer.v1"},
-    "incorrect_spans_key": None,
-    "update_with_oracle_cut_size": 100
+    default_config={
+        "model": DEFAULT_MODEL,
+        "moves": None,
+        "scorer": {"@scorers": "spacy.ner_scorer.v1"},
+        "incorrect_spans_key": None,
+        "update_with_oracle_cut_size": 100,
+        "eval_frequency": 100,
     },
     default_score_weights={
+        "f1_micro": 1.0,
+        "f1_macro": 1.0,
+        "f1_weighted": 1.0,
         "f1_COMPONENT": 1.0,
         "f1_SYSTEM": 1.0,
         "f1_ATTRIBUTE": 1.0,
         "ents_p": 0.0,
         "ents_r": 0.0,
-        "f1_macro": 1.0,
-        "f1_weighted": 1.0,
-        "ents_f": 1.0,
     })
-def create_ner_all_metrics(nlp, name, model, moves, scorer, incorrect_spans_key, update_with_oracle_cut_size):
-    return NERWithAllMetrics(nlp.vocab, model, name=name, moves=moves, scorer=scorer,
-                              incorrect_spans_key=incorrect_spans_key,
-                              update_with_oracle_cut_size=update_with_oracle_cut_size)
+def create_ner_all_metrics(
+    nlp, name, 
+    model, moves, 
+    scorer, incorrect_spans_key, 
+    update_with_oracle_cut_size, eval_frequency
+):
+    return NERWithAllMetrics(
+        nlp.vocab, model, 
+        name=name, moves=moves, 
+        scorer=scorer, incorrect_spans_key=incorrect_spans_key,
+        update_with_oracle_cut_size=update_with_oracle_cut_size, eval_frequency=eval_frequency
+    )
+
 
 class NERWithAllMetrics(EntityRecognizer):
+    
+    def __init__(self, *args, eval_frequency=100, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.metric_history = []
+        self.max_f1 = 0
+        self.max_f1_step = 0
+        self.eval_frequency = eval_frequency
+        self.start_learning_time = None
+        
     def score(self, examples, **kwargs):
         scores = super().score(examples, **kwargs)
         scores = dict(list(scores.items()) + list(self.custom_scorer(examples).items()))
-        scores["ents_f"] = scores["f1_micro"]
-        del scores["f1_micro"]
+        tmp_scores = scores.copy()
+        tmp_scores["step"] = len(self.metric_history) * self.eval_frequency
+        if tmp_scores["f1_macro"] > self.max_f1:
+            self.max_f1 = tmp_scores["f1_macro"]
+            self.max_f1_step = tmp_scores["step"]
+        self.metric_history.append(tmp_scores)
         return scores
 
     def custom_scorer(self, examples):
@@ -85,3 +115,75 @@ class NERWithAllMetrics(EntityRecognizer):
       result["f1_weighted"] = f1_score(y_true, y_pred, average="weighted", labels=labels, zero_division=0)
 
       return result
+
+    def preprocess_metric_history(self):
+        result = {
+            "metric_name": [],
+            "metric_value": [],
+            "step": []
+        }
+        for cur_metrics in self.metric_history:
+            cur_step = cur_metrics["step"]
+            for key, value in cur_metrics.items():
+                if key != "step" and isinstance(value, float):
+                    result["metric_name"].append(key)
+                    result["metric_value"].append(value)
+                    result["step"].append(cur_step)
+        return result
+
+    def save_metrics_history(self, path):
+        if self.start_learning_time is None:
+            self.start_learning_time = time.monotonic()
+
+        if self.metric_history:
+
+            metrics_history_to_save = self.preprocess_metric_history()
+            fig = px.line(metrics_history_to_save, x="step", y="metric_value", color="metric_name")
+            for trace in fig.data:
+                if trace.name in ["f1_micro", "f1_macro", "f1_weighted"]:
+                    trace.line.width = 6
+                else:
+                    trace.line.width = 1
+
+                idx = list(trace.x).index(self.max_f1_step)
+                highlight_y = list(trace.y)[idx]
+                line_color = trace.line.color
+                line_name = trace.name
+                fig.add_trace(go.Scatter(
+                    x=[self.max_f1_step], y=[highlight_y],
+                    mode='markers+text',
+                    marker=dict(
+                        color=line_color, size=10),
+                        text=[f"{round(highlight_y, 2)}"],
+                        textposition="top center",
+                        name=f"{line_name} best"
+                    ))
+
+            current_time = time.monotonic()
+            current_time_of_training = current_time - self.start_learning_time
+            current_time_of_training_text = f"{int(current_time_of_training // 3600)} hrs {int(current_time_of_training % 3600) // 60} min {round(current_time_of_training % 60)} sec"
+
+            fig.update_layout(title = dict(
+                text="Training statistics",
+                subtitle=dict(
+                    text=f"Training time amounted to {current_time_of_training_text}",
+                    font=dict(color="gray", size=13),
+                )
+            ))
+
+            output_dir = os.path.join(str(path), "logs")
+            os.makedirs(output_dir, exist_ok=True)
+            fig_path = os.path.join(output_dir, "training_metrics.html")
+            json_path = os.path.join(output_dir, "training_metrics.json")
+            fig.write_html(fig_path)
+            with open(json_path, "w", encoding="utf-8") as f:
+                json.dump({
+                    "data": metrics_history_to_save,
+                    "train_time_s": current_time_of_training
+                }, f, indent=2, ensure_ascii=False)
+
+    def to_disk(self, path, *args, **kwargs):
+        super().to_disk(path, *args, **kwargs)
+        output_dir = Path(path)
+        output_dir_metrics = output_dir.parent.parent
+        self.save_metrics_history(output_dir_metrics)

meta.json

@@ -1,14 +1,14 @@
 {
   "lang":"ru",
   "name":"patents_ner",
-  "version":"1.0.1",
+  "version":"1.0.0",
   "description":"",
   "author":"",
   "email":"",
   "url":"",
   "license":"",
-  "spacy_version":">=3.8.4,<3.9.0",
-  "spacy_git_version":"85cc763",
+  "spacy_version":">=3.8.5,<3.9.0",
+  "spacy_git_version":"d0c705c",
   "vectors":{
     "width":300,
     "vectors":500002,
@@ -37,36 +37,37 @@
 
   ],
   "performance":{
-    "f1_COMPONENT":0.6689663984,
-    "f1_SYSTEM":0.6772068511,
-    "f1_ATTRIBUTE":0.3617454217,
-    "ents_p":0.5982281487,
-    "ents_r":0.6152545557,
-    "f1_macro":0.5693062237,
-    "f1_weighted":0.6031160932,
-    "ents_f":0.6066219032,
+    "f1_micro":0.612435439,
+    "f1_macro":0.5482288349,
+    "f1_weighted":0.6009293658,
+    "f1_COMPONENT":0.6719853518,
+    "f1_SYSTEM":0.6479250335,
+    "f1_ATTRIBUTE":0.3247761194,
+    "ents_p":0.6187035922,
+    "ents_r":0.6062930187,
+    "ents_f":0.612435439,
     "ents_per_type":{
       "SYSTEM":{
-        "p":0.6692708333,
-        "r":0.6853333333,
-        "f":0.6772068511
+        "p":0.6419098143,
+        "r":0.6540540541,
+        "f":0.6479250335
       },
       "ATTRIBUTE":{
-        "p":0.3771805752,
-        "r":0.3475238923,
-        "f":0.3617454217
+        "p":0.4121212121,
+        "r":0.2679802956,
+        "f":0.3247761194
       },
       "COMPONENT":{
-        "p":0.6496277916,
-        "r":0.689491704,
-        "f":0.6689663984
+        "p":0.6510390269,
+        "r":0.6943243243,
+        "f":0.6719853518
       }
     },
-    "transformer_loss":9419.470294231,
-    "ner_loss":14866.390625
+    "transformer_loss":1444.5232329108,
+    "ner_loss":2226.6513070879
   },
   "requirements":[
     "spacy-transformers>=1.3.8,<1.4.0",
-    "spacy>=3.8.4,<3.9.0"
+    "spacy>=3.8.5,<3.9.0"
   ]
 }

ner/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ef3ac16ec06d58f22ee3f23b71b37b706697b3c5099cfdbd27f42ec5fcc4155a
-size 285226
+oid sha256:6e5cb4fb854f36288d0807225cf54aa7f980104e14b948225622b496282217d6
+size 219690

ner/moves

@@ -1 +1 @@
-��moves��{"0":{},"1":{"COMPONENT":60434,"ATTRIBUTE":22497,"SYSTEM":8460},"2":{"COMPONENT":60434,"ATTRIBUTE":22497,"SYSTEM":8460},"3":{"COMPONENT":60434,"ATTRIBUTE":22497,"SYSTEM":8460},"4":{"COMPONENT":60434,"ATTRIBUTE":22497,"SYSTEM":8460,"":1},"5":{"":1}}�cfg��neg_key�
+��moves��{"0":{},"1":{"COMPONENT":63715,"ATTRIBUTE":22585,"SYSTEM":8551},"2":{"COMPONENT":63715,"ATTRIBUTE":22585,"SYSTEM":8551},"3":{"COMPONENT":63715,"ATTRIBUTE":22585,"SYSTEM":8551},"4":{"COMPONENT":63715,"ATTRIBUTE":22585,"SYSTEM":8551,"":1},"5":{"":1}}�cfg��neg_key�

ru_patents_ner-any-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f78862070ad285cfd7c0bce13a2457d8c70eeb8e5ece21d00215d83f2c22caba
-size 1822037615
+oid sha256:c30bb3920ae29fc3d514a4b955afddd4511e6bbe75d6c43adf6c505a9256c263
+size 1161805382

transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:993ccc8d74fab373bc75f00ef70e63c9acd93ed7006a5d49c3de375694aa808f
-size 1430062590
+oid sha256:59df9f2f2fd2593383e86d5e08a2f1583c6b2364caf5b20fb9734ead15fc8884
+size 716719271

vocab/strings.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48b62483d2ae9ba9e53ee3819ce1ad644df11b4c3a89256e9a1ccca552435c16
-size 49634993
+oid sha256:8af09429a518e34158d1f63f2eeb6326a59efbd36602a82fb0bec19c7ebe60ef
+size 49607747