Update spaCy pipeline

README.md

@@ -13,25 +13,25 @@ model-index:
     metrics:
     - name: NER Precision
       type: precision
-      value: 0.61849542
+      value: 0.5982281487
     - name: NER Recall
       type: recall
-      value: 0.5961863611
+      value: 0.6152545557
     - name: NER F Score
       type: f_score
-      value: 0.6071360245
+      value: 0.6066219032
 ---
 | Feature | Description |
 | --- | --- |
 | **Name** | `ru_patents_ner` |
-| **Version** | `1.0.0` |
+| **Version** | `1.0.1` |
 | **spaCy** | `>=3.8.4,<3.9.0` |
 | **Default Pipeline** | `transformer`, `ner` |
 | **Components** | `transformer`, `ner` |
 | **Vectors** | 500002 keys, 500002 unique vectors (300 dimensions) |
 | **Sources** | n/a |
 | **License** | n/a |
-| **Author** | [Delicalib](https://huggingface.co/Delicalib) |
+| **Author** | [n/a]() |
 
 ### Label Scheme
 
@@ -49,8 +49,13 @@ model-index:
 
 | Type | Score |
 | --- | --- |
-| `ENTS_F` | 60.71 |
-| `ENTS_P` | 61.85 |
-| `ENTS_R` | 59.62 |
-| `TRANSFORMER_LOSS` | 21357700.30 |
-| `NER_LOSS` | 883914.55 |
+| `F1_MICRO` | 59.95 |
+| `F1_MACRO` | 55.92 |
+| `F1_WEIGHTED` | 59.51 |
+| `F1_COMPONENT` | 66.29 |
+| `F1_SYSTEM` | 66.62 |
+| `F1_ATTRIBUTE` | 34.84 |
+| `ENTS_P` | 60.95 |
+| `ENTS_R` | 58.99 |
+| `TRANSFORMER_LOSS` | 907006.78 |
+| `NER_LOSS` | 1524129.39 |

config.cfg

@@ -22,7 +22,7 @@ vectors = {"@vectors":"spacy.Vectors.v1"}
 [components]
 
 [components.ner]
-factory = "ner"
+factory = "ner_all_metrics"
 incorrect_spans_key = null
 moves = null
 scorer = {"@scorers":"spacy.ner_scorer.v1"}
@@ -84,12 +84,12 @@ limit = 0
 augmenter = null
 
 [training]
-accumulate_gradient = 3
+accumulate_gradient = 1
 dev_corpus = "corpora.dev"
 train_corpus = "corpora.train"
 seed = ${system.seed}
 gpu_allocator = ${system.gpu_allocator}
-dropout = 0.1
+dropout = 0.2
 patience = 1600
 max_epochs = 0
 max_steps = 20000
@@ -102,7 +102,7 @@ before_update = null
 [training.batcher]
 @batchers = "spacy.batch_by_padded.v1"
 discard_oversize = true
-size = 2000
+size = 1000
 buffer = 256
 get_length = null
 
@@ -122,14 +122,19 @@ eps = 0.00000001
 
 [training.optimizer.learn_rate]
 @schedules = "warmup_linear.v1"
-warmup_steps = 250
+warmup_steps = 300
 total_steps = 20000
-initial_rate = 0.00005
+initial_rate = 0.000005
 
 [training.score_weights]
-ents_f = 1.0
+f1_COMPONENT = 0.17
+f1_SYSTEM = 0.17
+f1_ATTRIBUTE = 0.17
 ents_p = 0.0
 ents_r = 0.0
+f1_macro = 0.17
+f1_weighted = 0.17
+ents_f = 0.17
 ents_per_type = null
 
 [pretraining]

custom_factory.py

@@ -0,0 +1,87 @@
+from spacy.pipeline.ner import EntityRecognizer
+from spacy.language import DEFAULT_CONFIG, Language
+from thinc.api import Config
+from sklearn.metrics import f1_score, precision_recall_fscore_support
+
+
+default_model_config = """
+[model]
+@architectures = "spacy.TransitionBasedParser.v2"
+state_type = "ner"
+extra_state_tokens = false
+hidden_width = 64
+maxout_pieces = 2
+use_upper = false
+nO = null
+
+[model.tok2vec]
+@architectures = "spacy-transformers.TransformerListener.v1"
+grad_factor = 1.0
+pooling = {"@layers":"reduce_mean.v1"}
+upstream = "*"
+"""
+DEFAULT_MODEL = Config().from_str(default_model_config)["model"]
+
+@Language.factory("ner_all_metrics",
+  default_config={
+    "model": DEFAULT_MODEL,
+    "moves": None,
+    "scorer": {"@scorers": "spacy.ner_scorer.v1"},
+    "incorrect_spans_key": None,
+    "update_with_oracle_cut_size": 100
+    },
+    default_score_weights={
+        "f1_COMPONENT": 1.0,
+        "f1_SYSTEM": 1.0,
+        "f1_ATTRIBUTE": 1.0,
+        "ents_p": 0.0,
+        "ents_r": 0.0,
+        "f1_macro": 1.0,
+        "f1_weighted": 1.0,
+        "ents_f": 1.0,
+    })
+def create_ner_all_metrics(nlp, name, model, moves, scorer, incorrect_spans_key, update_with_oracle_cut_size):
+    return NERWithAllMetrics(nlp.vocab, model, name=name, moves=moves, scorer=scorer,
+                              incorrect_spans_key=incorrect_spans_key,
+                              update_with_oracle_cut_size=update_with_oracle_cut_size)
+
+class NERWithAllMetrics(EntityRecognizer):
+    def score(self, examples, **kwargs):
+        scores = super().score(examples, **kwargs)
+        scores = dict(list(scores.items()) + list(self.custom_scorer(examples).items()))
+        scores["ents_f"] = scores["f1_micro"]
+        del scores["f1_micro"]
+        return scores
+
+    def custom_scorer(self, examples):
+      y_true = []
+      y_pred = []
+      for example in examples:
+          gold = {(ent.start_char, ent.end_char, ent.label_) for ent in example.reference.ents}
+          pred = {(ent.start_char, ent.end_char, ent.label_) for ent in example.predicted.ents}
+          all_spans = gold | pred
+          for span in all_spans:
+              if span in gold and span in pred:
+                  y_true.append(span[2])
+                  y_pred.append(span[2])
+              elif span in gold:
+                  y_true.append(span[2])
+                  y_pred.append("O")
+              elif span in pred:
+                  y_true.append("O")
+                  y_pred.append(span[2])
+
+      labels = sorted({label for label in y_true if label != "O"})
+
+      precision, recall, f1, support = precision_recall_fscore_support(
+          y_true, y_pred, labels=labels, zero_division=0, average=None
+      )
+      result = {}
+      for l, p, r, f in zip(labels, precision, recall, f1):
+          result[f"f1_{l}"] = f
+
+      result["f1_micro"] = f1_score(y_true, y_pred, average="micro", labels=labels, zero_division=0)
+      result["f1_macro"] = f1_score(y_true, y_pred, average="macro", labels=labels, zero_division=0)
+      result["f1_weighted"] = f1_score(y_true, y_pred, average="weighted", labels=labels, zero_division=0)
+
+      return result

meta.json

@@ -1,7 +1,7 @@
 {
   "lang":"ru",
   "name":"patents_ner",
-  "version":"1.0.0",
+  "version":"1.0.1",
   "description":"",
   "author":"",
   "email":"",
@@ -37,30 +37,36 @@
 
   ],
   "performance":{
-    "ents_f":0.6071360245,
-    "ents_p":0.61849542,
-    "ents_r":0.5961863611,
+    "f1_COMPONENT":0.6689663984,
+    "f1_SYSTEM":0.6772068511,
+    "f1_ATTRIBUTE":0.3617454217,
+    "ents_p":0.5982281487,
+    "ents_r":0.6152545557,
+    "f1_macro":0.5693062237,
+    "f1_weighted":0.6031160932,
+    "ents_f":0.6066219032,
     "ents_per_type":{
       "SYSTEM":{
-        "p":0.7098765432,
-        "r":0.6133333333,
-        "f":0.6580829757
+        "p":0.6692708333,
+        "r":0.6853333333,
+        "f":0.6772068511
       },
       "ATTRIBUTE":{
-        "p":0.4098671727,
-        "r":0.2814943527,
-        "f":0.3337625547
+        "p":0.3771805752,
+        "r":0.3475238923,
+        "f":0.3617454217
       },
       "COMPONENT":{
-        "p":0.652184738,
-        "r":0.6898867527,
-        "f":0.6705061752
+        "p":0.6496277916,
+        "r":0.689491704,
+        "f":0.6689663984
       }
     },
-    "transformer_loss":213577.0030247931,
-    "ner_loss":8839.1455078125
+    "transformer_loss":9419.470294231,
+    "ner_loss":14866.390625
   },
   "requirements":[
-    "spacy-transformers>=1.3.8,<1.4.0"
+    "spacy-transformers>=1.3.8,<1.4.0",
+    "spacy>=3.8.4,<3.9.0"
   ]
 }

ner/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5da257b2d82f31304f6c22db4498523c379293b02170d815f86e06716f641632
+oid sha256:ef3ac16ec06d58f22ee3f23b71b37b706697b3c5099cfdbd27f42ec5fcc4155a
 size 285226

ru_patents_ner-any-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c04a8a1e0082c81fb457ac785059f6c2ebf63305224e3be65996a85ad6b5cf2
-size 1822022001
+oid sha256:f78862070ad285cfd7c0bce13a2457d8c70eeb8e5ece21d00215d83f2c22caba
+size 1822037615

transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c4949f5d3d8e08a828297066f3c69d91a090b7584e37c93a67cc83649e9c1ed1
+oid sha256:993ccc8d74fab373bc75f00ef70e63c9acd93ed7006a5d49c3de375694aa808f
 size 1430062590