                  best = i
      return best
  
  
  class ParserModel(Model):
      def __init__(self, tok2vec, lower_model, upper_model, unseen_classes=None):
          # don't define nO for this object, because we can't dynamically change it
          Model.__init__(self, name="parser_model", forward=forward, dims={"nI": None})
          if tok2vec.has_dim("nI"):
              self.set_dim("nI", tok2vec.get_dim("nI"))
          self._layers = [tok2vec, lower_model]
          if upper_model is not None:
              self._layers.append(upper_model)
          self.unseen_classes = set()
          if unseen_classes:
              for class_ in unseen_classes:
                  self.unseen_classes.add(class_)
          self.set_ref("tok2vec", tok2vec)
  
      def predict(self, docs):
          step_model = ParserStepModel(docs, self._layers,
                          unseen_classes=self.unseen_classes, train=False)
          return step_model
  
      def resize_output(self, new_nO):
          if len(self._layers) == 2:
              return
          if self.upper.has_dim("nO") and (new_nO == self.upper.get_dim("nO")):
              return
          smaller = self.upper
          nI = None
          if smaller.has_dim("nI"):
              nI = smaller.get_dim("nI")
          with use_ops('numpy'):
              larger = Linear(nO=new_nO, nI=nI)
              larger.init = smaller.init
          # it could be that the model is not initialized yet, then skip this bit
          if nI:
              larger_W = larger.ops.alloc2f(new_nO, nI)
              larger_b = larger.ops.alloc1f(new_nO)
              smaller_W = smaller.get_param("W")
              smaller_b = smaller.get_param("b")
              # Weights are stored in (nr_out, nr_in) format, so we're basically
              # just adding rows here.
              if smaller.has_dim("nO"):
                  larger_W[:smaller.get_dim("nO")] = smaller_W
                  larger_b[:smaller.get_dim("nO")] = smaller_b
                  for i in range(smaller.get_dim("nO"), new_nO):
                      self.unseen_classes.add(i)
  
              larger.set_param("W", larger_W)
              larger.set_param("b", larger_b)
          self._layers[-1] = larger
  
      def initialize(self, X=None, Y=None):
          self.tok2vec.initialize()
          self.lower.initialize(X=X, Y=Y)
          if self.upper is not None:
              # In case we need to trigger the callbacks
              statevecs = self.ops.alloc((2, self.lower.get_dim("nO")))
              self.upper.initialize(X=statevecs)
  
      def finish_update(self, optimizer):
          self.tok2vec.finish_update(optimizer)
          self.lower.finish_update(optimizer)
          if self.upper is not None:
              self.upper.finish_update(optimizer)
  
      @property
      def tok2vec(self):
          return self._layers[0]
  
      @property
      def lower(self):
          return self._layers[1]
  
      @property
      def upper(self):
          return self._layers[2]
  
  
  def forward(model:ParserModel, X, is_train):
      step_model = ParserStepModel(X, model._layers, unseen_classes=model.unseen_classes,
          train=is_train)
  
      return step_model, step_model.finish_steps
  
  
  class ParserStepModel(Model):
      def __init__(self, docs, layers, unseen_classes=None, train=True):
          Model.__init__(self, name="parser_step_model", forward=step_forward)
          self.tokvecs, self.bp_tokvecs = layers[0](docs, is_train=train)
 
</s> remove     def __init__(self, docs, layers, unseen_classes=None, train=True):
 </s> add     def __init__(self, docs, layers, *, has_upper, unseen_classes=None, train=True): </s> remove         if self.vec2scores is None:
            self._class_mask = numpy.zeros((self.state2vec.nO,), dtype='f')
        else:
            self._class_mask = numpy.zeros((self.vec2scores.get_dim("nO"),), dtype='f')
 </s> add         self._class_mask = numpy.zeros((self.nO,), dtype='f') </s> remove     if model.vec2scores is not None:
 </s> add     if model.attrs["has_upper"]: </s> remove     if "ner" not in nlp.pipe_names:
        ner = nlp.create_pipe("ner")
 </s> add     if "simple_ner" not in nlp.pipe_names:
        ner = nlp.create_pipe("simple_ner")
                  best = i
      return best
  
  



















































































  
  class ParserStepModel(Model):
      def __init__(self, docs, layers, unseen_classes=None, train=True):
          Model.__init__(self, name="parser_step_model", forward=step_forward)
          self.tokvecs, self.bp_tokvecs = layers[0](docs, is_train=train)
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace keep keep keep keep keep
 Adapt parser and NER for transformers (#5449)

* Draft layer for BILUO actions

* Fixes to biluo layer

* WIP on BILUO layer

* Add tests for BILUO layer

* Format

* Fix transitions

* Update test

* Link in the simple_ner

* Update BILUO tagger

* Update __init__

* Import simple_ner

* Update test

* Import

* Add files

* Add config

* Fix label passing for BILUO and tagger

* Fix label handling for simple_ner component

* Update simple NER test

* Update config

* Hack train script

* Update BILUO layer

* Fix SimpleNER component

* Update train_from_config

* Add biluo_to_iob helper

* Add IOB layer

* Add IOBTagger model

* Update biluo layer

* Update SimpleNER tagger

* Update BILUO

* Read random seed in train-from-config

* Update use of normal_init

* Fix normalization of gradient in SimpleNER

* Update IOBTagger

* Remove print

* Tweak masking in BILUO

* Add dropout in SimpleNER

* Update thinc

* Tidy up simple_ner

* Fix biluo model

* Unhack train-from-config

* Update setup.cfg and requirements

* Add tb_framework.py for parser model

* Try to avoid memory leak in BILUO

* Move ParserModel into spacy.ml, avoid need for subclass.

* Use updated parser model

* Remove incorrect call to model.initializre in PrecomputableAffine

* Update parser model

* Avoid divide by zero in tagger

* Add extra dropout layer in tagger

* Refine minibatch_by_words function to avoid oom

* Fix parser model after refactor

* Try to avoid div-by-zero in SimpleNER

* Fix infinite loop in minibatch_by_words

* Use SequenceCategoricalCrossentropy in Tagger

* Fix parser model when hidden layer

* Remove extra dropout from tagger

* Add extra nan check in tagger

* Fix thinc version

* Update tests and imports

* Fix test

* Update test

* Update tests

* Fix tests

* Fix test

Co-authored-by: Ines Montani <ines@ines.io> 
spacy/syntax/_parser_model.pyx