      results = []
      losses = {}
      for step, batch in enumerate(train_data):
          dropout = next(dropouts)
          for subbatch in subdivide_batch(batch):
              nlp.update(subbatch, drop=dropout, losses=losses, sgd=False)
          for name, proc in nlp.pipeline:
              if hasattr(proc, "model"):
                  proc.model.finish_update(optimizer)
          optimizer.step_schedules()
 
</s> remove def subdivide_batch(batch):
    return [batch]
 </s> add def subdivide_batch(batch, accumulate_gradient):
    batch = list(batch)
    batch.sort(key=lambda eg: len(eg.doc))
    sub_len = len(batch) // accumulate_gradient
    start = 0
    for i in range(accumulate_gradient):
        subbatch = batch[start : start + sub_len]
        if subbatch:
            yield subbatch
        start += len(subbatch)
    subbatch = batch[start : ]
    if subbatch:
        yield subbatch </s> remove                     drop=0.5,  # dropout - make it harder to memorise data
 </s> add                     drop=0.0,  # dropout - make it harder to memorise data </s> remove         )
        for batch in util.minibatch_by_words(train_examples, size=cfg["batch_size"]):
 </s> add         ))
        random.shuffle(train_examples)
        batches = util.minibatch_by_words(train_examples, size=cfg["batch_size"])
        for batch in batches: </s> remove         scores = self.model.ops.flatten(scores)
        tag_index = {tag: i for i, tag in enumerate(self.labels)}
        cdef int idx = 0
        correct = numpy.zeros((scores.shape[0],), dtype="i")
        guesses = scores.argmax(axis=1)
        known_labels = numpy.ones((scores.shape[0], 1), dtype="f")
        for ex in examples:
            gold = ex.gold
            for tag in gold.tags:
                if tag is None:
                    correct[idx] = guesses[idx]
                elif tag in tag_index:
                    correct[idx] = tag_index[tag]
                else:
                    correct[idx] = 0
                    known_labels[idx] = 0.
                idx += 1
        correct = self.model.ops.xp.array(correct, dtype="i")
        d_scores = scores - to_categorical(correct, n_classes=scores.shape[1])
        d_scores *= self.model.ops.asarray(known_labels)
        loss = (d_scores**2).sum()
        docs = [ex.doc for ex in examples]
        d_scores = self.model.ops.unflatten(d_scores, [len(d) for d in docs])
 </s> add         loss_func = SequenceCategoricalCrossentropy(names=self.labels)
        truths = [eg.gold.tags for eg in examples]
        d_scores, loss = loss_func(scores, truths)
        if self.model.ops.xp.isnan(loss):
            raise ValueError("nan value when computing loss")
        for subbatch in subdivide_batch(batch, accumulate_gradient):
      results = []
      losses = {}
      for step, batch in enumerate(train_data):
          dropout = next(dropouts)
        for subbatch in subdivide_batch(batch, accumulate_gradient):
              nlp.update(subbatch, drop=dropout, losses=losses, sgd=False)
          for name, proc in nlp.pipeline:
              if hasattr(proc, "model"):
                  proc.model.finish_update(optimizer)
          optimizer.step_schedules()
keep keep keep keep replace keep keep keep keep keep
 Adapt parser and NER for transformers (#5449)

* Draft layer for BILUO actions

* Fixes to biluo layer

* WIP on BILUO layer

* Add tests for BILUO layer

* Format

* Fix transitions

* Update test

* Link in the simple_ner

* Update BILUO tagger

* Update __init__

* Import simple_ner

* Update test

* Import

* Add files

* Add config

* Fix label passing for BILUO and tagger

* Fix label handling for simple_ner component

* Update simple NER test

* Update config

* Hack train script

* Update BILUO layer

* Fix SimpleNER component

* Update train_from_config

* Add biluo_to_iob helper

* Add IOB layer

* Add IOBTagger model

* Update biluo layer

* Update SimpleNER tagger

* Update BILUO

* Read random seed in train-from-config

* Update use of normal_init

* Fix normalization of gradient in SimpleNER

* Update IOBTagger

* Remove print

* Tweak masking in BILUO

* Add dropout in SimpleNER

* Update thinc

* Tidy up simple_ner

* Fix biluo model

* Unhack train-from-config

* Update setup.cfg and requirements

* Add tb_framework.py for parser model

* Try to avoid memory leak in BILUO

* Move ParserModel into spacy.ml, avoid need for subclass.

* Use updated parser model

* Remove incorrect call to model.initializre in PrecomputableAffine

* Update parser model

* Avoid divide by zero in tagger

* Add extra dropout layer in tagger

* Refine minibatch_by_words function to avoid oom

* Fix parser model after refactor

* Try to avoid div-by-zero in SimpleNER

* Fix infinite loop in minibatch_by_words

* Use SequenceCategoricalCrossentropy in Tagger

* Fix parser model when hidden layer

* Remove extra dropout from tagger

* Add extra nan check in tagger

* Fix thinc version

* Update tests and imports

* Fix test

* Update test

* Update tests

* Fix tests

* Fix test

Co-authored-by: Ines Montani <ines@ines.io> 
spacy/cli/train_from_config.py