                      gold_ent = ents_by_offset["{}_{}".format(start, end)]
                      if gold_ent is None:
                          raise RuntimeError(Errors.E147.format(method="update", msg="gold entity not found"))
  
                      type_vector = [0 for i in range(len(type_to_int))]
                      if len(type_to_int) > 0:
                          type_vector[type_to_int[gold_ent.label_]] = 1
  
                      # store data
                      entity_encodings.append(entity_encoding)
                      context_docs.append(doc)
                      type_vectors.append(type_vector)
  
                      if self.cfg.get("prior_weight", 1) > 0:
                          priors.append([prior_prob])
                      else:
                          priors.append([0])
  
          if len(entity_encodings) > 0:
              if not (len(priors) == len(entity_encodings) == len(context_docs) == len(type_vectors)):
                  raise RuntimeError(Errors.E147.format(method="update", msg="vector lengths not equal"))
  
 
</s> remove                 for kb_id, value in kb_dict.items():
                    entity_encoding = self.kb.get_vector(kb_id)
                    prior_prob = self.kb.get_prior_prob(kb_id, mention)

                    gold_ent = ents_by_offset["{}_{}".format(start, end)]
                    if gold_ent is None:
                        raise RuntimeError(Errors.E147.format(method="update", msg="gold entity not found"))
 </s> add  </s> remove         if len(entity_encodings) > 0:
            if not (len(priors) == len(entity_encodings) == len(context_docs) == len(type_vectors)):
                raise RuntimeError(Errors.E147.format(method="update", msg="vector lengths not equal"))
 </s> add         context_encodings, bp_context = self.model.begin_update(context_docs, drop=drop)
        loss, d_scores = self.get_similarity_loss(scores=context_encodings, golds=golds, docs=None)
        bp_context(d_scores, sgd=sgd) </s> remove             entity_encodings = self.model.ops.asarray(entity_encodings, dtype="float32")
 </s> add         if losses is not None:
            losses[self.name] += loss
        return loss </s> remove                     type_vector = [0 for i in range(len(type_to_int))]
                    if len(type_to_int) > 0:
                        type_vector[type_to_int[ent.label_]] = 1
 </s> add  </s> remove                         if self.cfg.get("context_weight", 1) > 0:
 </s> add                         # add in similarity from the context
                        if self.cfg.get("incl_context", True):
                for kb_id, value in kb_dict.items():
                    # Currently only training on the positive instances
                    if value:
                        context_docs.append(doc)
                      gold_ent = ents_by_offset["{}_{}".format(start, end)]
                      if gold_ent is None:
                          raise RuntimeError(Errors.E147.format(method="update", msg="gold entity not found"))
  
                for kb_id, value in kb_dict.items():
                    # Currently only training on the positive instances
                    if value:
                        context_docs.append(doc)
                for kb_id, value in kb_dict.items():
                    # Currently only training on the positive instances
                    if value:
                        context_docs.append(doc)
                for kb_id, value in kb_dict.items():
                    # Currently only training on the positive instances
                    if value:
                        context_docs.append(doc)
                for kb_id, value in kb_dict.items():
                    # Currently only training on the positive instances
                    if value:
                        context_docs.append(doc)
                for kb_id, value in kb_dict.items():
                    # Currently only training on the positive instances
                    if value:
                        context_docs.append(doc)
                for kb_id, value in kb_dict.items():
                    # Currently only training on the positive instances
                    if value:
                        context_docs.append(doc)
                for kb_id, value in kb_dict.items():
                    # Currently only training on the positive instances
                    if value:
                        context_docs.append(doc)
                for kb_id, value in kb_dict.items():
                    # Currently only training on the positive instances
                    if value:
                        context_docs.append(doc)
                for kb_id, value in kb_dict.items():
                    # Currently only training on the positive instances
                    if value:
                        context_docs.append(doc)
                for kb_id, value in kb_dict.items():
                    # Currently only training on the positive instances
                    if value:
                        context_docs.append(doc)
                for kb_id, value in kb_dict.items():
                    # Currently only training on the positive instances
                    if value:
                        context_docs.append(doc)
                for kb_id, value in kb_dict.items():
                    # Currently only training on the positive instances
                    if value:
                        context_docs.append(doc)
                for kb_id, value in kb_dict.items():
                    # Currently only training on the positive instances
                    if value:
                        context_docs.append(doc)
  
          if len(entity_encodings) > 0:
              if not (len(priors) == len(entity_encodings) == len(context_docs) == len(type_vectors)):
                  raise RuntimeError(Errors.E147.format(method="update", msg="vector lengths not equal"))
  
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace keep keep keep keep keep
 CLI scripts for entity linking (wikipedia & generic) (#4091)

* document token ent_kb_id

* document span kb_id

* update pipeline documentation

* prior and context weights as bool's instead

* entitylinker api documentation

* drop for both models

* finish entitylinker documentation

* small fixes

* documentation for KB

* candidate documentation

* links to api pages in code

* small fix

* frequency examples as counts for consistency

* consistent documentation about tensors returned by predict

* add entity linking to usage 101

* add entity linking infobox and KB section to 101

* entity-linking in linguistic features

* small typo corrections

* training example and docs for entity_linker

* predefined nlp and kb

* revert back to similarity encodings for simplicity (for now)

* set prior probabilities to 0 when excluded

* code clean up

* bugfix: deleting kb ID from tokens when entities were removed

* refactor train el example to use either model or vocab

* pretrain_kb example for example kb generation

* add to training docs for KB + EL example scripts

* small fixes

* error numbering

* ensure the language of vocab and nlp stay consistent across serialization

* equality with =

* avoid conflict in errors file

* add error 151

* final adjustements to the train scripts - consistency

* update of goldparse documentation

* small corrections

* push commit

* turn kb_creator into CLI script (wip)

* proper parameters for training entity vectors

* wikidata pipeline split up into two executable scripts

* remove context_width

* move wikidata scripts in bin directory, remove old dummy script

* refine KB script with logs and preprocessing options

* small edits

* small improvements to logging of EL CLI script 
spacy/pipeline/pipes.pyx