  
          for entity, prob in zip(entities, probabilities):
              entity_hash = self.vocab.strings[entity]
              if not entity_hash in self._entry_index:
                  raise ValueError(Errors.E134.format(alias=alias, entity=entity))
  
              entry_index = <int64_t>self._entry_index.get(entity_hash)
              entry_indices.push_back(int(entry_index))
              probs.push_back(float(prob))
  
 
</s> remove             aliases, entities, normalizations = get_wp_links(clean_line)
            for alias, entity, norm in zip(aliases, entities, normalizations):
                _store_alias(alias, entity, normalize_alias=norm, normalize_entity=True)
                _store_alias(alias, entity, normalize_alias=norm, normalize_entity=True)
 </s> add             # we attempt at reading the article's ID (but not the revision or contributor ID)
            if "<revision>" in clean_line or "<contributor>" in clean_line:
                read_id = False
            if "<page>" in clean_line:
                read_id = True

            if read_id:
                ids = id_regex.search(clean_line)
                if ids:
                    current_article_id = ids[0]

            # only processing prior probabilities from true training (non-dev) articles
            if not is_dev(current_article_id):
                aliases, entities, normalizations = get_wp_links(clean_line)
                for alias, entity, norm in zip(aliases, entities, normalizations):
                    _store_alias(
                        alias, entity, normalize_alias=norm, normalize_entity=True
                    ) </s> remove                 # currently, the context is the same for each entity in a sentence (should be refined)
                context_encoding = context_encodings[i]
                context_enc_t = context_encoding.T
                norm_1 = xp.linalg.norm(context_enc_t)
                for ent in doc.ents:
                    entity_count += 1

                    candidates = self.kb.get_candidates(ent.text)
                    if not candidates:
                        final_kb_ids.append(self.NIL)  # no prediction possible for this entity
                        final_tensors.append(context_encoding)
                    else:
                        random.shuffle(candidates)

                        # this will set all prior probabilities to 0 if they should be excluded from the model
                        prior_probs = xp.asarray([c.prior_prob for c in candidates])
                        if not self.cfg.get("incl_prior", True):
                            prior_probs = xp.asarray([0.0 for c in candidates])
                        scores = prior_probs

                        # add in similarity from the context
                        if self.cfg.get("incl_context", True):
                            entity_encodings = xp.asarray([c.entity_vector for c in candidates])
                            norm_2 = xp.linalg.norm(entity_encodings, axis=1)

                            if len(entity_encodings) != len(prior_probs):
                                raise RuntimeError(Errors.E147.format(method="predict", msg="vectors not of equal length"))

                             # cosine similarity
                            sims = xp.dot(entity_encodings, context_enc_t) / (norm_1 * norm_2)
                            if sims.shape != prior_probs.shape:
                                raise ValueError(Errors.E161)
                            scores = prior_probs + sims - (prior_probs*sims)

                        # TODO: thresholding
                        best_index = scores.argmax()
                        best_candidate = candidates[best_index]
                        final_kb_ids.append(best_candidate.entity_)
                        final_tensors.append(context_encoding)
 </s> add                 # Looping through each sentence and each entity
                # This may go wrong if there are entities across sentences - because they might not get a KB ID
                for sent in doc.ents:
                    sent_doc = sent.as_doc()
                    # currently, the context is the same for each entity in a sentence (should be refined)
                    sentence_encoding = self.model([sent_doc])[0]
                    xp = get_array_module(sentence_encoding)
                    sentence_encoding_t = sentence_encoding.T
                    sentence_norm = xp.linalg.norm(sentence_encoding_t)

                    for ent in sent_doc.ents:
                        entity_count += 1

                        if ent.label_ in self.cfg.get("labels_discard", []):
                            # ignoring this entity - setting to NIL
                            final_kb_ids.append(self.NIL)
                            final_tensors.append(sentence_encoding)

                        else:
                            candidates = self.kb.get_candidates(ent.text)
                            if not candidates:
                                # no prediction possible for this entity - setting to NIL
                                final_kb_ids.append(self.NIL)
                                final_tensors.append(sentence_encoding)

                            elif len(candidates) == 1:
                                # shortcut for efficiency reasons: take the 1 candidate

                                # TODO: thresholding
                                final_kb_ids.append(candidates[0].entity_)
                                final_tensors.append(sentence_encoding)

                            else:
                                random.shuffle(candidates)

                                # this will set all prior probabilities to 0 if they should be excluded from the model
                                prior_probs = xp.asarray([c.prior_prob for c in candidates])
                                if not self.cfg.get("incl_prior", True):
                                    prior_probs = xp.asarray([0.0 for c in candidates])
                                scores = prior_probs

                                # add in similarity from the context
                                if self.cfg.get("incl_context", True):
                                    entity_encodings = xp.asarray([c.entity_vector for c in candidates])
                                    entity_norm = xp.linalg.norm(entity_encodings, axis=1)

                                    if len(entity_encodings) != len(prior_probs):
                                        raise RuntimeError(Errors.E147.format(method="predict", msg="vectors not of equal length"))

                                    # cosine similarity
                                    sims = xp.dot(entity_encodings, sentence_encoding_t) / (sentence_norm * entity_norm)
                                    if sims.shape != prior_probs.shape:
                                        raise ValueError(Errors.E161)
                                    scores = prior_probs + sims - (prior_probs*sims)

                                # TODO: thresholding
                                best_index = scores.argmax()
                                best_candidate = candidates[best_index]
                                final_kb_ids.append(best_candidate.entity_)
                                final_tensors.append(sentence_encoding) </s> remove                 # only evaluating on positive examples
 </s> add 
                raise ValueError(Errors.E134.format(entity=entity))
  
          for entity, prob in zip(entities, probabilities):
              entity_hash = self.vocab.strings[entity]
              if not entity_hash in self._entry_index:
                raise ValueError(Errors.E134.format(entity=entity))
  
              entry_index = <int64_t>self._entry_index.get(entity_hash)
              entry_indices.push_back(int(entry_index))
              probs.push_back(float(prob))
  
keep keep keep keep replace keep keep keep keep keep
 KB extensions and better parsing of WikiData (#4375)

* fix overflow error on windows

* more documentation & logging fixes

* md fix

* 3 different limit parameters to play with execution time

* bug fixes directory locations

* small fixes

* exclude dev test articles from prior probabilities stats

* small fixes

* filtering wikidata entities, removing numeric and meta items

* adding aliases from wikidata also to the KB

* fix adding WD aliases

* adding also new aliases to previously added entities

* fixing comma's

* small doc fixes

* adding subclassof filtering

* append alias functionality in KB

* prevent appending the same entity-alias pair

* fix for appending WD aliases

* remove date filter

* remove unnecessary import

* small corrections and reformatting

* remove WD aliases for now (too slow)

* removing numeric entities from training and evaluation

* small fixes

* shortcut during prediction if there is only one candidate

* add counts and fscore logging, remove FP NER from evaluation

* fix entity_linker.predict to take docs instead of single sentences

* remove enumeration sentences from the WP dataset

* entity_linker.update to process full doc instead of single sentence

* spelling corrections and dump locations in readme

* NLP IO fix

* reading KB is unnecessary at the end of the pipeline

* small logging fix

* remove empty files 
spacy/kb.pyx