  
          if isinstance(docs, Doc):
              docs = [docs]
  
          context_encodings = self.model(docs)
          xp = get_array_module(context_encodings)
  
          for i, doc in enumerate(docs):
              if len(doc) > 0:
                  # currently, the context is the same for each entity in a sentence (should be refined)
                  context_encoding = context_encodings[i]
                  context_enc_t = context_encoding.T
 
</s> remove                 # currently, the context is the same for each entity in a sentence (should be refined)
                context_encoding = context_encodings[i]
                context_enc_t = context_encoding.T
                norm_1 = xp.linalg.norm(context_enc_t)
                for ent in doc.ents:
                    entity_count += 1

                    candidates = self.kb.get_candidates(ent.text)
                    if not candidates:
                        final_kb_ids.append(self.NIL)  # no prediction possible for this entity
                        final_tensors.append(context_encoding)
                    else:
                        random.shuffle(candidates)

                        # this will set all prior probabilities to 0 if they should be excluded from the model
                        prior_probs = xp.asarray([c.prior_prob for c in candidates])
                        if not self.cfg.get("incl_prior", True):
                            prior_probs = xp.asarray([0.0 for c in candidates])
                        scores = prior_probs

                        # add in similarity from the context
                        if self.cfg.get("incl_context", True):
                            entity_encodings = xp.asarray([c.entity_vector for c in candidates])
                            norm_2 = xp.linalg.norm(entity_encodings, axis=1)

                            if len(entity_encodings) != len(prior_probs):
                                raise RuntimeError(Errors.E147.format(method="predict", msg="vectors not of equal length"))

                             # cosine similarity
                            sims = xp.dot(entity_encodings, context_enc_t) / (norm_1 * norm_2)
                            if sims.shape != prior_probs.shape:
                                raise ValueError(Errors.E161)
                            scores = prior_probs + sims - (prior_probs*sims)

                        # TODO: thresholding
                        best_index = scores.argmax()
                        best_candidate = candidates[best_index]
                        final_kb_ids.append(best_candidate.entity_)
                        final_tensors.append(context_encoding)
 </s> add                 # Looping through each sentence and each entity
                # This may go wrong if there are entities across sentences - because they might not get a KB ID
                for sent in doc.ents:
                    sent_doc = sent.as_doc()
                    # currently, the context is the same for each entity in a sentence (should be refined)
                    sentence_encoding = self.model([sent_doc])[0]
                    xp = get_array_module(sentence_encoding)
                    sentence_encoding_t = sentence_encoding.T
                    sentence_norm = xp.linalg.norm(sentence_encoding_t)

                    for ent in sent_doc.ents:
                        entity_count += 1

                        if ent.label_ in self.cfg.get("labels_discard", []):
                            # ignoring this entity - setting to NIL
                            final_kb_ids.append(self.NIL)
                            final_tensors.append(sentence_encoding)

                        else:
                            candidates = self.kb.get_candidates(ent.text)
                            if not candidates:
                                # no prediction possible for this entity - setting to NIL
                                final_kb_ids.append(self.NIL)
                                final_tensors.append(sentence_encoding)

                            elif len(candidates) == 1:
                                # shortcut for efficiency reasons: take the 1 candidate

                                # TODO: thresholding
                                final_kb_ids.append(candidates[0].entity_)
                                final_tensors.append(sentence_encoding)

                            else:
                                random.shuffle(candidates)

                                # this will set all prior probabilities to 0 if they should be excluded from the model
                                prior_probs = xp.asarray([c.prior_prob for c in candidates])
                                if not self.cfg.get("incl_prior", True):
                                    prior_probs = xp.asarray([0.0 for c in candidates])
                                scores = prior_probs

                                # add in similarity from the context
                                if self.cfg.get("incl_context", True):
                                    entity_encodings = xp.asarray([c.entity_vector for c in candidates])
                                    entity_norm = xp.linalg.norm(entity_encodings, axis=1)

                                    if len(entity_encodings) != len(prior_probs):
                                        raise RuntimeError(Errors.E147.format(method="predict", msg="vectors not of equal length"))

                                    # cosine similarity
                                    sims = xp.dot(entity_encodings, sentence_encoding_t) / (sentence_norm * entity_norm)
                                    if sims.shape != prior_probs.shape:
                                        raise ValueError(Errors.E161)
                                    scores = prior_probs + sims - (prior_probs*sims)

                                # TODO: thresholding
                                best_index = scores.argmax()
                                best_candidate = candidates[best_index]
                                final_kb_ids.append(best_candidate.entity_)
                                final_tensors.append(sentence_encoding) </s> remove         context_docs = []
 </s> add         sentence_docs = [] </s> remove     # using only context
    el_pipe.cfg["incl_context"] = True
    el_pipe.cfg["incl_prior"] = False
    results = get_eval_results(dev_data, el_pipe)
    logger.info(results.report_metrics("context only"))

    # measuring combined accuracy (prior + context)
    el_pipe.cfg["incl_context"] = True
    el_pipe.cfg["incl_prior"] = True
    results = get_eval_results(dev_data, el_pipe)
    logger.info(results.report_metrics("context and prior"))
 </s> add         # measuring combined accuracy (prior + context)
        el_pipe.cfg["incl_context"] = True
        el_pipe.cfg["incl_prior"] = True
        results = get_eval_results(dev_data, el_pipe)
        logger.info(results.report_metrics("context and prior")) </s> remove             current_count = entity_to_count.get(entity, 0)
            entity_to_count[entity] = current_count + count
 </s> add     read_ids = set()

    with output.open("a", encoding="utf8") as descr_file, training_output.open(
        "w", encoding="utf8"
    ) as entity_file:
        if parse_descriptions:
            _write_training_description(descr_file, "WD_id", "description")
        with bz2.open(wikipedia_input, mode="rb") as file:
            article_count = 0
            article_text = ""
            article_title = None
            article_id = None
            reading_text = False
            reading_revision = False

            for line in file:
                clean_line = line.strip().decode("utf-8")

                if clean_line == "<revision>":
                    reading_revision = True
                elif clean_line == "</revision>":
                    reading_revision = False

                # Start reading new page
                if clean_line == "<page>":
                    article_text = ""
                    article_title = None
                    article_id = None
                # finished reading this page
                elif clean_line == "</page>":
                    if article_id:
                        clean_text, entities = _process_wp_text(
                            article_title, article_text, wp_to_id
                        )
                        if clean_text is not None and entities is not None:
                            _write_training_entities(
                                entity_file, article_id, clean_text, entities
                            )

                            if article_title in wp_to_id and parse_descriptions:
                                description = " ".join(
                                    clean_text[:1000].split(" ")[:-1]
                                )
                                _write_training_description(
                                    descr_file, wp_to_id[article_title], description
                                )
                            article_count += 1
                            if article_count % 10000 == 0 and article_count > 0:
                                logger.info(
                                    "Processed {} articles".format(article_count)
                                )
                            if limit and article_count >= limit:
                                break
                    article_text = ""
                    article_title = None
                    article_id = None
                    reading_text = False
                    reading_revision = False

                # start reading text within a page
                if "<text" in clean_line:
                    reading_text = True

                if reading_text:
                    article_text += " " + clean_line

                # stop reading text within a page (we assume a new page doesn't start on the same line)
                if "</text" in clean_line:
                    reading_text = False

                # read the ID of this article (outside the revision portion of the document)
                if not reading_revision:
                    ids = id_regex.search(clean_line)
                    if ids:
                        article_id = ids[0]
                        if article_id in read_ids:
                            logger.info(
                                "Found duplicate article ID", article_id, clean_line
                            )  # This should never happen ...
                        read_ids.add(article_id)

                # read the title of this article (outside the revision portion of the document)
                if not reading_revision:
                    titles = title_regex.search(clean_line)
                    if titles:
                        article_title = titles[0].strip()
    logger.info("Finished. Processed {} articles".format(article_count))


def _process_wp_text(article_title, article_text, wp_to_id):
    # ignore meta Wikipedia pages
    if ns_regex.match(article_title):
        return None, None

    # remove the text tags
    text_search = text_regex.search(article_text)
    if text_search is None:
        return None, None
    text = text_search.group(0)

    # stop processing if this is a redirect page
    if text.startswith("#REDIRECT"):
        return None, None

    # get the raw text without markup etc, keeping only interwiki links
    clean_text, entities = _remove_links(_get_clean_wp_text(text), wp_to_id)
    return clean_text, entities


def _get_clean_wp_text(article_text):
    clean_text = article_text.strip()

    # remove bolding & italic markup
    clean_text = clean_text.replace("'''", "")
    clean_text = clean_text.replace("''", "")

    # remove nested {{info}} statements by removing the inner/smallest ones first and iterating
    try_again = True
    previous_length = len(clean_text)
    while try_again:
        clean_text = info_regex.sub(
            "", clean_text
        )  # non-greedy match excluding a nested {
        if len(clean_text) < previous_length:
            try_again = True
        else:
            try_again = False
        previous_length = len(clean_text)

    # remove HTML comments
    clean_text = html_regex.sub("", clean_text)

    # remove Category and File statements
    clean_text = category_regex.sub("", clean_text)
    clean_text = file_regex.sub("", clean_text)

    # remove multiple =
    while "==" in clean_text:
        clean_text = clean_text.replace("==", "=")

    clean_text = clean_text.replace(". =", ".")
    clean_text = clean_text.replace(" = ", ". ")
    clean_text = clean_text.replace("= ", ".")
    clean_text = clean_text.replace(" =", "") </s> remove             pass  # ignore namespaces at the beginning of the string
 </s> add             pass  # ignore the entity if it points to a "meta" page
  
          if isinstance(docs, Doc):
              docs = [docs]
  



          for i, doc in enumerate(docs):
              if len(doc) > 0:
                  # currently, the context is the same for each entity in a sentence (should be refined)
                  context_encoding = context_encodings[i]
                  context_enc_t = context_encoding.T
keep keep keep keep replace replace replace keep keep keep keep keep
 KB extensions and better parsing of WikiData (#4375)

* fix overflow error on windows

* more documentation & logging fixes

* md fix

* 3 different limit parameters to play with execution time

* bug fixes directory locations

* small fixes

* exclude dev test articles from prior probabilities stats

* small fixes

* filtering wikidata entities, removing numeric and meta items

* adding aliases from wikidata also to the KB

* fix adding WD aliases

* adding also new aliases to previously added entities

* fixing comma's

* small doc fixes

* adding subclassof filtering

* append alias functionality in KB

* prevent appending the same entity-alias pair

* fix for appending WD aliases

* remove date filter

* remove unnecessary import

* small corrections and reformatting

* remove WD aliases for now (too slow)

* removing numeric entities from training and evaluation

* small fixes

* shortcut during prediction if there is only one candidate

* add counts and fscore logging, remove FP NER from evaluation

* fix entity_linker.predict to take docs instead of single sentences

* remove enumeration sentences from the WP dataset

* entity_linker.update to process full doc instead of single sentence

* spelling corrections and dump locations in readme

* NLP IO fix

* reading KB is unnecessary at the end of the pipeline

* small logging fix

* remove empty files 
spacy/pipeline/pipes.pyx