          processed, loss = self._train_model(description_list)
          if to_print:
              logger.info(
                  "Trained entity descriptions on {} ".format(processed) +
                  "(non-unique) entities across {} ".format(self.epochs) +
                  "epochs"
              )
              logger.info("Final loss: {}".format(loss))
  
      def _train_model(self, description_list):
 
</s> remove             logger.info("encoded: {} entities".format(stop))
 </s> add             logger.info("Encoded: {} entities".format(stop)) </s> remove                         if found_link and parse_descriptions:
 </s> add                         if found_link and parse_descr: </s> remove         return ("{}: ".format(model_str) +
                "Recall = {} | ".format(round(recall, 3)) +
                "Precision = {} | ".format(round(precision, 3)) +
                "Precision by label = {}".format({k: v.calculate_precision()
                                                  for k, v in self.metrics_by_label.items()}))
 </s> add         fscore = self.metrics.calculate_fscore()
        return (
            "{}: ".format(model_str)
            + "F-score = {} | ".format(round(fscore, 3))
            + "Recall = {} | ".format(round(recall, 3))
            + "Precision = {} | ".format(round(precision, 3))
            + "F-score by label = {}".format(
                {k: v.calculate_fscore() for k, v in sorted(self.metrics_by_label.items())}
            )
        ) </s> remove         wd.write_entity_files(entity_defs_path, title_to_id)
        if not descriptions_from_wikipedia:
            wd.write_entity_description_files(entity_descr_path, id_to_descr)
    logger.info("STEP 4: read entity definitions" + message)

    # STEP 5: Getting gold entities from wikipedia
    message = " and descriptions" if descriptions_from_wikipedia else ""
    if (not training_entities_path.exists()) or (descriptions_from_wikipedia and not entity_descr_path.exists()):
        logger.info("STEP 5: parsing wikipedia for gold entities" + message)
        training_set_creator.create_training_examples_and_descriptions(
            wp_xml,
            entity_defs_path,
            entity_descr_path,
            training_entities_path,
            parse_descriptions=descriptions_from_wikipedia,
            limit=limit,
        )
    logger.info("STEP 5: read gold entities" + message)
 </s> add         io.write_title_to_id(entity_defs_path, title_to_id)

        logger.info("STEP 4b: Writing Wikidata entity aliases to {}".format(entity_alias_path))
        io.write_id_to_alias(entity_alias_path, id_to_alias)

        if not descr_from_wp:
            logger.info("STEP 4c: Writing Wikidata entity descriptions to {}".format(entity_descr_path))
            io.write_id_to_descr(entity_descr_path, id_to_descr)
    else:
        logger.info("STEP 4: Reading entity definitions from {}".format(entity_defs_path))
        logger.info("STEP 4b: Reading entity aliases from {}".format(entity_alias_path))
        if not descr_from_wp:
            logger.info("STEP 4c: Reading entity descriptions from {}".format(entity_descr_path))

    # STEP 5: Getting gold entities from Wikipedia
    if (not training_entities_path.exists()) or (descr_from_wp and not entity_descr_path.exists()):
        logger.info("STEP 5: Parsing and writing Wikipedia gold entities to {}".format(training_entities_path))
        if limit_train is not None:
            logger.warning("Warning: reading only {} lines of Wikipedia dump".format(limit_train))
        wp.create_training_and_desc(wp_xml, entity_defs_path, entity_descr_path,
                                    training_entities_path, descr_from_wp, limit_train)
        if descr_from_wp:
            logger.info("STEP 5b: Parsing and writing Wikipedia descriptions to {}".format(entity_descr_path))
    else:
        logger.info("STEP 5: Reading gold entities from {}".format(training_entities_path))
        if descr_from_wp:
            logger.info("STEP 5b: Reading entity descriptions from {}".format(entity_descr_path)) </s> remove     return title_to_id, id_to_descr


def write_entity_files(entity_def_output, title_to_id):
    with entity_def_output.open("w", encoding="utf8") as id_file:
        id_file.write("WP_title" + "|" + "WD_id" + "\n")
        for title, qid in title_to_id.items():
            id_file.write(title + "|" + str(qid) + "\n")
 </s> add     # log final number of lines processed
    logger.info("Finished. Processed {} lines of WikiData JSON dump".format(cnt))
    return title_to_id, id_to_descr, id_to_alias
                "(non-unique) descriptions across {} ".format(self.epochs) +
          processed, loss = self._train_model(description_list)
          if to_print:
              logger.info(
                  "Trained entity descriptions on {} ".format(processed) +
                "(non-unique) descriptions across {} ".format(self.epochs) +
                  "epochs"
              )
              logger.info("Final loss: {}".format(loss))
  
      def _train_model(self, description_list):
keep keep keep keep replace keep keep keep keep keep
 KB extensions and better parsing of WikiData (#4375)

* fix overflow error on windows

* more documentation & logging fixes

* md fix

* 3 different limit parameters to play with execution time

* bug fixes directory locations

* small fixes

* exclude dev test articles from prior probabilities stats

* small fixes

* filtering wikidata entities, removing numeric and meta items

* adding aliases from wikidata also to the KB

* fix adding WD aliases

* adding also new aliases to previously added entities

* fixing comma's

* small doc fixes

* adding subclassof filtering

* append alias functionality in KB

* prevent appending the same entity-alias pair

* fix for appending WD aliases

* remove date filter

* remove unnecessary import

* small corrections and reformatting

* remove WD aliases for now (too slow)

* removing numeric entities from training and evaluation

* small fixes

* shortcut during prediction if there is only one candidate

* add counts and fscore logging, remove FP NER from evaluation

* fix entity_linker.predict to take docs instead of single sentences

* remove enumeration sentences from the WP dataset

* entity_linker.update to process full doc instead of single sentence

* spelling corrections and dump locations in readme

* NLP IO fix

* reading KB is unnecessary at the end of the pipeline

* small logging fix

* remove empty files 
bin/wiki_entity_linking/train_descriptions.py