      # STEP 4: reading definitions and (possibly) descriptions from WikiData or from file
      message = " and descriptions" if not descriptions_from_wikipedia else ""
      if (not entity_defs_path.exists()) or (not descriptions_from_wikipedia and not entity_descr_path.exists()):
          # It takes about 10h to process 55M lines of Wikidata JSON dump
          logger.info("STEP 4: parsing wikidata for entity definitions" + message)
          title_to_id, id_to_descr = wd.read_wikidata_entities_json(
              wd_json,
              limit,
              to_print=False,
              lang=lang,
              parse_descriptions=(not descriptions_from_wikipedia),
 
</s> remove     message = " and descriptions" if not descriptions_from_wikipedia else ""
    if (not entity_defs_path.exists()) or (not descriptions_from_wikipedia and not entity_descr_path.exists()):
 </s> add     if (not entity_defs_path.exists()) or (not descr_from_wp and not entity_descr_path.exists()): </s> remove             limit,
 </s> add             limit_wd, </s> remove         logger.info("STEP 2: writing prior probabilities to {}".format(prior_prob_path))
        wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit)
    logger.info("STEP 2: reading prior probabilities from {}".format(prior_prob_path))

    # STEP 3: deduce entity frequencies from WP (takes only a few minutes)
    logger.info("STEP 3: calculating entity frequencies")
    wp.write_entity_counts(prior_prob_path, entity_freq_path, to_print=False)
 </s> add         logger.info("STEP 2: Writing prior probabilities to {}".format(prior_prob_path))
        if limit_prior is not None:
            logger.warning("Warning: reading only {} lines of Wikipedia dump".format(limit_prior))
        wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit_prior)
    else:
        logger.info("STEP 2: Reading prior probabilities from {}".format(prior_prob_path))

    # STEP 3: calculate entity frequencies
    if not entity_freq_path.exists():
        logger.info("STEP 3: Calculating and writing entity frequencies to {}".format(entity_freq_path))
        io.write_entity_to_count(prior_prob_path, entity_freq_path)
    else:
        logger.info("STEP 3: Reading entity frequencies from {}".format(entity_freq_path)) </s> remove         wd.write_entity_files(entity_defs_path, title_to_id)
        if not descriptions_from_wikipedia:
            wd.write_entity_description_files(entity_descr_path, id_to_descr)
    logger.info("STEP 4: read entity definitions" + message)

    # STEP 5: Getting gold entities from wikipedia
    message = " and descriptions" if descriptions_from_wikipedia else ""
    if (not training_entities_path.exists()) or (descriptions_from_wikipedia and not entity_descr_path.exists()):
        logger.info("STEP 5: parsing wikipedia for gold entities" + message)
        training_set_creator.create_training_examples_and_descriptions(
            wp_xml,
            entity_defs_path,
            entity_descr_path,
            training_entities_path,
            parse_descriptions=descriptions_from_wikipedia,
            limit=limit,
        )
    logger.info("STEP 5: read gold entities" + message)
 </s> add         io.write_title_to_id(entity_defs_path, title_to_id)

        logger.info("STEP 4b: Writing Wikidata entity aliases to {}".format(entity_alias_path))
        io.write_id_to_alias(entity_alias_path, id_to_alias)

        if not descr_from_wp:
            logger.info("STEP 4c: Writing Wikidata entity descriptions to {}".format(entity_descr_path))
            io.write_id_to_descr(entity_descr_path, id_to_descr)
    else:
        logger.info("STEP 4: Reading entity definitions from {}".format(entity_defs_path))
        logger.info("STEP 4b: Reading entity aliases from {}".format(entity_alias_path))
        if not descr_from_wp:
            logger.info("STEP 4c: Reading entity descriptions from {}".format(entity_descr_path))

    # STEP 5: Getting gold entities from Wikipedia
    if (not training_entities_path.exists()) or (descr_from_wp and not entity_descr_path.exists()):
        logger.info("STEP 5: Parsing and writing Wikipedia gold entities to {}".format(training_entities_path))
        if limit_train is not None:
            logger.warning("Warning: reading only {} lines of Wikipedia dump".format(limit_train))
        wp.create_training_and_desc(wp_xml, entity_defs_path, entity_descr_path,
                                    training_entities_path, descr_from_wp, limit_train)
        if descr_from_wp:
            logger.info("STEP 5b: Parsing and writing Wikipedia descriptions to {}".format(entity_descr_path))
    else:
        logger.info("STEP 5: Reading gold entities from {}".format(training_entities_path))
        if descr_from_wp:
            logger.info("STEP 5b: Reading entity descriptions from {}".format(entity_descr_path)) </s> remove             parse_descriptions=(not descriptions_from_wikipedia),
 </s> add             parse_descr=(not descr_from_wp),
        logger.info("STEP 4: Parsing and writing Wikidata entity definitions to {}".format(entity_defs_path))
        if limit_wd is not None:
            logger.warning("Warning: reading only {} lines of Wikidata dump".format(limit_wd))
        title_to_id, id_to_descr, id_to_alias = wd.read_wikidata_entities_json(
      # STEP 4: reading definitions and (possibly) descriptions from WikiData or from file
      message = " and descriptions" if not descriptions_from_wikipedia else ""
      if (not entity_defs_path.exists()) or (not descriptions_from_wikipedia and not entity_descr_path.exists()):
          # It takes about 10h to process 55M lines of Wikidata JSON dump
        logger.info("STEP 4: Parsing and writing Wikidata entity definitions to {}".format(entity_defs_path))
        if limit_wd is not None:
            logger.warning("Warning: reading only {} lines of Wikidata dump".format(limit_wd))
        title_to_id, id_to_descr, id_to_alias = wd.read_wikidata_entities_json(
        logger.info("STEP 4: Parsing and writing Wikidata entity definitions to {}".format(entity_defs_path))
        if limit_wd is not None:
            logger.warning("Warning: reading only {} lines of Wikidata dump".format(limit_wd))
        title_to_id, id_to_descr, id_to_alias = wd.read_wikidata_entities_json(
              wd_json,
              limit,
              to_print=False,
              lang=lang,
              parse_descriptions=(not descriptions_from_wikipedia),
keep keep keep keep replace replace keep keep keep keep keep
 KB extensions and better parsing of WikiData (#4375)

* fix overflow error on windows

* more documentation & logging fixes

* md fix

* 3 different limit parameters to play with execution time

* bug fixes directory locations

* small fixes

* exclude dev test articles from prior probabilities stats

* small fixes

* filtering wikidata entities, removing numeric and meta items

* adding aliases from wikidata also to the KB

* fix adding WD aliases

* adding also new aliases to previously added entities

* fixing comma's

* small doc fixes

* adding subclassof filtering

* append alias functionality in KB

* prevent appending the same entity-alias pair

* fix for appending WD aliases

* remove date filter

* remove unnecessary import

* small corrections and reformatting

* remove WD aliases for now (too slow)

* removing numeric entities from training and evaluation

* small fixes

* shortcut during prediction if there is only one candidate

* add counts and fscore logging, remove FP NER from evaluation

* fix entity_linker.predict to take docs instead of single sentences

* remove enumeration sentences from the WP dataset

* entity_linker.update to process full doc instead of single sentence

* spelling corrections and dump locations in readme

* NLP IO fix

* reading KB is unnecessary at the end of the pipeline

* small logging fix

* remove empty files 
bin/wiki_entity_linking/wikidata_pretrain_kb.py