  
      # STEP 2: create prior probabilities from WP
      if not prior_prob_path.exists():
          # It takes about 2h to process 1000M lines of Wikipedia XML dump
          logger.info("STEP 2: writing prior probabilities to {}".format(prior_prob_path))
          wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit)
      logger.info("STEP 2: reading prior probabilities from {}".format(prior_prob_path))
  
      # STEP 3: deduce entity frequencies from WP (takes only a few minutes)
      logger.info("STEP 3: calculating entity frequencies")
      wp.write_entity_counts(prior_prob_path, entity_freq_path, to_print=False)
  
      # STEP 4: reading definitions and (possibly) descriptions from WikiData or from file
      message = " and descriptions" if not descriptions_from_wikipedia else ""
      if (not entity_defs_path.exists()) or (not descriptions_from_wikipedia and not entity_descr_path.exists()):
          # It takes about 10h to process 55M lines of Wikidata JSON dump
 
</s> remove     message = " and descriptions" if not descriptions_from_wikipedia else ""
    if (not entity_defs_path.exists()) or (not descriptions_from_wikipedia and not entity_descr_path.exists()):
 </s> add     if (not entity_defs_path.exists()) or (not descr_from_wp and not entity_descr_path.exists()): </s> remove         logger.info("STEP 4: parsing wikidata for entity definitions" + message)
        title_to_id, id_to_descr = wd.read_wikidata_entities_json(
 </s> add         logger.info("STEP 4: Parsing and writing Wikidata entity definitions to {}".format(entity_defs_path))
        if limit_wd is not None:
            logger.warning("Warning: reading only {} lines of Wikidata dump".format(limit_wd))
        title_to_id, id_to_descr, id_to_alias = wd.read_wikidata_entities_json( </s> remove         wd.write_entity_files(entity_defs_path, title_to_id)
        if not descriptions_from_wikipedia:
            wd.write_entity_description_files(entity_descr_path, id_to_descr)
    logger.info("STEP 4: read entity definitions" + message)

    # STEP 5: Getting gold entities from wikipedia
    message = " and descriptions" if descriptions_from_wikipedia else ""
    if (not training_entities_path.exists()) or (descriptions_from_wikipedia and not entity_descr_path.exists()):
        logger.info("STEP 5: parsing wikipedia for gold entities" + message)
        training_set_creator.create_training_examples_and_descriptions(
            wp_xml,
            entity_defs_path,
            entity_descr_path,
            training_entities_path,
            parse_descriptions=descriptions_from_wikipedia,
            limit=limit,
        )
    logger.info("STEP 5: read gold entities" + message)
 </s> add         io.write_title_to_id(entity_defs_path, title_to_id)

        logger.info("STEP 4b: Writing Wikidata entity aliases to {}".format(entity_alias_path))
        io.write_id_to_alias(entity_alias_path, id_to_alias)

        if not descr_from_wp:
            logger.info("STEP 4c: Writing Wikidata entity descriptions to {}".format(entity_descr_path))
            io.write_id_to_descr(entity_descr_path, id_to_descr)
    else:
        logger.info("STEP 4: Reading entity definitions from {}".format(entity_defs_path))
        logger.info("STEP 4b: Reading entity aliases from {}".format(entity_alias_path))
        if not descr_from_wp:
            logger.info("STEP 4c: Reading entity descriptions from {}".format(entity_descr_path))

    # STEP 5: Getting gold entities from Wikipedia
    if (not training_entities_path.exists()) or (descr_from_wp and not entity_descr_path.exists()):
        logger.info("STEP 5: Parsing and writing Wikipedia gold entities to {}".format(training_entities_path))
        if limit_train is not None:
            logger.warning("Warning: reading only {} lines of Wikipedia dump".format(limit_train))
        wp.create_training_and_desc(wp_xml, entity_defs_path, entity_descr_path,
                                    training_entities_path, descr_from_wp, limit_train)
        if descr_from_wp:
            logger.info("STEP 5b: Parsing and writing Wikipedia descriptions to {}".format(entity_descr_path))
    else:
        logger.info("STEP 5: Reading gold entities from {}".format(training_entities_path))
        if descr_from_wp:
            logger.info("STEP 5b: Reading entity descriptions from {}".format(entity_descr_path)) </s> remove             limit,
 </s> add             limit_wd, </s> remove     train_data = training_set_creator.read_training(
 </s> add     train_data = wikipedia_processor.read_training(
        logger.info("STEP 2: Writing prior probabilities to {}".format(prior_prob_path))
        if limit_prior is not None:
            logger.warning("Warning: reading only {} lines of Wikipedia dump".format(limit_prior))
        wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit_prior)
    else:
        logger.info("STEP 2: Reading prior probabilities from {}".format(prior_prob_path))

    # STEP 3: calculate entity frequencies
    if not entity_freq_path.exists():
        logger.info("STEP 3: Calculating and writing entity frequencies to {}".format(entity_freq_path))
        io.write_entity_to_count(prior_prob_path, entity_freq_path)
    else:
        logger.info("STEP 3: Reading entity frequencies from {}".format(entity_freq_path))
  
      # STEP 2: create prior probabilities from WP
      if not prior_prob_path.exists():
          # It takes about 2h to process 1000M lines of Wikipedia XML dump
        logger.info("STEP 2: Writing prior probabilities to {}".format(prior_prob_path))
        if limit_prior is not None:
            logger.warning("Warning: reading only {} lines of Wikipedia dump".format(limit_prior))
        wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit_prior)
    else:
        logger.info("STEP 2: Reading prior probabilities from {}".format(prior_prob_path))

    # STEP 3: calculate entity frequencies
    if not entity_freq_path.exists():
        logger.info("STEP 3: Calculating and writing entity frequencies to {}".format(entity_freq_path))
        io.write_entity_to_count(prior_prob_path, entity_freq_path)
    else:
        logger.info("STEP 3: Reading entity frequencies from {}".format(entity_freq_path))
        logger.info("STEP 2: Writing prior probabilities to {}".format(prior_prob_path))
        if limit_prior is not None:
            logger.warning("Warning: reading only {} lines of Wikipedia dump".format(limit_prior))
        wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit_prior)
    else:
        logger.info("STEP 2: Reading prior probabilities from {}".format(prior_prob_path))

    # STEP 3: calculate entity frequencies
    if not entity_freq_path.exists():
        logger.info("STEP 3: Calculating and writing entity frequencies to {}".format(entity_freq_path))
        io.write_entity_to_count(prior_prob_path, entity_freq_path)
    else:
        logger.info("STEP 3: Reading entity frequencies from {}".format(entity_freq_path))
        logger.info("STEP 2: Writing prior probabilities to {}".format(prior_prob_path))
        if limit_prior is not None:
            logger.warning("Warning: reading only {} lines of Wikipedia dump".format(limit_prior))
        wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit_prior)
    else:
        logger.info("STEP 2: Reading prior probabilities from {}".format(prior_prob_path))

    # STEP 3: calculate entity frequencies
    if not entity_freq_path.exists():
        logger.info("STEP 3: Calculating and writing entity frequencies to {}".format(entity_freq_path))
        io.write_entity_to_count(prior_prob_path, entity_freq_path)
    else:
        logger.info("STEP 3: Reading entity frequencies from {}".format(entity_freq_path))
        logger.info("STEP 2: Writing prior probabilities to {}".format(prior_prob_path))
        if limit_prior is not None:
            logger.warning("Warning: reading only {} lines of Wikipedia dump".format(limit_prior))
        wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit_prior)
    else:
        logger.info("STEP 2: Reading prior probabilities from {}".format(prior_prob_path))

    # STEP 3: calculate entity frequencies
    if not entity_freq_path.exists():
        logger.info("STEP 3: Calculating and writing entity frequencies to {}".format(entity_freq_path))
        io.write_entity_to_count(prior_prob_path, entity_freq_path)
    else:
        logger.info("STEP 3: Reading entity frequencies from {}".format(entity_freq_path))
        logger.info("STEP 2: Writing prior probabilities to {}".format(prior_prob_path))
        if limit_prior is not None:
            logger.warning("Warning: reading only {} lines of Wikipedia dump".format(limit_prior))
        wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit_prior)
    else:
        logger.info("STEP 2: Reading prior probabilities from {}".format(prior_prob_path))

    # STEP 3: calculate entity frequencies
    if not entity_freq_path.exists():
        logger.info("STEP 3: Calculating and writing entity frequencies to {}".format(entity_freq_path))
        io.write_entity_to_count(prior_prob_path, entity_freq_path)
    else:
        logger.info("STEP 3: Reading entity frequencies from {}".format(entity_freq_path))
        logger.info("STEP 2: Writing prior probabilities to {}".format(prior_prob_path))
        if limit_prior is not None:
            logger.warning("Warning: reading only {} lines of Wikipedia dump".format(limit_prior))
        wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit_prior)
    else:
        logger.info("STEP 2: Reading prior probabilities from {}".format(prior_prob_path))

    # STEP 3: calculate entity frequencies
    if not entity_freq_path.exists():
        logger.info("STEP 3: Calculating and writing entity frequencies to {}".format(entity_freq_path))
        io.write_entity_to_count(prior_prob_path, entity_freq_path)
    else:
        logger.info("STEP 3: Reading entity frequencies from {}".format(entity_freq_path))
        logger.info("STEP 2: Writing prior probabilities to {}".format(prior_prob_path))
        if limit_prior is not None:
            logger.warning("Warning: reading only {} lines of Wikipedia dump".format(limit_prior))
        wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit_prior)
    else:
        logger.info("STEP 2: Reading prior probabilities from {}".format(prior_prob_path))

    # STEP 3: calculate entity frequencies
    if not entity_freq_path.exists():
        logger.info("STEP 3: Calculating and writing entity frequencies to {}".format(entity_freq_path))
        io.write_entity_to_count(prior_prob_path, entity_freq_path)
    else:
        logger.info("STEP 3: Reading entity frequencies from {}".format(entity_freq_path))
  
      # STEP 4: reading definitions and (possibly) descriptions from WikiData or from file
      message = " and descriptions" if not descriptions_from_wikipedia else ""
      if (not entity_defs_path.exists()) or (not descriptions_from_wikipedia and not entity_descr_path.exists()):
          # It takes about 10h to process 55M lines of Wikidata JSON dump
keep keep keep keep replace replace replace replace replace replace replace keep keep keep keep keep
 KB extensions and better parsing of WikiData (#4375)

* fix overflow error on windows

* more documentation & logging fixes

* md fix

* 3 different limit parameters to play with execution time

* bug fixes directory locations

* small fixes

* exclude dev test articles from prior probabilities stats

* small fixes

* filtering wikidata entities, removing numeric and meta items

* adding aliases from wikidata also to the KB

* fix adding WD aliases

* adding also new aliases to previously added entities

* fixing comma's

* small doc fixes

* adding subclassof filtering

* append alias functionality in KB

* prevent appending the same entity-alias pair

* fix for appending WD aliases

* remove date filter

* remove unnecessary import

* small corrections and reformatting

* remove WD aliases for now (too slow)

* removing numeric entities from training and evaluation

* small fixes

* shortcut during prediction if there is only one candidate

* add counts and fscore logging, remove FP NER from evaluation

* fix entity_linker.predict to take docs instead of single sentences

* remove enumeration sentences from the WP dataset

* entity_linker.update to process full doc instead of single sentence

* spelling corrections and dump locations in readme

* NLP IO fix

* reading KB is unnecessary at the end of the pipeline

* small logging fix

* remove empty files 
bin/wiki_entity_linking/wikidata_pretrain_kb.py