      entity_vector_length=("Length of entity vectors (default 64)", "option", "v", int),
      loc_prior_prob=("Location to file with prior probabilities", "option", "p", Path),
      loc_entity_defs=("Location to file with entity definitions", "option", "d", Path),
      loc_entity_desc=("Location to file with entity descriptions", "option", "s", Path),
      descriptions_from_wikipedia=("Flag for using wp descriptions not wd", "flag", "wp"),
      limit=("Optional threshold to limit lines read from dumps", "option", "l", int),
      lang=("Optional language for which to get wikidata titles. Defaults to 'en'", "option", "la", str),
  )
  def main(
      wd_json,
      wp_xml,
      output_dir,
 
</s> remove         logger.info("STEP 4: parsing wikidata for entity definitions" + message)
        title_to_id, id_to_descr = wd.read_wikidata_entities_json(
 </s> add         logger.info("STEP 4: Parsing and writing Wikidata entity definitions to {}".format(entity_defs_path))
        if limit_wd is not None:
            logger.warning("Warning: reading only {} lines of Wikidata dump".format(limit_wd))
        title_to_id, id_to_descr, id_to_alias = wd.read_wikidata_entities_json( </s> remove     The full file takes about 2h to parse 1100M lines.
    It works relatively fast because it runs line by line, irrelevant of which article the intrawiki is from.
 </s> add     The full file takes about 2-3h to parse 1100M lines.
    It works relatively fast because it runs line by line, irrelevant of which article the intrawiki is from,
    though dev test articles are excluded in order not to get an artificially strong baseline. </s> remove     message = " and descriptions" if not descriptions_from_wikipedia else ""
    if (not entity_defs_path.exists()) or (not descriptions_from_wikipedia and not entity_descr_path.exists()):
 </s> add     if (not entity_defs_path.exists()) or (not descr_from_wp and not entity_descr_path.exists()): </s> remove         logger.info("STEP 2: writing prior probabilities to {}".format(prior_prob_path))
        wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit)
    logger.info("STEP 2: reading prior probabilities from {}".format(prior_prob_path))

    # STEP 3: deduce entity frequencies from WP (takes only a few minutes)
    logger.info("STEP 3: calculating entity frequencies")
    wp.write_entity_counts(prior_prob_path, entity_freq_path, to_print=False)
 </s> add         logger.info("STEP 2: Writing prior probabilities to {}".format(prior_prob_path))
        if limit_prior is not None:
            logger.warning("Warning: reading only {} lines of Wikipedia dump".format(limit_prior))
        wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit_prior)
    else:
        logger.info("STEP 2: Reading prior probabilities from {}".format(prior_prob_path))

    # STEP 3: calculate entity frequencies
    if not entity_freq_path.exists():
        logger.info("STEP 3: Calculating and writing entity frequencies to {}".format(entity_freq_path))
        io.write_entity_to_count(prior_prob_path, entity_freq_path)
    else:
        logger.info("STEP 3: Reading entity frequencies from {}".format(entity_freq_path))
    descr_from_wp=("Flag for using wp descriptions not wd", "flag", "wp"),
    limit_prior=("Threshold to limit lines read from WP for prior probabilities", "option", "lp", int),
    limit_train=("Threshold to limit lines read from WP for training set", "option", "lt", int),
    limit_wd=("Threshold to limit lines read from WD", "option", "lw", int),
    lang=("Optional language for which to get Wikidata titles. Defaults to 'en'", "option", "la", str),
      entity_vector_length=("Length of entity vectors (default 64)", "option", "v", int),
      loc_prior_prob=("Location to file with prior probabilities", "option", "p", Path),
      loc_entity_defs=("Location to file with entity definitions", "option", "d", Path),
      loc_entity_desc=("Location to file with entity descriptions", "option", "s", Path),
    descr_from_wp=("Flag for using wp descriptions not wd", "flag", "wp"),
    limit_prior=("Threshold to limit lines read from WP for prior probabilities", "option", "lp", int),
    limit_train=("Threshold to limit lines read from WP for training set", "option", "lt", int),
    limit_wd=("Threshold to limit lines read from WD", "option", "lw", int),
    lang=("Optional language for which to get Wikidata titles. Defaults to 'en'", "option", "la", str),
    descr_from_wp=("Flag for using wp descriptions not wd", "flag", "wp"),
    limit_prior=("Threshold to limit lines read from WP for prior probabilities", "option", "lp", int),
    limit_train=("Threshold to limit lines read from WP for training set", "option", "lt", int),
    limit_wd=("Threshold to limit lines read from WD", "option", "lw", int),
    lang=("Optional language for which to get Wikidata titles. Defaults to 'en'", "option", "la", str),
    descr_from_wp=("Flag for using wp descriptions not wd", "flag", "wp"),
    limit_prior=("Threshold to limit lines read from WP for prior probabilities", "option", "lp", int),
    limit_train=("Threshold to limit lines read from WP for training set", "option", "lt", int),
    limit_wd=("Threshold to limit lines read from WD", "option", "lw", int),
    lang=("Optional language for which to get Wikidata titles. Defaults to 'en'", "option", "la", str),
  )
  def main(
      wd_json,
      wp_xml,
      output_dir,
keep keep keep keep replace replace replace keep keep keep keep keep
 KB extensions and better parsing of WikiData (#4375)

* fix overflow error on windows

* more documentation & logging fixes

* md fix

* 3 different limit parameters to play with execution time

* bug fixes directory locations

* small fixes

* exclude dev test articles from prior probabilities stats

* small fixes

* filtering wikidata entities, removing numeric and meta items

* adding aliases from wikidata also to the KB

* fix adding WD aliases

* adding also new aliases to previously added entities

* fixing comma's

* small doc fixes

* adding subclassof filtering

* append alias functionality in KB

* prevent appending the same entity-alias pair

* fix for appending WD aliases

* remove date filter

* remove unnecessary import

* small corrections and reformatting

* remove WD aliases for now (too slow)

* removing numeric entities from training and evaluation

* small fixes

* shortcut during prediction if there is only one candidate

* add counts and fscore logging, remove FP NER from evaluation

* fix entity_linker.predict to take docs instead of single sentences

* remove enumeration sentences from the WP dataset

* entity_linker.update to process full doc instead of single sentence

* spelling corrections and dump locations in readme

* NLP IO fix

* reading KB is unnecessary at the end of the pipeline

* small logging fix

* remove empty files 
bin/wiki_entity_linking/wikidata_pretrain_kb.py