  
      # STEP 2: create a training dataset from WP
      logger.info("STEP 2: reading training dataset from {}".format(training_path))
  
      train_data = training_set_creator.read_training(
          nlp=nlp,
          entity_file_path=training_path,
          dev=False,
          limit=train_inst,
          kb=kb,
 
</s> remove     # STEP 2: create a training dataset from WP
    logger.info("STEP 2: reading training dataset from {}".format(training_path))
 </s> add     # STEP 2: read the training dataset previously created from WP
    logger.info("STEP 2: Reading training dataset from {}".format(training_path))

    if labels_discard:
        labels_discard = [x.strip() for x in labels_discard.split(",")]
        logger.info("Discarding {} NER types: {}".format(len(labels_discard), labels_discard)) </s> remove         kb=kb,
 </s> add         kb=None,
        labels_discard=labels_discard </s> remove         logger.info("STEP 2: writing prior probabilities to {}".format(prior_prob_path))
        wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit)
    logger.info("STEP 2: reading prior probabilities from {}".format(prior_prob_path))

    # STEP 3: deduce entity frequencies from WP (takes only a few minutes)
    logger.info("STEP 3: calculating entity frequencies")
    wp.write_entity_counts(prior_prob_path, entity_freq_path, to_print=False)
 </s> add         logger.info("STEP 2: Writing prior probabilities to {}".format(prior_prob_path))
        if limit_prior is not None:
            logger.warning("Warning: reading only {} lines of Wikipedia dump".format(limit_prior))
        wp.read_prior_probs(wp_xml, prior_prob_path, limit=limit_prior)
    else:
        logger.info("STEP 2: Reading prior probabilities from {}".format(prior_prob_path))

    # STEP 3: calculate entity frequencies
    if not entity_freq_path.exists():
        logger.info("STEP 3: Calculating and writing entity frequencies to {}".format(entity_freq_path))
        io.write_entity_to_count(prior_prob_path, entity_freq_path)
    else:
        logger.info("STEP 3: Reading entity frequencies from {}".format(entity_freq_path)) </s> remove     # for testing, get all pos instances, whether or not they are in the kb
    dev_data = training_set_creator.read_training(
 </s> add     # for testing, get all pos instances (independently of KB)
    dev_data = wikipedia_processor.read_training( </s> remove     # STEP 3: create and train the entity linking pipe
    logger.info("STEP 3: training Entity Linking pipe")
 </s> add     # STEP 3: create and train an entity linking pipe
    logger.info("STEP 3: Creating and training an Entity Linking pipe")
    train_data = wikipedia_processor.read_training(
  
      # STEP 2: create a training dataset from WP
      logger.info("STEP 2: reading training dataset from {}".format(training_path))
  
    train_data = wikipedia_processor.read_training(
          nlp=nlp,
          entity_file_path=training_path,
          dev=False,
          limit=train_inst,
          kb=kb,
keep keep keep keep replace keep keep keep keep keep
 KB extensions and better parsing of WikiData (#4375)

* fix overflow error on windows

* more documentation & logging fixes

* md fix

* 3 different limit parameters to play with execution time

* bug fixes directory locations

* small fixes

* exclude dev test articles from prior probabilities stats

* small fixes

* filtering wikidata entities, removing numeric and meta items

* adding aliases from wikidata also to the KB

* fix adding WD aliases

* adding also new aliases to previously added entities

* fixing comma's

* small doc fixes

* adding subclassof filtering

* append alias functionality in KB

* prevent appending the same entity-alias pair

* fix for appending WD aliases

* remove date filter

* remove unnecessary import

* small corrections and reformatting

* remove WD aliases for now (too slow)

* removing numeric entities from training and evaluation

* small fixes

* shortcut during prediction if there is only one candidate

* add counts and fscore logging, remove FP NER from evaluation

* fix entity_linker.predict to take docs instead of single sentences

* remove enumeration sentences from the WP dataset

* entity_linker.update to process full doc instead of single sentence

* spelling corrections and dump locations in readme

* NLP IO fix

* reading KB is unnecessary at the end of the pipeline

* small logging fix

* remove empty files 
bin/wiki_entity_linking/wikidata_train_entity_linker.py