      parse_labels = False
      parse_aliases = False
      parse_claims = False
  
      with gzip.open(wikidata_file, mode='rb') as file:
          for cnt, line in enumerate(file):
              if limit and cnt >= limit:
                  break
              if cnt % 500000 == 0:
                  logger.info("processed {} lines of WikiData dump".format(cnt))
 
</s> remove             if cnt % 500000 == 0:
                logger.info("processed {} lines of WikiData dump".format(cnt))
 </s> add             if cnt % 500000 == 0 and cnt > 0:
                logger.info("processed {} lines of WikiData JSON dump".format(cnt)) </s> remove     parse_aliases = False
    parse_claims = False
 </s> add     parse_aliases = True
    parse_claims = True </s> remove             if cnt % 25000000 == 0:
 </s> add             if cnt % 25000000 == 0 and cnt > 0: </s> remove         cnt = 0
 </s> add 
    with bz2.open(wikidata_file, mode='rb') as file:
      parse_labels = False
      parse_aliases = False
      parse_claims = False
  
    with bz2.open(wikidata_file, mode='rb') as file:
          for cnt, line in enumerate(file):
              if limit and cnt >= limit:
                  break
              if cnt % 500000 == 0:
                  logger.info("processed {} lines of WikiData dump".format(cnt))
keep keep keep keep replace keep keep keep keep keep
 KB extensions and better parsing of WikiData (#4375)

* fix overflow error on windows

* more documentation & logging fixes

* md fix

* 3 different limit parameters to play with execution time

* bug fixes directory locations

* small fixes

* exclude dev test articles from prior probabilities stats

* small fixes

* filtering wikidata entities, removing numeric and meta items

* adding aliases from wikidata also to the KB

* fix adding WD aliases

* adding also new aliases to previously added entities

* fixing comma's

* small doc fixes

* adding subclassof filtering

* append alias functionality in KB

* prevent appending the same entity-alias pair

* fix for appending WD aliases

* remove date filter

* remove unnecessary import

* small corrections and reformatting

* remove WD aliases for now (too slow)

* removing numeric entities from training and evaluation

* small fixes

* shortcut during prediction if there is only one candidate

* add counts and fscore logging, remove FP NER from evaluation

* fix entity_linker.predict to take docs instead of single sentences

* remove enumeration sentences from the WP dataset

* entity_linker.update to process full doc instead of single sentence

* spelling corrections and dump locations in readme

* NLP IO fix

* reading KB is unnecessary at the end of the pipeline

* small logging fix

* remove empty files 
bin/wiki_entity_linking/wikidata_processor.py