  def read_wikidata_entities_json(wikidata_file, limit=None, to_print=False):
      # Read the JSON wiki data and parse out the entities. Takes about 7u30 to parse 55M lines.
      # get latest-all.json.bz2 from https://dumps.wikimedia.org/wikidatawiki/entities/
  
      lang = 'en'
      site_filter = 'enwiki'
  
      # properties filter (currently disabled to get ALL data)
      prop_filter = dict()
      # prop_filter = {'P31': {'Q5', 'Q15632617'}}     # currently defined as OR: one property suffices to be selected
  
 
</s> remove def read_prior_probs(wikipedia_input, prior_prob_output):
 </s> add def read_prior_probs(wikipedia_input, prior_prob_output, limit=None): </s> remove     _process_wikipedia_texts(wikipedia_input, wp_to_id, training_output, limit=None)
 </s> add     _process_wikipedia_texts(wikipedia_input, wp_to_id, training_output, limit=limit) </s> remove     print(" * _get_entity_frequencies", datetime.datetime.now())
 </s> add     print(now(), " *  get entity frequencies:") </s> remove 
 </s> add     E152 = ("The `nlp` object should have a pre-trained `ner` component.")
    E153 = ("Either provide a path to a preprocessed training directory, "
            "or to the original Wikipedia XML dump.")
    E154 = ("Either the `nlp` model or the `vocab` should be specified.")
    E155 = ("The `nlp` object should have access to pre-trained word vectors, cf. "
            "https://spacy.io/usage/models#languages.")
    lang = "en"
    site_filter = "enwiki"
  def read_wikidata_entities_json(wikidata_file, limit=None, to_print=False):
      # Read the JSON wiki data and parse out the entities. Takes about 7u30 to parse 55M lines.
      # get latest-all.json.bz2 from https://dumps.wikimedia.org/wikidatawiki/entities/
  
    lang = "en"
    site_filter = "enwiki"
    lang = "en"
    site_filter = "enwiki"
  
      # properties filter (currently disabled to get ALL data)
      prop_filter = dict()
      # prop_filter = {'P31': {'Q5', 'Q15632617'}}     # currently defined as OR: one property suffices to be selected
  
keep keep keep keep replace replace keep keep keep keep keep
 CLI scripts for entity linking (wikipedia & generic) (#4091)

* document token ent_kb_id

* document span kb_id

* update pipeline documentation

* prior and context weights as bool's instead

* entitylinker api documentation

* drop for both models

* finish entitylinker documentation

* small fixes

* documentation for KB

* candidate documentation

* links to api pages in code

* small fix

* frequency examples as counts for consistency

* consistent documentation about tensors returned by predict

* add entity linking to usage 101

* add entity linking infobox and KB section to 101

* entity-linking in linguistic features

* small typo corrections

* training example and docs for entity_linker

* predefined nlp and kb

* revert back to similarity encodings for simplicity (for now)

* set prior probabilities to 0 when excluded

* code clean up

* bugfix: deleting kb ID from tokens when entities were removed

* refactor train el example to use either model or vocab

* pretrain_kb example for example kb generation

* add to training docs for KB + EL example scripts

* small fixes

* error numbering

* ensure the language of vocab and nlp stay consistent across serialization

* equality with =

* avoid conflict in errors file

* add error 151

* final adjustements to the train scripts - consistency

* update of goldparse documentation

* small corrections

* push commit

* turn kb_creator into CLI script (wip)

* proper parameters for training entity vectors

* wikidata pipeline split up into two executable scripts

* remove context_width

* move wikidata scripts in bin directory, remove old dummy script

* refine KB script with logs and preprocessing options

* small edits

* small improvements to logging of EL CLI script 
bin/wiki_entity_linking/wikidata_processor.py