                  obj = json.loads(clean_line)
                  entry_type = obj["type"]
  
                  if entry_type == "item":
                      # filtering records on their properties (currently disabled to get ALL data)
                      # keep = False
                      keep = True
  
                      claims = obj["claims"]
                      if parse_claims:
                          for prop, value_set in prop_filter.items():
 
</s> remove                         for prop, value_set in prop_filter.items():
 </s> add                         for prop, value_set in neg_prop_filter.items(): </s> remove                                         keep = True
 </s> add                                         keep = False </s> remove     # properties filter (currently disabled to get ALL data)
    prop_filter = dict()
    # prop_filter = {'P31': {'Q5', 'Q15632617'}}     # currently defined as OR: one property suffices to be selected
 </s> add     # filter: currently defined as OR: one hit suffices to be removed from further processing
    exclude_list = WD_META_ITEMS

    # punctuation
    exclude_list.extend(["Q1383557", "Q10617810"])

    # letters etc
    exclude_list.extend(["Q188725", "Q19776628", "Q3841820", "Q17907810", "Q9788", "Q9398093"])

    neg_prop_filter = {
        'P31': exclude_list,    # instance of
        'P279': exclude_list    # subclass
    } </s> remove def read_wikidata_entities_json(wikidata_file, limit=None, to_print=False, lang="en", parse_descriptions=True):
    # Read the JSON wiki data and parse out the entities. Takes about 7u30 to parse 55M lines.
 </s> add def read_wikidata_entities_json(wikidata_file, limit=None, to_print=False, lang="en", parse_descr=True):
    # Read the JSON wiki data and parse out the entities. Takes about 7-10h to parse 55M lines. </s> remove     parse_aliases = False
    parse_claims = False
 </s> add     parse_aliases = True
    parse_claims = True
                  obj = json.loads(clean_line)
                  entry_type = obj["type"]
  
                  if entry_type == "item":


                      keep = True
  
                      claims = obj["claims"]
                      if parse_claims:
                          for prop, value_set in prop_filter.items():
keep keep keep keep replace replace keep keep keep keep keep
 KB extensions and better parsing of WikiData (#4375)

* fix overflow error on windows

* more documentation & logging fixes

* md fix

* 3 different limit parameters to play with execution time

* bug fixes directory locations

* small fixes

* exclude dev test articles from prior probabilities stats

* small fixes

* filtering wikidata entities, removing numeric and meta items

* adding aliases from wikidata also to the KB

* fix adding WD aliases

* adding also new aliases to previously added entities

* fixing comma's

* small doc fixes

* adding subclassof filtering

* append alias functionality in KB

* prevent appending the same entity-alias pair

* fix for appending WD aliases

* remove date filter

* remove unnecessary import

* small corrections and reformatting

* remove WD aliases for now (too slow)

* removing numeric entities from training and evaluation

* small fixes

* shortcut during prediction if there is only one candidate

* add counts and fscore logging, remove FP NER from evaluation

* fix entity_linker.predict to take docs instead of single sentences

* remove enumeration sentences from the WP dataset

* entity_linker.update to process full doc instead of single sentence

* spelling corrections and dump locations in readme

* NLP IO fix

* reading KB is unnecessary at the end of the pipeline

* small logging fix

* remove empty files 
bin/wiki_entity_linking/wikidata_processor.py