                          article_doc = nlp(text)
  
                      entries_by_offset = entries_per_article.get(article_id, dict())
  
                      gold_entities = list()
                      for ent in article_doc.ents:
                          start = ent.start_char
                          end = ent.end_char
  
                          entity_tuple = entries_by_offset.get(str(start) + "-" + str(end), None)
                          if entity_tuple:
                              alias, wp_title = entity_tuple
                              if ent.text != alias:
                                  print("Non-matching entity in", article_id, start, end)
                              else:
                                  gold_entities.append((start, end, wp_title))
  
                      if gold_entities:
                          gold = GoldParse(doc=article_doc, links=gold_entities)
                          data.append((article_doc, gold))
  
                      cnt += 1
                  except Exception as e:
                      print("Problem parsing article", article_id)
                      print(e)
                      raise e
  
      if to_print:
          print()
          print("Processed", cnt, "training articles, dev=" + str(dev))
          print()
      return data
 
</s> remove                     entries_by_offset = entries_per_article.get(article_id, dict())
 </s> add         except Exception as e:
            print("Problem parsing article", article_id)
            print(e)
            raise e </s> remove     cnt = 0
    files = listdir(training_dir)
    for f in files:
        if not limit or cnt < limit:
            if dev == run_el.is_dev(f):
                article_id = f.replace(".txt", "")
                if cnt % 500 == 0 and to_print:
                    print(datetime.datetime.now(), "processed", cnt, "files in the training dataset")

                try:
                    # parse the article text
                    with open(os.path.join(training_dir, f), mode="r", encoding='utf8') as file:
                        text = file.read()
                        article_doc = nlp(text)
 </s> add             if gold_entities:
                gold = GoldParse(doc=article_doc, links=gold_entities)
                data.append((article_doc, gold)) </s> remove             entries_by_offset = entries_per_article.get(article_id, dict())
            entries_by_offset[start + "-" + end] = (alias, wp_title)
 </s> add                     alias = fields[1]
                    wp_title = fields[2]
                    start = fields[3]
                    end = fields[4]

                    entries_by_offset = entries_per_article.get(article_id, dict())
                    entries_by_offset[start + "-" + end] = (alias, wp_title) </s> remove                                             print("alias (" + lang + "):", item["value"])
 </s> add                                              print("alias (" + lang + "):", item["value"]) </s> remove             entries_per_article[article_id] = entries_by_offset
 </s> add                     entries_per_article[article_id] = entries_by_offset
                          article_doc = nlp(text)
  
                      entries_by_offset = entries_per_article.get(article_id, dict())
  



























      return data
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace keep
 reprocessing all of wikipedia for training data 
examples/pipeline/wiki_entity_linking/training_set_creator.py