docstr: Initialize AIDA CoNLL-YAGO Entity Linking corpus.
  
  The corpus got introduced here https://www.mpi-inf.mpg.de/departments/databases-and-information-systems/research/ambiverse-nlu/aida/downloads.
  License: https://creativecommons.org/licenses/by-sa/3.0/deed.en_US
  If you call the constructor the first time the dataset gets automatically downloaded.
  
  Parameters
  ----------
  base_path : Union[str, Path], optional
      Default is None, meaning that corpus gets auto-downloaded and loaded. You can override this
      to point to a different folder but typically this should not be necessary.
  in_memory: bool
      If True, keeps dataset in memory giving speedups in training.
  use_ids_and_check_existence: bool
      If True the existence of the given wikipedia ids/pagenames is checked and non existent ids/names will be ignored. This also means that one works with
      current wikipedia-arcticle names and possibly alter some of the out-dated ones in the original dataset
docstr_tok: ["Initialize","AIDA","CoNLL-YAGO","Entity","Linking","corpus",".","The","corpus","got","introduced","here","https",":","\/\/www.mpi-inf.mpg.de\/departments\/databases-and-information-systems\/research\/ambiverse-nlu\/aida\/downloads",".","License",":","https",":","\/\/creativecommons.org\/licenses\/by-sa\/3.0\/deed.en_US","If","you","call","the","constructor","the","first","time","the","dataset","gets","automatically","downloaded",".","Parameters","--","--","--","--","--","base_path",":","Union","[","str",",","Path","]",",","optional","Default","is","None",",","meaning","that","corpus","gets","auto-downloaded","and","loaded",".","You","can","override","this","to","point","to","a","different","folder","but","typically","this","should","not","be","necessary",".","in_memory",":","bool","If","True",",","keeps","dataset","in","memory","giving","speedups","in","training",".","use_ids_and_check_existence",":","bool","If","True","the","existence","of","the","given","wikipedia","ids\/pagenames","is","checked","and","non","existent","ids\/names","will","be","ignored",".","This","also","means","that","one","works","with","current","wikipedia-arcticle","names","and","possibly","alter","some","of","the","out-dated","ones","in","the","original","dataset"]
code: def __init__(
      self,
      base_path: Optional[Union[str, Path]] = None,
      in_memory: bool = True,
      use_ids_and_check_existence: bool = False,
      **corpusargs,
  ) -> None:
      """Initialize AIDA CoNLL-YAGO Entity Linking corpus.
  
      The corpus got introduced here https://www.mpi-inf.mpg.de/departments/databases-and-information-systems/research/ambiverse-nlu/aida/downloads.
      License: https://creativecommons.org/licenses/by-sa/3.0/deed.en_US
      If you call the constructor the first time the dataset gets automatically downloaded.
  
      Parameters
      ----------
      base_path : Union[str, Path], optional
          Default is None, meaning that corpus gets auto-downloaded and loaded. You can override this
          to point to a different folder but typically this should not be necessary.
      in_memory: bool
          If True, keeps dataset in memory giving speedups in training.
      use_ids_and_check_existence: bool
          If True the existence of the given wikipedia ids/pagenames is checked and non existent ids/names will be ignored. This also means that one works with
          current wikipedia-arcticle names and possibly alter some of the out-dated ones in the original dataset
      """
      base_path = (
          flair.cache_root / "datasets"
          if not base_path
          else Path(base_path)
      )
  
      # this dataset name
      dataset_name = self.__class__.__name__.lower()
      dataset_name = (
          dataset_name + "_ids"
          if use_ids_and_check_existence
          else dataset_name + "_raw"
      )
  
      data_folder = base_path / dataset_name
  
      conll_yago_path = "https://nlp.informatik.hu-berlin.de/resources/datasets/conll_entity_linking/"
      corpus_file_name = "train"
      parsed_dataset = data_folder / corpus_file_name
  
      if not parsed_dataset.exists():
          testa_unprocessed_path = cached_path(
              f"{conll_yago_path}aida_conll_testa",
              Path("datasets") / dataset_name,
          )
          testb_unprocessed_path = cached_path(
              f"{conll_yago_path}aida_conll_testb",
              Path("datasets") / dataset_name,
          )
          train_unprocessed_path = cached_path(
              f"{conll_yago_path}aida_conll_train",
              Path("datasets") / dataset_name,
          )
  
          if use_ids_and_check_existence:
              # we use the wikiids in the data instead of directly utilizing the wikipedia urls.
              # like this we can quickly check if the corresponding page exists
              import wikipediaapi
  
              wiki_wiki = wikipediaapi.Wikipedia(language="en")
              wikiid_wikiname_dict = self._get_wikiid_wikiname_dict(
                  data_folder
              )
  
          for name, path in zip(
              ["train", "testa", "testb"],
              [
                  train_unprocessed_path,
                  testa_unprocessed_path,
                  testb_unprocessed_path,
              ],
          ):
              with open(
                  data_folder / name, "w", encoding="utf-8"
              ) as write, open(path, encoding="utf-8") as read:
                  for line in read:
                      line_list = line.split("\t")
                      if len(line_list) <= 4:
                          if (
                              line_list[0][:10] == "-DOCSTART-"
                          ):  # Docstart
                              write.write("-DOCSTART-\n\n")
                          elif line_list[0] == "\n":  # empty line
                              write.write("\n")
                          else:  # text without annotation or marked '--NME--' (no matching entity)
                              if len(line_list) == 1:
                                  write.write(
                                      line_list[0][:-1] + "\tO\n"
                                  )
                              else:
                                  write.write(line_list[0] + "\tO\n")
                      else:  # line with annotation
                          if use_ids_and_check_existence:
                              wikiname = wikiid_wikiname_dict[
                                  line_list[5].strip()
                              ]
                              if wikiname != "O":
                                  write.write(
                                      line_list[0]
                                      + "\t"
                                      + line_list[1]
                                      + "-"
                                      + wikiname
                                      + "\n"
                                  )
                              else:
                                  # if there is a bad wikiid we can check if the given url in the data exists using wikipediaapi
                                  wikiname = line_list[4].split("/")[-1]
                                  page = wiki_wiki.page(wikiname)
                                  if page.exists():
                                      write.write(
                                          line_list[0]
                                          + "\t"
                                          + line_list[1]
                                          + "-"
                                          + wikiname
                                          + "\n"
                                      )
                                  else:  # neither the wikiid nor the url exist
                                      write.write(
                                          line_list[0] + "\tO\n"
                                      )
                          else:  # write wikipedia names as given in the file
                              wikiname = line_list[4].split("/")[-1]
                              write.write(
                                  line_list[0]
                                  + "\t"
                                  + line_list[1]
                                  + "-"
                                  + wikiname
                                  + "\n"
                              )
  
              # delete unprocessed file
              os.remove(path)
  
      super().__init__(
          data_folder,
          column_format={0: "text", 1: "nel"},
          train_file=corpus_file_name,
          dev_file="testa",
          test_file="testb",
          in_memory=in_memory,
          **corpusargs,
      )
  
imports: import csv
  import logging
  import os
  from pathlib import Path
  from typing import Dict, List, Optional, Union
  import requests
  import flair
  from flair.data import Corpus, MultiCorpus, Sentence
  from flair.datasets.sequence_labeling import ColumnCorpus, MultiFileColumnCorpus
  from flair.file_utils import cached_path, unpack_file
  from flair.splitter import SegtokSentenceSplitter, SentenceSplitter
label_desc: Use image node_id 1 to create a new NEL_ENGLISH_AIDA object from inherited base classes: ColumnCorpus with example: obj = NEL_ENGLISH_AIDA(base_path, in_memory, use_ids_and_check_existence)