docstr: Initialize a sentence-segmented version of the HIPE entity linking corpus for historical German.
  
  see description of HIPE at https://impresso.github.io/CLEF-HIPE-2020/.
  
  This version was segmented by @stefan-it and is hosted at https://github.com/stefan-it/clef-hipe.
  If you call the constructor the first time the dataset gets automatically downloaded and transformed in tab-separated column format.
  
  Parameters
  ----------
  base_path : Union[str, Path], optional
      Default is None, meaning that corpus gets auto-downloaded and loaded. You can override this
      to point to a different folder but typically this should not be necessary.
  in_memory: bool
      If True, keeps dataset in memory giving speedups in training.
  wiki_language: str
      specify the language of the names of the wikipedia pages, i.e. which language version of
      Wikipedia URLs to use. Since the text is in german the default language is German.
docstr_tok: ["Initialize","a","sentence-segmented","version","of","the","HIPE","entity","linking","corpus","for","historical","German",".","see","description","of","HIPE","at","https",":","\/\/impresso.github.io\/CLEF-HIPE-2020\/",".","This","version","was","segmented","by","@","stefan-it","and","is","hosted","at","https",":","\/\/github.com\/stefan-it\/clef-hipe",".","If","you","call","the","constructor","the","first","time","the","dataset","gets","automatically","downloaded","and","transformed","in","tab-separated","column","format",".","Parameters","--","--","--","--","--","base_path",":","Union","[","str",",","Path","]",",","optional","Default","is","None",",","meaning","that","corpus","gets","auto-downloaded","and","loaded",".","You","can","override","this","to","point","to","a","different","folder","but","typically","this","should","not","be","necessary",".","in_memory",":","bool","If","True",",","keeps","dataset","in","memory","giving","speedups","in","training",".","wiki_language",":","str","specify","the","language","of","the","names","of","the","wikipedia","pages",",","i.e",".","which","language","version","of","Wikipedia","URLs","to","use",".","Since","the","text","is","in","german","the","default","language","is","German","."]
code: def __init__(
      self,
      base_path: Optional[Union[str, Path]] = None,
      in_memory: bool = True,
      wiki_language: str = "dewiki",
      **corpusargs,
  ) -> None:
      """Initialize a sentence-segmented version of the HIPE entity linking corpus for historical German.
  
      see description of HIPE at https://impresso.github.io/CLEF-HIPE-2020/.
  
      This version was segmented by @stefan-it and is hosted at https://github.com/stefan-it/clef-hipe.
      If you call the constructor the first time the dataset gets automatically downloaded and transformed in tab-separated column format.
  
      Parameters
      ----------
      base_path : Union[str, Path], optional
          Default is None, meaning that corpus gets auto-downloaded and loaded. You can override this
          to point to a different folder but typically this should not be necessary.
      in_memory: bool
          If True, keeps dataset in memory giving speedups in training.
      wiki_language: str
          specify the language of the names of the wikipedia pages, i.e. which language version of
          Wikipedia URLs to use. Since the text is in german the default language is German.
      """
      self.wiki_language = wiki_language
      base_path = (
          flair.cache_root / "datasets"
          if not base_path
          else Path(base_path)
      )
  
      # this dataset name
      dataset_name = self.__class__.__name__.lower()
  
      data_folder = base_path / dataset_name
  
      dev_raw_url = "https://raw.githubusercontent.com/stefan-it/clef-hipe/main/data/future/dev-v1.2/de/HIPE-data-v1.2-dev-de-normalized-manual-eos.tsv"
      test_raw_url = "https://raw.githubusercontent.com/stefan-it/clef-hipe/main/data/future/test-v1.3/de/HIPE-data-v1.3-test-de-normalized-manual-eos.tsv"
      train_raw_url = "https://raw.githubusercontent.com/stefan-it/clef-hipe/main/data/future/training-v1.2/de/HIPE-data-v1.2-train-de-normalized-manual-eos.tsv"
      train_file_name = wiki_language + "_train.tsv"
      parsed_dataset = data_folder / train_file_name
  
      # download and parse data if necessary
      if not parsed_dataset.exists():
          # from qwikidata.linked_data_interface import get_entity_dict_from_api
  
          original_train_path = cached_path(
              f"{train_raw_url}", Path("datasets") / dataset_name
          )
          original_test_path = cached_path(
              f"{test_raw_url}", Path("datasets") / dataset_name
          )
          original_dev_path = cached_path(
              f"{dev_raw_url}", Path("datasets") / dataset_name
          )
  
          # generate qid wikiname dictionaries
          log.info("Get wikinames from wikidata...")
          train_dict = self._get_qid_wikiname_dict(
              path=original_train_path
          )
          test_dict = self._get_qid_wikiname_dict(original_test_path)
          dev_dict = self._get_qid_wikiname_dict(original_dev_path)
          log.info("...done!")
  
          # merge dictionaries
          qid_wikiname_dict = {**train_dict, **test_dict, **dev_dict}
  
          for doc_path, file_name in zip(
              [
                  original_train_path,
                  original_test_path,
                  original_dev_path,
              ],
              [
                  train_file_name,
                  wiki_language + "_test.tsv",
                  wiki_language + "_dev.tsv",
              ],
          ):
              with open(doc_path, encoding="utf-8") as read, open(
                  data_folder / file_name, "w", encoding="utf-8"
              ) as write:
                  # ignore first line
                  read.readline()
                  line = read.readline()
                  last_eos = True
  
                  while line:
                      # commented and empty lines
                      if line[0] == "#" or line == "\n":
                          if (
                              line[2:13] == "document_id"
                          ):  # beginning of new document
                              if last_eos:
                                  write.write("-DOCSTART-\n\n")
                                  last_eos = False
                              else:
                                  write.write("\n-DOCSTART-\n\n")
  
                      else:
                          line_list = line.split("\t")
                          if line_list[7] not in [
                              "_",
                              "NIL",
                          ]:  # line has wikidata link
                              wikiname = qid_wikiname_dict[line_list[7]]
  
                              annotation = (
                                  line_list[1][:2] + wikiname
                                  if wikiname != "O"
                                  else "O"
                              )
  
                          else:
                              annotation = "O"
  
                          write.write(
                              line_list[0] + "\t" + annotation + "\n"
                          )
  
                          if (
                              line_list[-1][-4:-1] == "EOS"
                          ):  # end of sentence
                              write.write("\n")
                              last_eos = True
                          else:
                              last_eos = False
  
                      line = read.readline()
  
      super().__init__(
          data_folder,
          column_format={0: "text", 1: "nel"},
          train_file=train_file_name,
          dev_file=wiki_language + "_dev.tsv",
          test_file=wiki_language + "_test.tsv",
          in_memory=in_memory,
          **corpusargs,
      )
  
imports: import csv
  import logging
  import os
  from pathlib import Path
  from typing import Dict, List, Optional, Union
  import requests
  import flair
  from flair.data import Corpus, MultiCorpus, Sentence
  from flair.datasets.sequence_labeling import ColumnCorpus, MultiFileColumnCorpus
  from flair.file_utils import cached_path, unpack_file
  from flair.splitter import SegtokSentenceSplitter, SentenceSplitter
label_desc: Use image node_id 1 to create a new NEL_GERMAN_HIPE object from inherited base classes: ColumnCorpus with example: obj = NEL_GERMAN_HIPE(base_path, in_memory, wiki_language)