docstr: Initialize Aquaint Entity Linking corpus.
  
  introduced in: D. Milne and I. H. Witten. Learning to link with wikipedia
  https://www.cms.waikato.ac.nz/~ihw/papers/08-DNM-IHW-LearningToLinkWithWikipedia.pdf . If you call the constructor the first
  time the dataset gets automatically downloaded and transformed in tab-separated column format (aquaint.txt).
  
  Parameters
  ----------
  base_path : Union[str, Path], optional
      Default is None, meaning that corpus gets auto-downloaded and loaded. You can override this
      to point to a different folder but typically this should not be necessary.
  in_memory: bool
      If True, keeps dataset in memory giving speedups in training.
  agreement_threshold: float
      Some link annotations come with an agreement_score representing the agreement from the human annotators. The score ranges from lowest 0.2
      to highest 1.0. The lower the score, the less "important" is the entity because fewer annotators thought it was worth linking.
      Default is 0.5 which means the majority of annotators must have annoteted the respective entity mention.
  sentence_splitter: `SentenceSplitter`
      The sentencesplitter that is used to split the articles into sentences.
docstr_tok: ["Initialize","Aquaint","Entity","Linking","corpus",".","introduced","in",":","D.","Milne","and","I.","H.","Witten",".","Learning","to","link","with","wikipedia","https",":","\/\/www.cms.waikato.ac.nz\/~ihw\/papers\/08-DNM-IHW-LearningToLinkWithWikipedia.pdf",".","If","you","call","the","constructor","the","first","time","the","dataset","gets","automatically","downloaded","and","transformed","in","tab-separated","column","format","(","aquaint.txt",")",".","Parameters","--","--","--","--","--","base_path",":","Union","[","str",",","Path","]",",","optional","Default","is","None",",","meaning","that","corpus","gets","auto-downloaded","and","loaded",".","You","can","override","this","to","point","to","a","different","folder","but","typically","this","should","not","be","necessary",".","in_memory",":","bool","If","True",",","keeps","dataset","in","memory","giving","speedups","in","training",".","agreement_threshold",":","float","Some","link","annotations","come","with","an","agreement_score","representing","the","agreement","from","the","human","annotators",".","The","score","ranges","from","lowest","0.2","to","highest","1.0",".","The","lower","the","score",",","the","less","``","important","''","is","the","entity","because","fewer","annotators","thought","it","was","worth","linking",".","Default","is","0.5","which","means","the","majority","of","annotators","must","have","annoteted","the","respective","entity","mention",".","sentence_splitter",":","`","SentenceSplitter","`","The","sentencesplitter","that","is","used","to","split","the","articles","into","sentences","."]
code: def __init__(
      self,
      base_path: Optional[Union[str, Path]] = None,
      in_memory: bool = True,
      agreement_threshold: float = 0.5,
      sentence_splitter: SentenceSplitter = SegtokSentenceSplitter(),
      **corpusargs,
  ) -> None:
      """Initialize Aquaint Entity Linking corpus.
  
      introduced in: D. Milne and I. H. Witten. Learning to link with wikipedia
      https://www.cms.waikato.ac.nz/~ihw/papers/08-DNM-IHW-LearningToLinkWithWikipedia.pdf . If you call the constructor the first
      time the dataset gets automatically downloaded and transformed in tab-separated column format (aquaint.txt).
  
      Parameters
      ----------
      base_path : Union[str, Path], optional
          Default is None, meaning that corpus gets auto-downloaded and loaded. You can override this
          to point to a different folder but typically this should not be necessary.
      in_memory: bool
          If True, keeps dataset in memory giving speedups in training.
      agreement_threshold: float
          Some link annotations come with an agreement_score representing the agreement from the human annotators. The score ranges from lowest 0.2
          to highest 1.0. The lower the score, the less "important" is the entity because fewer annotators thought it was worth linking.
          Default is 0.5 which means the majority of annotators must have annoteted the respective entity mention.
      sentence_splitter: `SentenceSplitter`
          The sentencesplitter that is used to split the articles into sentences.
      """
      base_path = (
          flair.cache_root / "datasets"
          if not base_path
          else Path(base_path)
      )
  
      self.agreement_threshold = agreement_threshold
  
      # this dataset name
      dataset_name = (
          self.__class__.__name__.lower()
          + "_"
          + type(sentence_splitter).__name__
      )
  
      # default dataset folder is the cache root
  
      data_folder = base_path / dataset_name
  
      aquaint_el_path = (
          "https://www.nzdl.org/wikification/data/wikifiedStories.zip"
      )
      corpus_file_name = "aquaint.txt"
      parsed_dataset = data_folder / corpus_file_name
  
      # download and parse data if necessary
      if not parsed_dataset.exists():
          aquaint_el_zip = cached_path(
              f"{aquaint_el_path}", Path("datasets") / dataset_name
          )
          unpack_file(aquaint_el_zip, data_folder, "zip", False)
  
          try:
              with open(
                  parsed_dataset, "w", encoding="utf-8"
              ) as txt_out:
                  # iterate over all html files
                  for file in os.listdir(data_folder):
                      if not file.endswith(".htm"):
                          continue
  
                      with open(
                          str(data_folder / file), encoding="utf-8"
                      ) as txt_in:
                          text = txt_in.read()
  
                      # get rid of html syntax, we only need the text
                      strings = text.split("<p> ")
                      strings[0] = strings[0].split('<h1 id="header">')[
                          1
                      ][:-7]
  
                      for i in range(1, len(strings) - 1):
                          strings[i] = strings[i][:-7]
  
                      strings[-1] = strings[-1][:-23]
  
                      # between all documents we write a separator symbol
                      txt_out.write("-DOCSTART-\n\n")
  
                      for string in strings:
                          # skip empty strings
                          if not string:
                              continue
  
                          # process the annotation format in the text and collect triples (begin_mention, length_mention, wikiname)
                          indices = []
                          lengths = []
                          wikinames = []
  
                          current_entity = string.find(
                              "[["
                          )  # each annotation starts with '[['
                          while current_entity != -1:
                              wikiname = ""
                              surface_form = ""
                              j = current_entity + 2
  
                              while string[j] not in ["]", "|"]:
                                  wikiname += string[j]
                                  j += 1
  
                              if (
                                  string[j] == "]"
                              ):  # entity mention ends, i.e. looks like this [[wikiname]]
                                  surface_form = wikiname  # in this case entity mention = wiki-page name
                              else:  # string[j] == '|'
                                  j += 1
                                  while string[j] not in ["]", "|"]:
                                      surface_form += string[j]
                                      j += 1
  
                                  if (
                                      string[j] == "|"
                                  ):  # entity has a score, i.e. looks like this [[wikiname|surface_form|agreement_score]]
                                      agreement_score = float(
                                          string[j + 1 : j + 4]
                                      )
                                      j += 4  # points to first ']' of entity now
                                      if (
                                          agreement_score
                                          < self.agreement_threshold
                                      ):  # discard entity
                                          string = (
                                              string[:current_entity]
                                              + surface_form
                                              + string[j + 2 :]
                                          )
                                          current_entity = string.find(
                                              "[["
                                          )
                                          continue
  
                              # replace [[wikiname|surface_form|score]] by surface_form and save index, length and wikiname of mention
                              indices.append(current_entity)
                              lengths.append(len(surface_form))
                              wikinames.append(
                                  wikiname[0].upper()
                                  + wikiname.replace(" ", "_")[1:]
                              )
  
                              string = (
                                  string[:current_entity]
                                  + surface_form
                                  + string[j + 2 :]
                              )
  
                              current_entity = string.find("[[")
  
                          # sentence splitting and tokenization
                          sentences = sentence_splitter.split(string)
                          sentence_offsets = [
                              sentence.start_position or 0
                              for sentence in sentences
                          ]
  
                          # iterate through all annotations and add to corresponding tokens
                          for (
                              mention_start,
                              mention_length,
                              wikiname,
                          ) in zip(indices, lengths, wikinames):
                              # find sentence to which annotation belongs
                              sentence_index = 0
                              for i in range(1, len(sentences)):
                                  if (
                                      mention_start
                                      < sentence_offsets[i]
                                  ):
                                      break
                                  else:
                                      sentence_index += 1
  
                              # position within corresponding sentence
                              mention_start -= sentence_offsets[
                                  sentence_index
                              ]
                              mention_end = (
                                  mention_start + mention_length
                              )
  
                              # set annotation for tokens of entity mention
                              first = True
                              for token in sentences[
                                  sentence_index
                              ].tokens:
                                  assert (
                                      token.start_position is not None
                                  )
                                  assert token.end_position is not None
                                  if (
                                      token.start_position
                                      >= mention_start
                                      and token.end_position
                                      <= mention_end
                                  ):  # token belongs to entity mention
                                      if first:
                                          token.set_label(
                                              typename="nel",
                                              value="B-" + wikiname,
                                          )
                                          first = False
                                      else:
                                          token.set_label(
                                              typename="nel",
                                              value="I-" + wikiname,
                                          )
  
                          # write to out-file in column format
                          for sentence in sentences:
                              for token in sentence.tokens:
                                  labels = token.get_labels("nel")
  
                                  if len(labels) == 0:  # no entity
                                      txt_out.write(
                                          token.text + "\tO\n"
                                      )
  
                                  else:  # annotation
                                      txt_out.write(
                                          token.text
                                          + "\t"
                                          + labels[0].value
                                          + "\n"
                                      )
  
                              txt_out.write(
                                  "\n"
                              )  # empty line after each sentence
  
          except Exception:
              # in case something goes wrong, delete the dataset and raise error
              os.remove(parsed_dataset)
              raise
  
      super().__init__(
          data_folder,
          column_format={0: "text", 1: "nel"},
          train_file=corpus_file_name,
          in_memory=in_memory,
          **corpusargs,
      )
  
imports: import csv
  import logging
  import os
  from pathlib import Path
  from typing import Dict, List, Optional, Union
  import requests
  import flair
  from flair.data import Corpus, MultiCorpus, Sentence
  from flair.datasets.sequence_labeling import ColumnCorpus, MultiFileColumnCorpus
  from flair.file_utils import cached_path, unpack_file
  from flair.splitter import SegtokSentenceSplitter, SentenceSplitter
label_desc: Use image node_id 1 to create a new NEL_ENGLISH_AQUAINT object from inherited base classes: ColumnCorpus with example: obj = NEL_ENGLISH_AQUAINT(base_path, in_memory, agreement_threshold, sentence_splitter)