docstr: Initialize ITTB Entity Linking corpus.
  
  The corpus got introduced in "Collective Annotation of Wikipedia Entities in Web Text" Sayali Kulkarni, Amit Singh, Ganesh Ramakrishnan, and Soumen Chakrabarti.
  
  If you call the constructor the first time the dataset gets automatically downloaded.
  
  Parameters
  ----------
  base_path : Union[str, Path], optional
      Default is None, meaning that corpus gets auto-downloaded and loaded. You can override this
      to point to a different folder but typically this should not be necessary.
  in_memory: bool
      If True, keeps dataset in memory giving speedups in training.
  ignore_disagreements: bool
      If True annotations with annotator disagreement will be ignored.
  sentence_splitter: `SentenceSplitter`
      The sentencesplitter that is used to split the articles into sentences.
docstr_tok: ["Initialize","ITTB","Entity","Linking","corpus",".","The","corpus","got","introduced","in","``","Collective","Annotation","of","Wikipedia","Entities","in","Web","Text","''","Sayali","Kulkarni",",","Amit","Singh",",","Ganesh","Ramakrishnan",",","and","Soumen","Chakrabarti",".","If","you","call","the","constructor","the","first","time","the","dataset","gets","automatically","downloaded",".","Parameters","--","--","--","--","--","base_path",":","Union","[","str",",","Path","]",",","optional","Default","is","None",",","meaning","that","corpus","gets","auto-downloaded","and","loaded",".","You","can","override","this","to","point","to","a","different","folder","but","typically","this","should","not","be","necessary",".","in_memory",":","bool","If","True",",","keeps","dataset","in","memory","giving","speedups","in","training",".","ignore_disagreements",":","bool","If","True","annotations","with","annotator","disagreement","will","be","ignored",".","sentence_splitter",":","`","SentenceSplitter","`","The","sentencesplitter","that","is","used","to","split","the","articles","into","sentences","."]
code: def __init__(
      self,
      base_path: Optional[Union[str, Path]] = None,
      in_memory: bool = True,
      ignore_disagreements: bool = False,
      sentence_splitter: SentenceSplitter = SegtokSentenceSplitter(),
      **corpusargs,
  ) -> None:
      """Initialize ITTB Entity Linking corpus.
  
      The corpus got introduced in "Collective Annotation of Wikipedia Entities in Web Text" Sayali Kulkarni, Amit Singh, Ganesh Ramakrishnan, and Soumen Chakrabarti.
  
      If you call the constructor the first time the dataset gets automatically downloaded.
  
      Parameters
      ----------
      base_path : Union[str, Path], optional
          Default is None, meaning that corpus gets auto-downloaded and loaded. You can override this
          to point to a different folder but typically this should not be necessary.
      in_memory: bool
          If True, keeps dataset in memory giving speedups in training.
      ignore_disagreements: bool
          If True annotations with annotator disagreement will be ignored.
      sentence_splitter: `SentenceSplitter`
          The sentencesplitter that is used to split the articles into sentences.
      """
      base_path = (
          flair.cache_root / "datasets"
          if not base_path
          else Path(base_path)
      )
  
      # this dataset name
      dataset_name = (
          self.__class__.__name__.lower()
          + "_"
          + type(sentence_splitter).__name__
      )
  
      data_folder = base_path / dataset_name
  
      iitb_el_docs_path = "https://www.cse.iitb.ac.in/~soumen/doc/CSAW/Annot/CSAW_crawledDocs.tar.gz"
      iitb_el_annotations_path = "https://www.cse.iitb.ac.in/~soumen/doc/CSAW/Annot/CSAW_Annotations.xml"
      corpus_file_name = "iitb.txt"
      parsed_dataset = data_folder / corpus_file_name
  
      if not parsed_dataset.exists():
          docs_zip_path = cached_path(
              f"{iitb_el_docs_path}", Path("datasets") / dataset_name
          )
          annotations_xml_path = cached_path(
              f"{iitb_el_annotations_path}",
              Path("datasets") / dataset_name,
          )
  
          unpack_file(docs_zip_path, data_folder, "tar", False)
  
          import xml.etree.ElementTree as ET
  
          tree = ET.parse(annotations_xml_path)
          root = tree.getroot()
  
          # names of raw text documents
          doc_names = set()
          for elem in root:
              if elem[0].text is not None:
                  doc_names.add(elem[0].text)
  
          # open output_file
          with open(parsed_dataset, "w", encoding="utf-8") as write:
              # iterate through all documents
              for doc_name in doc_names:
                  with open(
                      data_folder / "crawledDocs" / doc_name,
                      encoding="utf-8",
                  ) as read:
                      text = read.read()
  
                      # split sentences and tokenize
                      sentences = sentence_splitter.split(text)
                      sentence_offsets = [
                          sentence.start_position or 0
                          for sentence in sentences
                      ]
  
                      # iterate through all annotations and add to corresponding tokens
                      for elem in root:
                          if (
                              elem[0].text == doc_name and elem[2].text
                          ):  # annotation belongs to current document
                              wikiname = elem[2].text.replace(" ", "_")
                              assert elem[3].text is not None
                              assert elem[4].text is not None
                              mention_start = int(elem[3].text)
                              mention_length = int(elem[4].text)
  
                              # find sentence to which annotation belongs
                              sentence_index = 0
                              for i in range(1, len(sentences)):
                                  if (
                                      mention_start
                                      < sentence_offsets[i]
                                  ):
                                      break
                                  else:
                                      sentence_index += 1
  
                              # position within corresponding sentence
                              mention_start -= sentence_offsets[
                                  sentence_index
                              ]
                              mention_end = (
                                  mention_start + mention_length
                              )
  
                              # set annotation for tokens of entity mention
                              first = True
                              for token in sentences[
                                  sentence_index
                              ].tokens:
                                  assert (
                                      token.start_position is not None
                                  )
                                  assert token.end_position is not None
                                  if (
                                      token.start_position
                                      >= mention_start
                                      and token.end_position
                                      <= mention_end
                                  ):  # token belongs to entity mention
                                      assert elem[1].text is not None
                                      if first:
                                          token.set_label(
                                              typename=elem[1].text,
                                              value="B-" + wikiname,
                                          )
                                          first = False
                                      else:
                                          token.set_label(
                                              typename=elem[1].text,
                                              value="I-" + wikiname,
                                          )
  
                      # write to out file
                      write.write(
                          "-DOCSTART-\n\n"
                      )  # each file is one document
  
                      for sentence in sentences:
                          for token in sentence.tokens:
                              labels = token.labels
  
                              if len(labels) == 0:  # no entity
                                  write.write(token.text + "\tO\n")
  
                              elif (
                                  len(labels) == 1
                                  or labels[0].value == labels[1].value
                              ):
                                  # annotation from one annotator or two agreeing annotators
                                  write.write(
                                      token.text
                                      + "\t"
                                      + labels[0].value
                                      + "\n"
                                  )
  
                              else:  # annotators disagree: ignore or arbitrarily take first annotation
                                  if ignore_disagreements:
                                      write.write(token.text + "\tO\n")
  
                                  else:
                                      write.write(
                                          token.text
                                          + "\t"
                                          + labels[0].value
                                          + "\n"
                                      )
  
                          write.write(
                              "\n"
                          )  # empty line after each sentence
  
      super().__init__(
          data_folder,
          column_format={0: "text", 1: "nel"},
          train_file=corpus_file_name,
          in_memory=in_memory,
          **corpusargs,
      )
  
imports: import csv
  import logging
  import os
  from pathlib import Path
  from typing import Dict, List, Optional, Union
  import requests
  import flair
  from flair.data import Corpus, MultiCorpus, Sentence
  from flair.datasets.sequence_labeling import ColumnCorpus, MultiFileColumnCorpus
  from flair.file_utils import cached_path, unpack_file
  from flair.splitter import SegtokSentenceSplitter, SentenceSplitter
label_desc: Use image node_id 1 to create a new NEL_ENGLISH_IITB object from inherited base classes: ColumnCorpus with example: obj = NEL_ENGLISH_IITB(base_path, in_memory, ignore_disagreements, sentence_splitter)