docstr: Initialize a custom corpus with any Word Sense Disambiguation (WSD) datasets in the UFSAC format.
  
   see https://github.com/getalp/UFSAC.
  
  If the constructor is called for the first time the data is automatically downloaded and transformed from xml to a tab separated column format.
  Since only the WordNet 3.0 version for senses is consistently available for all provided datasets we will only consider this version.
  Also we ignore the id annotation used in datasets that were originally created for evaluation tasks
  
  Args:
      filenames: Here you can pass a single datasetname or a list of datasetnames. The available names are: 'masc', 'omsti', 'raganato_ALL', 'raganato_semeval2007', 'raganato_semeval2013', 'raganato_semeval2015', 'raganato_senseval2', 'raganato_senseval3', 'semcor', 'semeval2007task17', 'semeval2007task7', 'semeval2013task12', 'semeval2015task13', 'senseval2', 'senseval2_lexical_sample_test', 'senseval2_lexical_sample_train', 'senseval3task1', 'senseval3task6_test', 'senseval3task6_train', 'trainomatic', 'wngt',
      base_path: You can override this to point to a specific folder but typically this should not be necessary.
      in_memory: If True, keeps dataset in memory giving speedups in training.
      cut_multisense: Boolean that determines whether the wn30_key tag should be cut if it contains multiple possible senses. If True only the first listed sense will be used and the suffix '_cut' will be added to the name of the CoNLL file. Otherwise the whole list of senses will be detected as one new sense. The default is True.
      columns: Columns to consider when loading the dataset. You can add 1: "lemma" or 2: "pos" to the default dict {0: "text", 3: "sense"} if you want to use additional pos and/or lemma for the words.
      banned_sentences: Optionally remove sentences from the corpus. Works only if `in_memory` is true
      sample_missing_splits_in_multicorpus: Whether to sample missing splits when loading the multicorpus (this is redundant if sample_missing_splits_in_each_corpus is True)
      sample_missing_splits_in_each_corpus: Whether to sample missing splits when loading each single corpus given in filenames.
      use_raganato_ALL_as_test_data: If True, the raganato_ALL dataset (Raganato et al. "Word Sense Disambiguation: A unified evaluation framework and empirical compariso") will be used as test data. Note that the sample_missing_splits parameters are set to 'only_dev' in this case if set to True.
      name: Name of your corpus
docstr_tok: ["Initialize","a","custom","corpus","with","any","Word","Sense","Disambiguation","(","WSD",")","datasets","in","the","UFSAC","format",".","see","https",":","\/\/github.com\/getalp\/UFSAC",".","If","the","constructor","is","called","for","the","first","time","the","data","is","automatically","downloaded","and","transformed","from","xml","to","a","tab","separated","column","format",".","Since","only","the","WordNet","3.0","version","for","senses","is","consistently","available","for","all","provided","datasets","we","will","only","consider","this","version",".","Also","we","ignore","the","id","annotation","used","in","datasets","that","were","originally","created","for","evaluation","tasks","Args",":","filenames",":","Here","you","can","pass","a","single","datasetname","or","a","list","of","datasetnames",".","The","available","names","are",":","'masc","'",",","'omsti","'",",","'raganato_ALL","'",",","'raganato_semeval2007","'",",","'raganato_semeval2013","'",",","'raganato_semeval2015","'",",","'raganato_senseval2","'",",","'raganato_senseval3","'",",","'semcor","'",",","'semeval2007task17","'",",","'semeval2007task7","'",",","'semeval2013task12","'",",","'semeval2015task13","'",",","'senseval2","'",",","'senseval2_lexical_sample_test","'",",","'senseval2_lexical_sample_train","'",",","'senseval3task1","'",",","'senseval3task6_test","'",",","'senseval3task6_train","'",",","'trainomatic","'",",","'wngt","'",",","base_path",":","You","can","override","this","to","point","to","a","specific","folder","but","typically","this","should","not","be","necessary",".","in_memory",":","If","True",",","keeps","dataset","in","memory","giving","speedups","in","training",".","cut_multisense",":","Boolean","that","determines","whether","the","wn30_key","tag","should","be","cut","if","it","contains","multiple","possible","senses",".","If","True","only","the","first","listed","sense","will","be","used","and","the","suffix","'_cut","'","will","be","added","to","the","name","of","the","CoNLL","file",".","Otherwise","the","whole","list","of","senses","will","be","detected","as","one","new","sense",".","The","default","is","True",".","columns",":","Columns","to","consider","when","loading","the","dataset",".","You","can","add","1",":","``","lemma","''","or","2",":","``","pos","''","to","the","default","dict","{","0",":","``","text","''",",","3",":","``","sense","''","}","if","you","want","to","use","additional","pos","and\/or","lemma","for","the","words",".","banned_sentences",":","Optionally","remove","sentences","from","the","corpus",".","Works","only","if","`","in_memory","`","is","true","sample_missing_splits_in_multicorpus",":","Whether","to","sample","missing","splits","when","loading","the","multicorpus","(","this","is","redundant","if","sample_missing_splits_in_each_corpus","is","True",")","sample_missing_splits_in_each_corpus",":","Whether","to","sample","missing","splits","when","loading","each","single","corpus","given","in","filenames",".","use_raganato_ALL_as_test_data",":","If","True",",","the","raganato_ALL","dataset","(","Raganato","et","al",".","``","Word","Sense","Disambiguation",":","A","unified","evaluation","framework","and","empirical","compariso","''",")","will","be","used","as","test","data",".","Note","that","the","sample_missing_splits","parameters","are","set","to","'only_dev","'","in","this","case","if","set","to","True",".","name",":","Name","of","your","corpus"]
code: def __init__(
      self,
      filenames: Union[str, List[str]] = ["masc", "semcor"],
      base_path: Optional[Union[str, Path]] = None,
      in_memory: bool = True,
      cut_multisense: bool = True,
      columns={0: "text", 3: "sense"},
      banned_sentences: Optional[List[str]] = None,
      sample_missing_splits_in_multicorpus: Union[bool, str] = True,
      sample_missing_splits_in_each_corpus: Union[bool, str] = True,
      use_raganato_ALL_as_test_data: bool = False,
      name: str = "multicorpus",
  ) -> None:
      """Initialize a custom corpus with any Word Sense Disambiguation (WSD) datasets in the UFSAC format.
  
       see https://github.com/getalp/UFSAC.
  
      If the constructor is called for the first time the data is automatically downloaded and transformed from xml to a tab separated column format.
      Since only the WordNet 3.0 version for senses is consistently available for all provided datasets we will only consider this version.
      Also we ignore the id annotation used in datasets that were originally created for evaluation tasks
  
      Args:
          filenames: Here you can pass a single datasetname or a list of datasetnames. The available names are: 'masc', 'omsti', 'raganato_ALL', 'raganato_semeval2007', 'raganato_semeval2013', 'raganato_semeval2015', 'raganato_senseval2', 'raganato_senseval3', 'semcor', 'semeval2007task17', 'semeval2007task7', 'semeval2013task12', 'semeval2015task13', 'senseval2', 'senseval2_lexical_sample_test', 'senseval2_lexical_sample_train', 'senseval3task1', 'senseval3task6_test', 'senseval3task6_train', 'trainomatic', 'wngt',
          base_path: You can override this to point to a specific folder but typically this should not be necessary.
          in_memory: If True, keeps dataset in memory giving speedups in training.
          cut_multisense: Boolean that determines whether the wn30_key tag should be cut if it contains multiple possible senses. If True only the first listed sense will be used and the suffix '_cut' will be added to the name of the CoNLL file. Otherwise the whole list of senses will be detected as one new sense. The default is True.
          columns: Columns to consider when loading the dataset. You can add 1: "lemma" or 2: "pos" to the default dict {0: "text", 3: "sense"} if you want to use additional pos and/or lemma for the words.
          banned_sentences: Optionally remove sentences from the corpus. Works only if `in_memory` is true
          sample_missing_splits_in_multicorpus: Whether to sample missing splits when loading the multicorpus (this is redundant if sample_missing_splits_in_each_corpus is True)
          sample_missing_splits_in_each_corpus: Whether to sample missing splits when loading each single corpus given in filenames.
          use_raganato_ALL_as_test_data: If True, the raganato_ALL dataset (Raganato et al. "Word Sense Disambiguation: A unified evaluation framework and empirical compariso") will be used as test data. Note that the sample_missing_splits parameters are set to 'only_dev' in this case if set to True.
          name: Name of your corpus
      """
      base_path = (
          flair.cache_root / "datasets"
          if not base_path
          else Path(base_path)
      )
  
      # this dataset name
      dataset_name = self.__class__.__name__.lower()
  
      data_folder = base_path / dataset_name
      original_data_folder = data_folder / "original_data"
  
      # check if data there, if not, download the data
      if not original_data_folder.exists():
          # create folder
          data_folder.mkdir(parents=True)
  
          # download data
          import gdown
  
          url = "https://drive.google.com/uc?id=1Oigo3kzRosz2VjyA44vpJZ58tDFyLRMO"
  
          output = data_folder / (dataset_name + ".tar")
  
          gdown.download(url, str(output), quiet=False)
  
          output = data_folder / (dataset_name + ".tar")
          unpack_file(
              file=output, unpack_to=data_folder, mode="tar", keep=False
          )
  
          os.rename(
              data_folder / "ufsac-public-2.1", original_data_folder
          )
  
      # transform data into column format if necessary
  
      # if no filenames are specified we use all the data
      if not filenames:
          filenames = [
              name[:-4]
              for name in os.listdir(original_data_folder)
              if "raganato" not in name
          ]
  
      if isinstance(filenames, str):
          filenames = [filenames]
  
      corpora: List[Corpus] = []
  
      log.info(
          "Transforming data into column format and creating corpora..."
      )
  
      if use_raganato_ALL_as_test_data:
          # in this case no test data should be generated by sampling from train data. But if the sample arguments are set to true, the dev set will be sampled
          if sample_missing_splits_in_each_corpus:
              sample_missing_splits_in_each_corpus = "only_dev"
          if sample_missing_splits_in_multicorpus:
              sample_missing_splits_in_multicorpus = "only_dev"
  
          # also we remove 'raganato_ALL' from filenames in case its in the list
          if "raganato_ALL" in filenames:
              filenames.remove("raganato_ALL")
  
          # generate the test file
          test_file = determine_tsv_file(
              filename="raganato_ALL",
              data_folder=data_folder,
              cut_multisense=cut_multisense,
          )
  
          corpus = ColumnCorpus(
              data_folder=data_folder,
              column_format=columns,
              test_file=test_file,  # corpus only has test data
              in_memory=in_memory,
              column_delimiter="\t",
              document_separator_token="-DOCSTART-",
              banned_sentences=banned_sentences,
              autofind_splits=False,
              sample_missing_splits=sample_missing_splits_in_each_corpus,
          )
          corpora.append(corpus)
  
      for filename in filenames:
          # make column file and save to data_folder
  
          new_filename = determine_tsv_file(
              filename=filename,
              data_folder=data_folder,
              cut_multisense=cut_multisense,
          )
  
          corpus = ColumnCorpus(
              data_folder=data_folder,
              column_format=columns,
              train_file=new_filename,
              in_memory=in_memory,
              column_delimiter="\t",
              document_separator_token="-DOCSTART-",
              banned_sentences=banned_sentences,
              autofind_splits=False,
              sample_missing_splits=sample_missing_splits_in_each_corpus,
          )
          corpora.append(corpus)
      log.info(
          "Done with transforming data into column format and creating corpora..."
      )
  
      super().__init__(
          corpora,
          sample_missing_splits=sample_missing_splits_in_multicorpus,
          name=name,
      )
  
imports: import csv
  import logging
  import os
  from pathlib import Path
  from typing import Dict, List, Optional, Union
  import requests
  import flair
  from flair.data import Corpus, MultiCorpus, Sentence
  from flair.datasets.sequence_labeling import ColumnCorpus, MultiFileColumnCorpus
  from flair.file_utils import cached_path, unpack_file
  from flair.splitter import SegtokSentenceSplitter, SentenceSplitter
label_desc: Use image node_id 1 to create a new WSD_UFSAC object from inherited base classes: MultiCorpus with example: obj = WSD_UFSAC(filenames, base_path, in_memory, cut_multisense, columns, banned_sentences, sample_missing_splits_in_multicorpus, sample_missing_splits_in_each_corpus, use_raganato_ALL_as_test_data, name)