docstr: Initialize LabelledTextDataAnalyzer.
  
  Args:
      preprocess_fn: the function that pre-processes the text, returns the processed text
      predefined_pattern: the dictionary maps a pattern name to its regex string
      tokenizer: the function tokenize a text into a list of tokens, the default tokenizer is the `word_tokenize`
                 in nltk.tokenize
      stop_words: the set of stop words that will be ignored in the final stats
      stop_words_by_languages: a list of language code (from nltk.corpus) to determine the stop words set by languages.
                               Supported by `nltk.corpus.stopwords`. If `stop_words` is not None, it will be ignored.
      tf_type: how the term frequency is calculated, default is
              - TF_ABSOLUTE: the occurrence of t in document d, i.e. f(t,d)
              - TF_BOOLEAN: 1 if term appeared in the document, otherwise 0
              - TF_NORMALIZE_BY_MAX: f(t,d) normalized by the maximum term frequency
              - TF_NORMALIZE_BY_DOC: f(t,d) normalized by the total number of terms in the document
              - TF_LOGARITHM: log (1+ f(t,d))
              - TF_AUGMENTED: 0.5 + 0.5 * (f(t,d) / max(f(t',d)) for t' in d)
docstr_tok: ["Initialize","LabelledTextDataAnalyzer",".","Args",":","preprocess_fn",":","the","function","that","pre-processes","the","text",",","returns","the","processed","text","predefined_pattern",":","the","dictionary","maps","a","pattern","name","to","its","regex","string","tokenizer",":","the","function","tokenize","a","text","into","a","list","of","tokens",",","the","default","tokenizer","is","the","`","word_tokenize","`","in","nltk.tokenize","stop_words",":","the","set","of","stop","words","that","will","be","ignored","in","the","final","stats","stop_words_by_languages",":","a","list","of","language","code","(","from","nltk.corpus",")","to","determine","the","stop","words","set","by","languages",".","Supported","by","`","nltk.corpus.stopwords","`",".","If","`","stop_words","`","is","not","None",",","it","will","be","ignored",".","tf_type",":","how","the","term","frequency","is","calculated",",","default","is","-","TF_ABSOLUTE",":","the","occurrence","of","t","in","document","d",",","i.e",".","f","(","t",",","d",")","-","TF_BOOLEAN",":","1","if","term","appeared","in","the","document",",","otherwise","0","-","TF_NORMALIZE_BY_MAX",":","f","(","t",",","d",")","normalized","by","the","maximum","term","frequency","-","TF_NORMALIZE_BY_DOC",":","f","(","t",",","d",")","normalized","by","the","total","number","of","terms","in","the","document","-","TF_LOGARITHM",":","log","(","1+","f","(","t",",","d",")",")","-","TF_AUGMENTED",":","0.5","+","0.5","*","(","f","(","t",",","d",")","\/","max","(","f","(","t","'",",","d",")",")","for","t","'","in","d",")"]
code: def __init__(
      self,
      preprocess_fn: Optional[Callable[[str], str]] = None,
      predefined_pattern: Optional[Dict[str, str]] = {},
      tokenizer: Optional[Callable[[str], List[str]]] = word_tokenize,
      stop_words: Optional[Set] = None,
      stop_words_by_languages: Optional[List[str]] = None,
      tf_type: Optional[int] = TermFrequencyType.TF_ABSOLUTE,
  ):
      """
      Initialize LabelledTextDataAnalyzer.
  
      Args:
          preprocess_fn: the function that pre-processes the text, returns the processed text
          predefined_pattern: the dictionary maps a pattern name to its regex string
          tokenizer: the function tokenize a text into a list of tokens, the default tokenizer is the `word_tokenize`
                     in nltk.tokenize
          stop_words: the set of stop words that will be ignored in the final stats
          stop_words_by_languages: a list of language code (from nltk.corpus) to determine the stop words set by languages.
                                   Supported by `nltk.corpus.stopwords`. If `stop_words` is not None, it will be ignored.
          tf_type: how the term frequency is calculated, default is
                  - TF_ABSOLUTE: the occurrence of t in document d, i.e. f(t,d)
                  - TF_BOOLEAN: 1 if term appeared in the document, otherwise 0
                  - TF_NORMALIZE_BY_MAX: f(t,d) normalized by the maximum term frequency
                  - TF_NORMALIZE_BY_DOC: f(t,d) normalized by the total number of terms in the document
                  - TF_LOGARITHM: log (1+ f(t,d))
                  - TF_AUGMENTED: 0.5 + 0.5 * (f(t,d) / max(f(t',d)) for t' in d)
      """
      super().__init__(data_analyzer_cls=TextDataAnalyzer)
      self._analyzer_cls_sample = TextDataAnalyzer(
          preprocess_fn=preprocess_fn,
          predefined_pattern=predefined_pattern,
          tokenizer=tokenizer,
          stop_words=stop_words,
          stop_words_by_languages=stop_words_by_languages,
          tf_type=tf_type,
      )
      self._all_analyzer = deepcopy(self._analyzer_cls_sample)
  
file_path: contextual-ai/xai/data/explorer/text/labelled_text_analyzer.py
imports: from copy import deepcopy
  from nltk.tokenize import word_tokenize
  from typing import Callable, Optional, Dict, List, Set
  from typing import Tuple, Union
  from xai.data.constants import TermFrequencyType
  from xai.data.explorer.abstract_labelled_analyzer import AbstractLabelledDataAnalyzer
  from xai.data.explorer.text.text_analyzer import TextDataAnalyzer
  from xai.data.explorer.text.text_stats import TextStats
label_desc: Use image node_id 1 to create a new LabelledTextDataAnalyzer object from inherited base classes: AbstractLabelledDataAnalyzer with example: obj = LabelledTextDataAnalyzer(preprocess_fn, predefined_pattern, tokenizer, stop_words, stop_words_by_languages, tf_type)