class_docstr: Prepare text data for the model specified by "prefix". For multiple models requiring text data,
  we need to create a TextProcessor for each related model so that they will have independent input.
class_docstr_tok: ["Prepare","text","data","for","the","model","specified","by","``","prefix","''",".","For","multiple","models","requiring","text","data",",","we","need","to","create","a","TextProcessor","for","each","related","model","so","that","they","will","have","independent","input","."]
docstr: Parameters
  ----------
  model
      The model for which this processor would be created.
  max_len
      The maximum length of text tokens.
  insert_sep
      Whether to insert SEP tokens.
  text_segment_num
      The number of text segments.
  stochastic_chunk
      Whether to use stochastic chunking, which will randomly slice each individual text.
  requires_column_info
      Whether to require feature column information in dataloader.
  text_detection_length
      A naive way to detect text column versus tabular column that were treated as text
  text_trivial_aug_maxscale
      Used in trivial augment as the maximum scale that can be random generated
      A value of 0 means turn off trivial augment
      https://arxiv.org/pdf/2103.10158.pdf
  train_augment_types
      All possible augmentation operations
  normalize_text
      Whether to normalize text to resolve encoding problems.
      Examples of normalized texts can be found at
      https://github.com/autogluon/autogluon/tree/master/examples/automm/kaggle_feedback_prize#15-a-few-examples-of-normalized-texts
docstr_tok: ["Parameters","--","--","--","--","--","model","The","model","for","which","this","processor","would","be","created",".","max_len","The","maximum","length","of","text","tokens",".","insert_sep","Whether","to","insert","SEP","tokens",".","text_segment_num","The","number","of","text","segments",".","stochastic_chunk","Whether","to","use","stochastic","chunking",",","which","will","randomly","slice","each","individual","text",".","requires_column_info","Whether","to","require","feature","column","information","in","dataloader",".","text_detection_length","A","naive","way","to","detect","text","column","versus","tabular","column","that","were","treated","as","text","text_trivial_aug_maxscale","Used","in","trivial","augment","as","the","maximum","scale","that","can","be","random","generated","A","value","of","0","means","turn","off","trivial","augment","https",":","\/\/arxiv.org\/pdf\/2103.10158.pdf","train_augment_types","All","possible","augmentation","operations","normalize_text","Whether","to","normalize","text","to","resolve","encoding","problems",".","Examples","of","normalized","texts","can","be","found","at","https",":","\/\/github.com\/autogluon\/autogluon\/tree\/master\/examples\/automm\/kaggle_feedback_prize","#","15-a-few-examples-of-normalized-texts"]
code: def __init__(
      self,
      model: nn.Module,
      max_len: Optional[int] = None,
      insert_sep: Optional[bool] = True,
      text_segment_num: Optional[int] = 1,
      stochastic_chunk: Optional[bool] = False,
      requires_column_info: bool = False,
      text_detection_length: Optional[int] = None,
      text_trivial_aug_maxscale: Optional[float] = 0.0,
      train_augment_types: Optional[List[str]] = None,
      template_config: Optional[DictConfig] = None,
      normalize_text: Optional[bool] = False,
  ):
      """
      Parameters
      ----------
      model
          The model for which this processor would be created.
      max_len
          The maximum length of text tokens.
      insert_sep
          Whether to insert SEP tokens.
      text_segment_num
          The number of text segments.
      stochastic_chunk
          Whether to use stochastic chunking, which will randomly slice each individual text.
      requires_column_info
          Whether to require feature column information in dataloader.
      text_detection_length
          A naive way to detect text column versus tabular column that were treated as text
      text_trivial_aug_maxscale
          Used in trivial augment as the maximum scale that can be random generated
          A value of 0 means turn off trivial augment
          https://arxiv.org/pdf/2103.10158.pdf
      train_augment_types
          All possible augmentation operations
      normalize_text
          Whether to normalize text to resolve encoding problems.
          Examples of normalized texts can be found at
          https://github.com/autogluon/autogluon/tree/master/examples/automm/kaggle_feedback_prize#15-a-few-examples-of-normalized-texts
      """
      self.prefix = model.prefix
      self.requires_column_info = requires_column_info
      self.tokenizer_name = model.tokenizer_name
      # model should have a tokenizer
      self.tokenizer = model.tokenizer
      if hasattr(self.tokenizer, "deprecation_warnings"):
          # Disable the warning "Token indices sequence length is longer than the specified maximum sequence..."
          # See https://github.com/huggingface/transformers/blob/6ac77534bfe97c00e0127bb4fc846ae0faf1c9c5/src/transformers/tokenization_utils_base.py#L3362
          self.tokenizer.deprecation_warnings[
              "sequence-length-is-longer-than-the-specified-maximum"
          ] = True
  
      (
          self.cls_token_id,
          self.sep_token_id,
          self.eos_token_id,
      ) = self.get_special_tokens(tokenizer=self.tokenizer)
      self.max_len = get_text_token_max_len(
          provided_max_len=max_len,
          config=model.config,
          tokenizer=self.tokenizer,
          checkpoint_name=model.checkpoint_name,
      )
      logger.debug(f"text max length: {self.max_len}")
      self.insert_sep = insert_sep
      self.eos_only = (
          self.cls_token_id == self.sep_token_id == self.eos_token_id
      )
  
      extracted = extract_value_from_config(
          config=model.config.to_diff_dict(), keys=("type_vocab_size",)
      )
      if len(extracted) == 0:
          default_segment_num = 1
      elif len(extracted) == 1:
          default_segment_num = extracted[0]
      else:
          raise ValueError(
              f" more than one type_vocab_size values are detected: {extracted}"
          )
  
      if default_segment_num <= 0:
          default_segment_num = 1
  
      if text_segment_num < default_segment_num:
          warnings.warn(
              f"provided text_segment_num: {text_segment_num} "
              f"is smaller than {model.checkpoint_name}'s default: {default_segment_num}"
          )
      self.text_segment_num = min(text_segment_num, default_segment_num)
      assert self.text_segment_num >= 1
      logger.debug(f"text segment num: {self.text_segment_num}")
  
      self.stochastic_chunk = stochastic_chunk
      self.normalize_text = normalize_text
  
      # construct augmentor
      self.train_augment_types = train_augment_types
      self.text_detection_length = text_detection_length
      self.text_trivial_aug_maxscale = text_trivial_aug_maxscale
      self.train_augmenter = self.construct_text_augmenter(
          self.text_trivial_aug_maxscale, self.train_augment_types
      )
      self.template_config = template_config
      if self.template_config.turn_on:
          self.template_engine = TemplateEngine(self.template_config)
      else:
          self.template_engine = None
  
      if self.normalize_text:
          register_encoding_decoding_error_handlers()
  
imports: import ast
  import logging
  import os
  import warnings
  from copy import deepcopy
  from typing import Any, Dict, List, Optional, Union
  import numpy
  from nptyping import NDArray
  from omegaconf import DictConfig
  from torch import nn
  from ..constants import CHOICES_IDS, COLUMN, TEXT, TEXT_SEGMENT_IDS, TEXT_TOKEN_IDS, TEXT_VALID_LENGTH
  from .collator import PadCollator, StackCollator
  from .template_engine import TemplateEngine
  from .trivial_augmenter import TrivialAugment
  from .utils import extract_value_from_config, get_text_token_max_len, normalize_txt, register_encoding_decoding_error_handlers
label_desc: Use image node_id 1 to create a new TextProcessor object  with example: obj = TextProcessor(model, max_len, insert_sep, text_segment_num, stochastic_chunk, requires_column_info, text_detection_length, text_trivial_aug_maxscale, train_augment_types, template_config, normalize_text)