class_docstr: Prepare text data for the model specified by "prefix". For multiple models requiring text data,
  we need to create a TextProcessor for each related model so that they will have independent input.
class_docstr_tok: ["Prepare","text","data","for","the","model","specified","by","``","prefix","''",".","For","multiple","models","requiring","text","data",",","we","need","to","create","a","TextProcessor","for","each","related","model","so","that","they","will","have","independent","input","."]
docstr: Extract the cls, sep, and eos token ids from a huggingface tokenizer. In most cases,
  we can use the attributes "cls_token_id" and "sep_token_id". But for CLIP, we
  need to use "bos_token_id" and "eos_token_id".
  
  Parameters
  ----------
  tokenizer
      A huggingface tokenizer instance.
  
  Returns
  -------
  The cls, sep, and eos token ids.
docstr_tok: ["Extract","the","cls",",","sep",",","and","eos","token","ids","from","a","huggingface","tokenizer",".","In","most","cases",",","we","can","use","the","attributes","``","cls_token_id","''","and","``","sep_token_id","''",".","But","for","CLIP",",","we","need","to","use","``","bos_token_id","''","and","``","eos_token_id","''",".","Parameters","--","--","--","--","--","tokenizer","A","huggingface","tokenizer","instance",".","Returns","--","--","--","-","The","cls",",","sep",",","and","eos","token","ids","."]
code: def get_special_tokens(tokenizer):
      """
      Extract the cls, sep, and eos token ids from a huggingface tokenizer. In most cases,
      we can use the attributes "cls_token_id" and "sep_token_id". But for CLIP, we
      need to use "bos_token_id" and "eos_token_id".
  
      Parameters
      ----------
      tokenizer
          A huggingface tokenizer instance.
  
      Returns
      -------
      The cls, sep, and eos token ids.
      """
      cls_id, sep_id, eos_id = (
          tokenizer.cls_token_id,
          tokenizer.sep_token_id,
          tokenizer.sep_token_id,
      )
      if cls_id is None or sep_id is None:
          # CLIP uses eos_token's feature as the pooled output.
          # See https://github.com/huggingface/transformers/blob/v4.14.1/src/transformers/models/clip/modeling_clip.py#L657
          cls_id, sep_id, eos_id = (
              tokenizer.bos_token_id,
              tokenizer.bos_token_id,
              tokenizer.eos_token_id,
          )
  
      if cls_id is None and sep_id is None:
          # Special treatment for T5 (EOS-only).
          cls_id = sep_id = eos_id
  
      if cls_id is None or sep_id is None or eos_id is None:
          raise ValueError(
              f"tokenizer class: {tokenizer.__class__.__name__} has no valid cls, sep, and eos ids."
          )
      return cls_id, sep_id, eos_id
  
imports: import ast
  import logging
  import os
  import warnings
  from copy import deepcopy
  from typing import Any, Dict, List, Optional, Union
  import numpy
  from nptyping import NDArray
  from omegaconf import DictConfig
  from torch import nn
  from ..constants import CHOICES_IDS, COLUMN, TEXT, TEXT_SEGMENT_IDS, TEXT_TOKEN_IDS, TEXT_VALID_LENGTH
  from .collator import PadCollator, StackCollator
  from .template_engine import TemplateEngine
  from .trivial_augmenter import TrivialAugment
  from .utils import extract_value_from_config, get_text_token_max_len, normalize_txt, register_encoding_decoding_error_handlers
label_desc: Use image node_id 10 for calling the TextProcessor obj's underlying member method code with example usage: obj.get_special_tokens(tokenizer) and returns: cls_id,  sep_id,  eos_id