class_docstr: Prepare text data for the model specified by "prefix". For multiple models requiring text data,
  we need to create a TextProcessor for each related model so that they will have independent input.
class_docstr_tok: ["Prepare","text","data","for","the","model","specified","by","``","prefix","''",".","For","multiple","models","requiring","text","data",",","we","need","to","create","a","TextProcessor","for","each","related","model","so","that","they","will","have","independent","input","."]
docstr: Construct one token sequence based on multiple token sequences coming from different
  text columns in a multimodal pd.DataFrame. The token sequence length and the text segment
  id are upper bounded by "self.max_len" and "self.text_segment_num".
  
  Parameters
  ----------
  text_tokens
      One sample's text token sequences from different text columns in a multimodal pd.DataFrame.
  
  Returns
  -------
  A dictionary containing one sample's text tokens, valid length, and segment ids.
docstr_tok: ["Construct","one","token","sequence","based","on","multiple","token","sequences","coming","from","different","text","columns","in","a","multimodal","pd.DataFrame",".","The","token","sequence","length","and","the","text","segment","id","are","upper","bounded","by","``","self.max_len","''","and","``","self.text_segment_num","''",".","Parameters","--","--","--","--","--","text_tokens","One","sample","'s","text","token","sequences","from","different","text","columns","in","a","multimodal","pd.DataFrame",".","Returns","--","--","--","-","A","dictionary","containing","one","sample","'s","text","tokens",",","valid","length",",","and","segment","ids","."]
code: def build_one_token_sequence(
      self,
      text_tokens: Dict[str, NDArray],
  ) -> Dict:
      """
      Construct one token sequence based on multiple token sequences coming from different
      text columns in a multimodal pd.DataFrame. The token sequence length and the text segment
      id are upper bounded by "self.max_len" and "self.text_segment_num".
  
      Parameters
      ----------
      text_tokens
          One sample's text token sequences from different text columns in a multimodal pd.DataFrame.
  
      Returns
      -------
      A dictionary containing one sample's text tokens, valid length, and segment ids.
      """
      if self.insert_sep:
          max_length = self.max_len - (len(text_tokens) + 1)
      else:
          max_length = self.max_len - 2
      if self.eos_only:
          # For EOS-only, the tokens will be combined as
          # [Field1 Tokens] [EOS] [Field2 Tokens] [EOS] [Field3 Tokens] [EOS]
          # Otherwise, the tokens will be combined as
          # [CLS] [Field1 Tokens] [SEP] [Field2 Tokens] [SEP] [Field3 Tokens] [EOS]
          max_length += 1
      trimmed_lengths = self.get_trimmed_lengths(
          [len(txt_token) for txt_token in text_tokens.values()],
          max_length,
          do_merge=True,
      )
      seg = 0
      if self.eos_only:
          # There is no cls token in the EOS-only mode
          token_ids = []
      else:
          token_ids = [self.cls_token_id]
  
      choices_ids = []
      segment_ids = [seg]
      ret = {}
  
      for (col_name, txt_token), trim_length in zip(
          text_tokens.items(), trimmed_lengths
      ):
          if col_name == CHOICES_IDS:
              choices_ids = txt_token
              continue
          segment_start = len(token_ids)
          if self.stochastic_chunk:
              start_ptr = np.random.randint(
                  0, len(txt_token) - trim_length + 1
              )
          else:
              start_ptr = 0
          token_ids.extend(
              txt_token[start_ptr : (start_ptr + trim_length)].tolist()
          )
          segment_ids.extend([seg] * trim_length)
          if self.requires_column_info:
              # np.int64 corresponds to torch.LongTensor
              col_token_idxs = np.array(
                  [segment_start, segment_start + trim_length],
                  dtype=np.int64,
              )
              ret[
                  f"{self.text_column_prefix}_{col_name}"
              ] = col_token_idxs
          if self.insert_sep:
              token_ids.append(self.sep_token_id)
              segment_ids.append(seg)
          seg = (seg + 1) % self.text_segment_num
  
      if hasattr(self, "eos_token_id"):
          if token_ids[-1] != self.eos_token_id:
              token_ids.append(self.eos_token_id)
              segment_ids.append(seg)
      else:  # backward compatibility
          if token_ids[-1] != self.sep_token_id:
              token_ids.append(self.sep_token_id)
              segment_ids.append(seg)
  
      ret.update(
          {
              self.text_token_ids_key: np.array(
                  token_ids, dtype=np.int32
              ),
              self.text_valid_length_key: len(token_ids),
              self.text_segment_ids_key: np.array(
                  segment_ids, dtype=np.int32
              ),
              self.choices_ids_key: np.array(
                  choices_ids, dtype=np.int32
              ),
          }
      )
  
      return ret
  
imports: import ast
  import logging
  import os
  import warnings
  from copy import deepcopy
  from typing import Any, Dict, List, Optional, Union
  import numpy
  from nptyping import NDArray
  from omegaconf import DictConfig
  from torch import nn
  from ..constants import CHOICES_IDS, COLUMN, TEXT, TEXT_SEGMENT_IDS, TEXT_TOKEN_IDS, TEXT_VALID_LENGTH
  from .collator import PadCollator, StackCollator
  from .template_engine import TemplateEngine
  from .trivial_augmenter import TrivialAugment
  from .utils import extract_value_from_config, get_text_token_max_len, normalize_txt, register_encoding_decoding_error_handlers
label_desc: Use image node_id 8 for calling the TextProcessor obj's underlying member method code with example usage: obj.build_one_token_sequence(text_tokens) and returns: ret