code: def _preprocess_text(
      text: List[str],
      tokenizer: Any,
      max_length: int = 512,
      truncation: bool = True,
      sort_according_length: bool = True,
      own_tokenizer: bool = False,
  ) -> Tuple[Dict[str, Tensor], Optional[Tensor]]:
      """Text pre-processing function using `transformers` `AutoTokenizer` instance.
  
      Args:
          text:
              An iterable of sentences.
          tokenizer:
              Either `AutoTokenizer` instance from `transformers` package, or a user's own tokenizer.
          max_length:
              A maximum sequence length.
          truncation:
              An indication of whether tokenized sequences should be padded only to the length of the longest sequence.
          sort_according_length:
              An indication of whether tokenized sequences should be sorted from shortest to longest. This is appropriate
              to do for leveraging dynamic padding during embedding calculation and thereby to hasten inference.
          own_tokenizer:
              An indication of whether a non-default user's own tokenizer is used.
  
      Return:
          A dictionary of tokenized sentences including input_ids and attention_mask.
  
      Raises:
          BaseException:
              If a tokenization with a user's own tokenizer is not successful.
  
      """
      if not own_tokenizer:
          tokenized_data = tokenizer(
              text,
              padding="max_length",
              max_length=max_length,
              truncation=truncation,
              return_tensors="pt",
          )
      else:
          try:
              tokenized_data = tokenizer(text, max_length)
          except BaseException as ex:
              raise RuntimeError(
                  f"Tokenization was not successful: {ex}"
              ) from ex
  
      if sort_according_length:
          (
              input_ids,
              attention_mask,
              sorting_indices,
          ) = _sort_data_according_length(
              tokenized_data["input_ids"],
              tokenized_data["attention_mask"],
          )
          input_dict = {
              "input_ids": input_ids,
              "attention_mask": attention_mask,
          }
      else:
          input_dict = {
              "input_ids": tokenized_data["input_ids"],
              "attention_mask": tokenized_data["attention_mask"],
          }
          sorting_indices = None
  
      return input_dict, sorting_indices
  
imports: import math
  import os
  from collections import Counter, defaultdict
  from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple, Union
  import torch
  from torch import Tensor
  from torch.utils.data import DataLoader, Dataset
  from torchmetrics.utilities.data import _cumsum
  from torchmetrics.utilities.imports import _TQDM_AVAILABLE, _TRANSFORMERS_GREATER_EQUAL_4_4
label_desc: Use image node_id 5 for calling a global function with example usage: _preprocess_text(text, tokenizer, max_length, truncation, sort_according_length, own_tokenizer) and returns: input_dict,  sorting_indices