code: def _text_to_ids(self, Xs, pad_token=None):
      Xs = self._format_for_encoding(Xs)
      if self.config.chunk_long_sequences and len(Xs) == 1:
          # can only chunk single sequence inputs
          encoded = self.text_encoder.encode_multi_input(
              Xs,
              max_length=sys.maxsize,
              remove_repeated_whitespace=self.config.collapse_whitespace,
              include_bos_eos=self.config.include_bos_eos,
          )
          length = len(encoded.token_ids)
          field_starts_and_ends = dict()
          for field in EncodedOutput._fields:
              field_value = getattr(encoded, field)
              if field_value is not None and len(field_value):
                  field_starts_and_ends[field] = (
                      field_value[0],
                      field_value[-1],
                  )
          if (
              self.config.chunk_context == 0
              and self.config.add_eos_bos_to_chunk
          ):
              warnings.warn(
                  """Chunk context of 0 will not capture the start
                                and end tokens added by add_eos_bos_to_chunk"""
              )
          for (
              start,
              end,
              (useful_start, useful_end),
          ) in self.chunker.generate_chunks(length):
              d = dict()
              for field in EncodedOutput._fields:
                  field_value = getattr(encoded, field)
                  if field_value is not None:
                      fv = field_value[start:end]
                      if self.config.add_eos_bos_to_chunk:
                          (
                              start_token,
                              end_token,
                          ) = field_starts_and_ends[field]
                          if fv[0] != start_token:
                              fv = np.concatenate(([start_token], fv))
                              # Update start and end only once
                              if field == EncodedOutput._fields[0]:
                                  useful_start += 1
                                  useful_end += 1
                          if fv[-1] != end_token:
                              fv = np.concatenate((fv, [end_token]))
                      d[field] = fv
              yield EncodedOutput(
                  useful_start=useful_start,
                  useful_end=useful_end,
                  input_text=Xs,
                  **d,
              )
      else:
          encoder_out = self.text_encoder.encode_multi_input(
              Xs,
              max_length=self.config.max_length,
              remove_repeated_whitespace=self.config.collapse_whitespace,
              include_bos_eos=self.config.include_bos_eos,
          )
  
          d = dict()
          for field in EncodedOutput._fields:
              field_value = getattr(encoder_out, field)
              if field_value is not None:
                  d[field] = field_value
  
          yield EncodedOutput(input_text=Xs, **d)
  
imports: import itertools
  import logging
  import sys
  import math
  import os
  import warnings
  from collections.abc import Iterable
  from collections import Counter
  from abc import ABCMeta, abstractmethod
  import tqdm
  import numpy
  import pandas
  import tensorflow
  from tensorflow.python.data import Dataset
  from sklearn.model_selection import train_test_split
  from sklearn.preprocessing import LabelBinarizer
  from sklearn.utils import shuffle
  import finetune
  from finetune.errors import FinetuneError
  from finetune.encoding.input_encoder import EncodedOutput, tokenize_context
  from finetune.util.imbalance import compute_class_weights
  from finetune.util.input_utils import InputMode, validation_settings, wrap_tqdm, Chunker, has_targets, batch_dataset
label_desc: Use image node_id 19 for calling the BasePipeline obj's underlying member method code with example usage: obj._text_to_ids(Xs, pad_token) without return types