code: def tokenize_ner_text(text, tokenizer):
      """
      Tokenization process for the NER task. It will be used for the token-level label generation
      and the input text tokenization.
  
      Parameters
      ----------
      text
          The raw text data.
      tokenizer
          The tokenizer to be used.
  
      Returns
      -------
      The output of tokenizer and word offsets.
      """
      # pre-tokenization is required for NER token-level label generation.
      words_with_offsets = (
          pre_tokenizers.BertPreTokenizer().pre_tokenize_str(text)
      )
      words_with_offsets = (
          is_space_counted(words_with_offsets)
          if len(words_with_offsets) > 1
          else words_with_offsets
      )
      words = [word for word, offset in words_with_offsets]
      word_offsets = np.array(
          [
              [offset[0], offset[1]]
              for word, offset in words_with_offsets
          ],
          dtype=np.int32,
      )
      col_tokens = tokenizer(
          words,
          is_split_into_words=True,
          return_offsets_mapping=True,
          padding="max_length",
          truncation=True,
          max_length=tokenizer.model_max_length,
          return_token_type_ids=True,
      )
      offset_mapping = np.array(
          col_tokens.offset_mapping, dtype=np.int32
      )
      if len(words_with_offsets) > 1:
          if offset_mapping.shape[0] > len(words):
              word_offsets = np.pad(
                  word_offsets,
                  ((0, offset_mapping.shape[0] - len(words)), (0, 0)),
                  "constant",
              )
          # token to word mappings: it will tell us which token belongs to which word.
          token_to_word_mappings = [
              i if i != None else -1 for i in col_tokens.word_ids()
          ]
          if len(set(token_to_word_mappings)) != len(words) + 1:
              warnings.warn(
                  f"The token to word mappings are incorrect!"
              )
      else:
          # If pre_tokenizer does not give word offsets, use word_ids and offset_mappings instead.
          word_offsets = np.append(offset_mapping[1:], [[0, 0]], axis=0)
          word_idx = np.arange(
              len(col_tokens.word_ids())
              - col_tokens.word_ids().count(None)
          )
          token_to_word_mappings = [
              val + word_idx[idx - 1] if val != None else -1
              for idx, val in enumerate(col_tokens.word_ids())
          ]
  
      return col_tokens, token_to_word_mappings, word_offsets
  
imports: import ast
  import codecs
  import copy
  import re
  import warnings
  from io import BytesIO
  from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union
  import numpy
  import pandas
  import PIL
  from omegaconf import ListConfig
  from text_unidecode import unidecode
  from timm.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD
  from tokenizers import pre_tokenizers
  from torchvision import transforms
  from ..constants import CLIP_IMAGE_MEAN, CLIP_IMAGE_STD, IDENTIFIER, IMAGE, IMAGE_BYTEARRAY, IMAGE_PATH, MMDET_IMAGE, MMLAB_MODELS
  from .collator import DictCollator
  from .preprocess_dataframe import MultiModalFeaturePreprocessor
  from .randaug import RandAugment
  from .trivial_augmenter import TrivialAugment
label_desc: Use image node_id 9 for calling a global function with example usage: tokenize_ner_text(text, tokenizer) and returns: col_tokens,  token_to_word_mappings,  word_offsets