code: def process_ner_annotations(
      ner_annotations, ner_text, entity_map, tokenizer, is_eval=False
  ):
      """
      Generate token-level/word-level labels with given text and NER annotations.
  
      Parameters
      ----------
      ner_annotations
          The NER annotations.
      ner_text
          The corresponding raw text.
      entity_map
          The map between tags and tag indexes. e.g., {"PER":2, "LOC":3}.
      tokenizer
          The tokenizer to be used.
      is_eval
          Whether it is for evaluation or not, default: False
  
      Returns
      -------
      Token-level/word-level labels and text features.
      """
      (
          col_tokens,
          token_to_word_mappings,
          word_offsets,
      ) = tokenize_ner_text(ner_text, tokenizer)
      num_words = len(set(token_to_word_mappings)) - 1
      word_label = [1] * num_words
      # TODO: Potentially optimize word label generation via binary search
      b_prefix = "B-"
      i_prefix = "I-"
      for annot in ner_annotations:
          custom_offset = annot[0]
          custom_label = annot[1]
          is_start_word = True
          for idx, word_offset in enumerate(
              word_offsets[:num_words, :]
          ):
              # support multiple words in an annotated offset range.
              # Allow partial overlapping between custom annotations and pretokenized words.
              if (word_offset[0] < custom_offset[1]) and (
                  custom_offset[0] < word_offset[1]
              ):
                  if not (
                      re.match(b_prefix, custom_label, re.IGNORECASE)
                      or re.match(i_prefix, custom_label, re.IGNORECASE)
                  ):
                      if (
                          is_start_word
                          and b_prefix + custom_label in entity_map
                      ):
                          word_label[idx] = entity_map[
                              b_prefix + custom_label
                          ]
                          is_start_word = False
                      elif i_prefix + custom_label in entity_map:
                          word_label[idx] = entity_map[
                              i_prefix + custom_label
                          ]
                  else:
                      if custom_label in entity_map:
                          word_label[idx] = entity_map[custom_label]
  
      token_label = [0] * len(col_tokens.input_ids)
      temp = set()
      counter = 0
      for idx, token_to_word in enumerate(token_to_word_mappings):
          if token_to_word != -1 and token_to_word not in temp:
              temp.add(token_to_word)
              token_label[idx] = word_label[counter]
              counter += 1
      if not is_eval:
          label = token_label  # return token-level labels for training
      else:
          label = word_label  # return word-level labels for evaluation
  
      return label, col_tokens, token_to_word_mappings, word_offsets
  
imports: import ast
  import codecs
  import copy
  import re
  import warnings
  from io import BytesIO
  from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union
  import numpy
  import pandas
  import PIL
  from omegaconf import ListConfig
  from text_unidecode import unidecode
  from timm.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD
  from tokenizers import pre_tokenizers
  from torchvision import transforms
  from ..constants import CLIP_IMAGE_MEAN, CLIP_IMAGE_STD, IDENTIFIER, IMAGE, IMAGE_BYTEARRAY, IMAGE_PATH, MMDET_IMAGE, MMLAB_MODELS
  from .collator import DictCollator
  from .preprocess_dataframe import MultiModalFeaturePreprocessor
  from .randaug import RandAugment
  from .trivial_augmenter import TrivialAugment
label_desc: Use image node_id 8 for calling a global function with example usage: process_ner_annotations(ner_annotations, ner_text, entity_map, tokenizer, is_eval) and returns: label,  col_tokens,  token_to_word_mappings,  word_offsets