code: def clip_tokenize(
      batch,
      rmv_full_stop,
      vqa_options_for_ce=False,
      num_responses=5,
      include_verb=False,
  ):
      """Tokenize the raw string using CLIP tokenizers.
  
      Args:
        batch: batch
        rmv_full_stop: remove full stop (they occur in hard negs)
        vqa_options_for_ce: having cross entropy loss
        num_responses: default 5 as one correct, 4 wrong in MSR-VTT MC.
        include_verb: whether the verb phrase should be included.
  
      Returns:
        batch: batch
      """
  
      if "text" not in batch:
          pos_strings = batch["string_pos"].reshape(
              (-1,) + batch["string_pos"].shape[2:]
          )
          neg_strings = batch["string_neg"].reshape(
              (-1,) + batch["string_neg"].shape[2:]
          )
          num_hard_neg = batch["string_neg"].shape[1]
          raw_text = []
          for idx in range(len(pos_strings)):
              raw_text.append(pos_strings[idx])
              raw_text += list(
                  neg_strings[
                      idx * num_hard_neg : (idx * num_hard_neg)
                      + num_hard_neg
                  ]
              )
          del batch["text_indices_pos"]
          del batch["text_indices_neg"]
          del batch["string_neg"]
          del batch["string_pos"]
      else:
          raw_text = batch["text"]
          raw_text = raw_text.reshape((-1,) + raw_text.shape[2:])
          del batch["text"]
      raw_text = [sentence.decode("utf-8") for sentence in raw_text]
      verbs, raw_verbs = False, False
      if include_verb and "verb" in batch:
          verbs = True
          raw_verbs = batch["verb"]
          raw_verbs = raw_verbs.reshape((-1,) + raw_verbs.shape[2:])
          del batch["verb"]
          raw_verbs = [verb.decode("utf-8") for verb in raw_verbs]
      if rmv_full_stop:
          raw_text = [sent.replace(".", "") for sent in raw_text]
      if vqa_options_for_ce:
          clean_raw_text = []
          for i in raw_text:
              if i:
                  options = i.split(";")[:num_responses]
                  # a few examples have only four answers
                  if len(options) == 4:
                      options += [""]
                  clean_raw_text.append(options)
          raw_text = np.concatenate(np.asarray(clean_raw_text))
      tokenizer = clip_tokenizer.build_tokenizer(truncate=True)
      text_tokens = tokenizer(raw_text)
      text_tokens = jnp.asarray(text_tokens)
      text_tokens = jnp.expand_dims(text_tokens, 1)
      text_masking = [int(not i) for i in raw_text]
      text_masking = jnp.asarray(text_masking)
      text_masking = jnp.expand_dims(text_masking, 1)
      batch["text_mask"] = text_masking
      batch["text_indices"] = text_tokens
      if include_verb and verbs:
          verb_tokens = tokenizer(raw_verbs)
          verb_tokens = jnp.asarray(verb_tokens)
          verb_tokens = jnp.expand_dims(verb_tokens, 1)
          verb_masking = [int(bool(i)) for i in raw_verbs]
          verb_masking = jnp.asarray(verb_masking)
          verb_masking = jnp.expand_dims(verb_masking, 1)
          batch["verb_mask"] = verb_masking
          batch["verb_indices"] = verb_tokens
      return batch
  
imports: import functools
  from typing import Dict, Iterator, List, Optional, Text, Tuple, Union
  from absl import logging
  from dmvr import modalities
  from flax import jax_utils
  import jax
  import jax.numpy
  import numpy
  from scenic.dataset_lib import dataset_utils
  from scenic.dataset_lib import datasets
  from scenic.dataset_lib import video_ops
  from scenic.projects.baselines.clip import tokenizer
  from scenic.projects.vivit.data import video_tfrecord_dataset
  import tensorflow
  from dmvr import tokenizers
label_desc: Use image node_id 4 for calling a global function with example usage: clip_tokenize(batch, rmv_full_stop, vqa_options_for_ce, num_responses, include_verb) and returns: batch