code: def get_pp_dmvr_tokenize(
      tokenizer_type: str,
      vocab_path: Optional[str] = None,
      prepend_bos: bool = False,
      append_eos: bool = False,
      max_num_tokens: Optional[int] = 77,
  ) -> Callable[[tf.Tensor], tf.Tensor]:
      """PP op to tokenize a text with a DMVR tokenizer."""
      tokenizer = _create_dmvr_tokenizer(tokenizer_type, vocab_path)
  
      def _dmvr_tokenize_inner(
          text: tf.Tensor,  # Shape: () or (1,)
      ) -> tf.Tensor:  # Shape: (L,)
          if tf.rank(text) == 0:
              text = tf.reshape(text, (-1,))
  
          token_ids = (
              tokenizer.string_tensor_to_indices(  # Shape: (1, L)
                  text,
                  prepend_bos=prepend_bos,
                  append_eos=append_eos,
                  max_num_tokens=max_num_tokens,
              )
          )
          return tf.squeeze(token_ids, axis=0)
  
      return _dmvr_tokenize_inner
  
file_path: scenic/scenic/projects/lang4video/pp_ops.py
imports: from collections.abc import Callable
  import functools
  from typing import Optional
  from dmvr import processors
  from dmvr import tokenizers
  import ml_collections
  from scenic.dataset_lib.big_transfer.preprocessing import utils
  from scenic.dataset_lib.big_transfer.registry import Registry
  from scenic.projects.lang4video import util
  import tensorflow
label_desc: Use image node_id 2 for calling a global function with example usage: get_pp_dmvr_tokenize(tokenizer_type, vocab_path, prepend_bos, append_eos, max_num_tokens) and returns: _dmvr_tokenize_inner, tf