code: def tokenize_with_metadata(text, tokenizer):
      """
      Performing tokenization while storing some important metadata for each token:
  
      * offsets: (int) Character index where the token begins in the original text
      * start_of_word: (bool) If the token is the start of a word. Particularly helpful for NER and QA tasks.
  
      We do this by first doing whitespace tokenization and then applying the model specific tokenizer to each "word".
  
      .. note::  We don't assume to preserve exact whitespaces in the tokens!
                 This means: tabs, new lines, multiple whitespace etc will all resolve to a single " ".
                 This doesn't make a difference for BERT + XLNet but it does for RoBERTa.
                 For RoBERTa it has the positive effect of a shorter sequence length, but some information about whitespace
                 type is lost which might be helpful for certain NLP tasks ( e.g tab for tables).
  
      :param text: Text to tokenize
      :type text: str
      :param tokenizer: Tokenizer (e.g. from Tokenizer.load())
      :return: Dictionary with "tokens", "offsets" and "start_of_word"
      :rtype: dict
  
      """
      # normalize all other whitespace characters to " "
      # Note: using text.split() directly would destroy the offset,
      # since \n\n\n would be treated similarly as a single \n
      text = re.sub(r"\s", " ", text)
      # Fast Tokenizers return offsets, so we don't need to calculate them ourselves
      if tokenizer.is_fast:
          # tokenized = tokenizer(text, return_offsets_mapping=True, return_special_tokens_mask=True)
          tokenized2 = tokenizer.encode_plus(
              text,
              return_offsets_mapping=True,
              return_special_tokens_mask=True,
          )
  
          tokens2 = tokenized2["input_ids"]
          offsets2 = np.array(
              [x[0] for x in tokenized2["offset_mapping"]]
          )
          # offsets2 = [x[0] for x in tokenized2["offset_mapping"]]
          words = np.array(tokenized2.encodings[0].words)
  
          # TODO check for validity for all tokenizer and special token types
          words[0] = -1
          words[-1] = words[-2]
          words += 1
          start_of_word2 = [0] + list(np.ediff1d(words))
          #######
  
          # start_of_word3 = []
          # last_word = -1
          # for word_id in tokenized2.encodings[0].words:
          #     if word_id is None or word_id == last_word:
          #         start_of_word3.append(0)
          #     else:
          #         start_of_word3.append(1)
          #         last_word = word_id
  
          tokenized_dict = {
              "tokens": tokens2,
              "offsets": offsets2,
              "start_of_word": start_of_word2,
          }
      else:
          # split text into "words" (here: simple whitespace tokenizer).
          words = text.split(" ")
          word_offsets = []
          cumulated = 0
          for idx, word in enumerate(words):
              word_offsets.append(cumulated)
              cumulated += (
                  len(word) + 1
              )  # 1 because we so far have whitespace tokenizer
  
          # split "words" into "subword tokens"
          tokens, offsets, start_of_word = _words_to_tokens(
              words, word_offsets, tokenizer
          )
  
          tokenized_dict = {
              "tokens": tokens,
              "offsets": offsets,
              "start_of_word": start_of_word,
          }
  
      return tokenized_dict
  
imports: from __future__ import absolute_import, division, print_function, unicode_literals
  import collections
  import json
  import logging
  import os
  import re
  from pathlib import Path
  import numpy
  from transformers import AlbertTokenizer, AlbertTokenizerFast, BertTokenizer, BertTokenizerFast, DistilBertTokenizer, DistilBertTokenizerFast, ElectraTokenizer, ElectraTokenizerFast, RobertaTokenizer, RobertaTokenizerFast, XLMRobertaTokenizer, XLMRobertaTokenizerFast, XLNetTokenizer, XLNetTokenizerFast, CamembertTokenizer, CamembertTokenizerFast, DPRContextEncoderTokenizer, DPRContextEncoderTokenizerFast, DPRQuestionEncoderTokenizer, DPRQuestionEncoderTokenizerFast, BigBirdTokenizer, BigBirdTokenizerFast
  from transformers.models.bert.tokenization_bert import load_vocab
  from transformers.tokenization_utils import PreTrainedTokenizer
  from transformers import AutoConfig
  from farm.data_handler.samples import SampleBasket
  from farm.modeling.wordembedding_utils import load_from_cache, EMBEDDING_VOCAB_FILES_MAP, run_split_on_punc
label_desc: Use image node_id 1 for calling a global function with example usage: tokenize_with_metadata(text, tokenizer) and returns: tokenized_dict